1use super::*;
16use crate::core::pools::{POOL_META_NAME, load_pool_meta_identity_observing};
17use crate::disk::{self, error::DiskError};
18use crate::services::rebalance::{REBAL_META_NAME, RebalStatus};
19use crate::set_disk::get_lock_acquire_timeout;
20use crate::storage_api_contracts::heal::HealOperations as _;
21use crate::storage_api_contracts::namespace::NamespaceLocking as _;
22use rustfs_common::mrf_channel::MrfDeleteMarkerPurge;
23use rustfs_lock::NamespaceLockGuard;
24use std::collections::BTreeSet;
25use tracing::trace;
26
27const LOG_COMPONENT_ECSTORE: &str = "ecstore";
28const LOG_SUBSYSTEM_HEAL: &str = "heal";
29const EVENT_HEAL_ABANDONED_PARTS: &str = "heal_abandoned_parts";
30const EVENT_HEAL_FORMAT_COMPLETED: &str = "heal_format_completed";
31const EVENT_HEAL_OBJECT_STARTED: &str = "heal_object_started";
32
33pub(crate) struct BucketHealScope {
35 pub(crate) bucket: String,
36 pub(crate) incarnation: uuid::Uuid,
37 pub(crate) store: Arc<ECStore>,
38 fence: super::BucketIncarnationFenceGuard,
39}
40
41impl BucketHealScope {
42 pub(crate) fn check(&self) -> disk::error::Result<()> {
43 if self.fence.is_lock_lost() {
44 return Err(DiskError::other("bucket heal incarnation fence was lost"));
45 }
46 Ok(())
47 }
48}
49
50tokio::task_local! {
51 static BUCKET_HEAL_SCOPE: Arc<BucketHealScope>;
52}
53
54pub(crate) fn bucket_heal_scope(bucket: &str) -> Option<Arc<BucketHealScope>> {
55 BUCKET_HEAL_SCOPE
56 .try_with(|scope| (scope.bucket == bucket).then(|| scope.clone()))
57 .ok()
58 .flatten()
59}
60
61pub(super) fn bucket_metadata_owner(object: &str) -> Option<&str> {
63 use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE};
64 let (prefix, rest) = object.split_once('/')?;
65 let (bucket, file) = rest.split_once('/')?;
66 (prefix == disk::BUCKET_META_PREFIX
67 && matches!(file, BUCKET_METADATA_FILE | BUCKET_INCARNATION_FILE)
68 && check_valid_bucket_name_strict(bucket).is_ok())
69 .then_some(bucket)
70}
71
72pub(crate) fn bucket_heal_scope_for_object(bucket: &str, object: &str) -> Option<Arc<BucketHealScope>> {
73 let owner = if bucket == RUSTFS_META_BUCKET {
74 bucket_metadata_owner(object)?
75 } else {
76 bucket
77 };
78 bucket_heal_scope(owner)
79}
80
81#[derive(Debug)]
84pub struct HealObjectAbsenceProof {
85 pub bucket: String,
86 pub object: String,
87 pub version_id: String,
88 pub bucket_incarnation_id: Uuid,
89 pub pool_index: Option<usize>,
90 pub set_index: Option<usize>,
91 pub locations: Vec<(usize, usize)>,
92 pub removed: bool,
93}
94
95#[derive(Debug)]
96pub struct HealObjectStorageResult {
97 pub item: HealResultItem,
98 pub error: Option<Error>,
99 pub absence: Option<HealObjectAbsenceProof>,
100}
101
102fn invalid_heal_pool_index(pool_idx: usize, pool_count: usize) -> Error {
103 StorageError::InvalidArgument(
104 "heal".to_string(),
105 "pool".to_string(),
106 format!("invalid heal pool index {pool_idx} for {pool_count} pools"),
107 )
108}
109
110fn is_pool_meta_object(bucket: &str, object: &str) -> bool {
111 bucket == RUSTFS_META_BUCKET && object == POOL_META_NAME
112}
113
114#[derive(Debug, Clone, Copy)]
115enum HealFormatPoolSkip {
116 Completed,
117 Retryable,
118}
119
120fn classify_heal_format_pool(
121 pool_idx: usize,
122 pool_cmd_line: &str,
123 pool_meta: &PoolMeta,
124 rebalance_meta: Option<&RebalanceMeta>,
125) -> Option<HealFormatPoolSkip> {
126 let Some(pool) = pool_meta.pools.get(pool_idx) else {
127 return Some(HealFormatPoolSkip::Retryable);
128 };
129
130 if pool.id != pool_idx || pool_cmd_line.is_empty() || pool.cmd_line.is_empty() || pool.cmd_line != pool_cmd_line {
131 return Some(HealFormatPoolSkip::Retryable);
132 }
133
134 if let Some(decommission) = pool.decommission.as_ref() {
135 if decommission.complete {
136 return Some(HealFormatPoolSkip::Completed);
137 }
138 if decommission.failed || decommission.canceled || decommission.queued || pool_meta.is_suspended(pool_idx) {
139 return Some(HealFormatPoolSkip::Retryable);
140 }
141 }
142
143 if let Some(meta) = rebalance_meta {
144 let Some(pool_stats) = meta.pool_stats.get(pool_idx) else {
145 return Some(HealFormatPoolSkip::Retryable);
146 };
147 if pool_stats.info.stopping || (pool_stats.participating && pool_stats.info.status == RebalStatus::Started) {
148 return Some(HealFormatPoolSkip::Retryable);
149 }
150 }
151
152 None
153}
154
155fn heal_format_pool_skip_error(skip: HealFormatPoolSkip) -> Error {
156 match skip {
157 HealFormatPoolSkip::Completed => StorageError::NoHealRequired,
158 HealFormatPoolSkip::Retryable => StorageError::SlowDown,
159 }
160}
161
162fn heal_format_fence_lost_error() -> Error {
163 StorageError::SlowDown
164}
165
166impl ECStore {
167 pub async fn purge_delete_marker_with_proof(
168 self: &Arc<Self>,
169 bucket: &str,
170 object: &str,
171 version_id: &str,
172 purge: &MrfDeleteMarkerPurge,
173 opts: &HealOpts,
174 ) -> Result<bool> {
175 if opts.dry_run || opts.no_lock || !purge.is_valid() {
176 return Err(StorageError::PreconditionFailed);
177 }
178 let version = Uuid::parse_str(version_id)
179 .ok()
180 .filter(|version| !version.is_nil())
181 .ok_or(StorageError::PreconditionFailed)?;
182 let (Some(pool_index), Some(set_index)) = (opts.pool, opts.set) else {
183 return Err(StorageError::PreconditionFailed);
184 };
185 let mut marker = rustfs_filemeta::MetaDeleteMarker::default();
186 let consumed = marker.unmarshal_msg(&purge.marker)?;
187 if usize::try_from(consumed).ok() != Some(purge.marker.len())
188 || marker.version_id != Some(version)
189 || marker.stable_identity() != purge.marker_identity
190 {
191 return Err(StorageError::PreconditionFailed);
192 }
193 let marker_info = marker.clone().into_fileinfo(bucket, object, false)?;
194 if !marker_info.is_canonical_delete_marker()
195 || marker_info.delete_marker_incarnation() != Some(purge.marker_incarnation_id)
196 {
197 return Err(StorageError::PreconditionFailed);
198 }
199
200 let current = self.bucket_incarnation_id_from_disk(bucket).await?;
201 if current.is_nil() {
202 return Err(StorageError::BucketNotFound(bucket.to_owned()));
203 }
204 let original = purge.bucket_incarnation_id;
205 let object = object.to_owned();
206 self.run_bucket_heal_at_incarnation(bucket, current, opts, move |store, bucket, _opts| async move {
207 let scope = bucket_heal_scope(&bucket).ok_or(StorageError::PreconditionFailed)?;
208 scope.check()?;
209 if original != current {
210 let retirement_store = crate::bucket::metadata_sys::object_store_if_initialized_in(&store.ctx)
211 .await
212 .ok_or(StorageError::PreconditionFailed)?;
213 if !crate::bucket::retirement::is_retired(retirement_store, &bucket, original).await? {
214 return Err(StorageError::PreconditionFailed);
215 }
216 }
217 let pool = store
218 .pools
219 .get(pool_index)
220 .ok_or_else(|| invalid_heal_pool_index(pool_index, store.pools.len()))?;
221 let set = pool.disk_set.get(set_index).cloned().ok_or_else(|| {
222 StorageError::InvalidArgument(
223 "heal".to_string(),
224 "set".to_string(),
225 format!("invalid heal set index {set_index} for pool {pool_index}"),
226 )
227 })?;
228 set.purge_delete_marker_exact(&bucket, &object, version, marker).await
229 })
230 .await
231 }
232
233 pub(super) async fn run_bucket_heal_at_incarnation<T, F, Fut>(
234 self: &Arc<Self>,
235 bucket: &str,
236 expected: uuid::Uuid,
237 opts: &HealOpts,
238 operation: F,
239 ) -> Result<T>
240 where
241 T: Send + 'static,
242 F: FnOnce(Arc<Self>, String, HealOpts) -> Fut + Send + 'static,
243 Fut: std::future::Future<Output = Result<T>> + Send + 'static,
244 {
245 if expected.is_nil() || self.ctx.lock_manager().is_disabled() {
246 return Err(Error::other(
247 "incarnation-bound bucket heal requires a valid identity and namespace locking",
248 ));
249 }
250 let fence = self.acquire_bucket_incarnation_fence(bucket, expected).await?;
252 let scope = Arc::new(BucketHealScope {
253 bucket: bucket.to_owned(),
254 incarnation: expected,
255 store: self.clone(),
256 fence,
257 });
258 let store = self.clone();
259 let bucket = bucket.to_owned();
260 let opts = *opts;
261 tokio::spawn(BUCKET_HEAL_SCOPE.scope(scope.clone(), async move {
264 scope.check().map_err(Error::from)?;
265 let result = operation(store, bucket, opts).await;
266 scope.check().map_err(Error::from)?;
267 result
268 }))
269 .await
270 .map_err(|error| Error::other_with_context("bucket heal owner task failed", error))?
271 }
272
273 pub async fn heal_bucket_at_incarnation(
274 self: &Arc<Self>,
275 bucket: &str,
276 expected: uuid::Uuid,
277 opts: &HealOpts,
278 ) -> Result<HealResultItem> {
279 self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
280 store.heal_bucket(&bucket, &opts).await
281 })
282 .await
283 }
284
285 pub async fn heal_local_bucket_at_incarnation(
286 self: &Arc<Self>,
287 bucket: &str,
288 expected: uuid::Uuid,
289 opts: &HealOpts,
290 fenced_pools: Vec<usize>,
291 pools: Option<Vec<usize>>,
292 ) -> Result<HealResultItem> {
293 self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
294 let peer =
295 crate::cluster::rpc::peer_s3_client::LocalPeerS3Client::new_with_instance_ctx(None, pools, store.ctx.clone());
296 crate::cluster::rpc::peer_s3_client::PeerS3Client::heal_bucket_with_fence(&peer, &bucket, &opts, &fenced_pools)
297 .await
298 .map_err(Error::from)
299 })
300 .await
301 }
302
303 pub async fn heal_object_at_incarnation(
304 self: &Arc<Self>,
305 bucket: &str,
306 object: &str,
307 version_id: &str,
308 expected: uuid::Uuid,
309 opts: &HealOpts,
310 ) -> Result<HealObjectStorageResult> {
311 let object = object.to_owned();
312 let version_id = version_id.to_owned();
313 self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
314 store.heal_object_with_proof(&bucket, &object, &version_id, &opts).await
315 })
316 .await
317 }
318
319 pub async fn heal_mrf_object_at_incarnation(
320 self: &Arc<Self>,
321 bucket: &str,
322 object: &str,
323 version_id: &str,
324 expected: uuid::Uuid,
325 opts: &HealOpts,
326 ) -> Result<HealObjectStorageResult> {
327 let object = object.to_owned();
328 let version_id = version_id.to_owned();
329 self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
330 store
331 .heal_object_with_authoritative_absence_proof(&bucket, &object, &version_id, &opts)
332 .await
333 })
334 .await
335 }
336
337 async fn acquire_heal_format_fence(
338 &self,
339 ) -> Result<(
340 tokio::sync::MutexGuard<'_, PoolMetaWriteState>,
341 NamespaceLockGuard,
342 NamespaceLockGuard,
343 PoolMeta,
344 Option<RebalanceMeta>,
345 )> {
346 let metadata_pool = self
347 .pools
348 .first()
349 .cloned()
350 .ok_or_else(|| Error::other("heal format requires at least one storage pool"))?;
351 let mut write_state = self.pool_meta_save_gate.lock().await;
352 write_state.ensure_write_safe("heal format fence failed")?;
353
354 let pool_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
360 let pool_guard = pool_lock.get_read_lock(get_lock_acquire_timeout()).await?;
361 let rebalance_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
362 let rebalance_guard = rebalance_lock.get_write_lock(get_lock_acquire_timeout()).await?;
363
364 if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
365 return Err(heal_format_fence_lost_error());
366 }
367
368 load_pool_meta_identity_observing(self.pools.clone(), &mut write_state).await?;
369 let mut pool_meta = PoolMeta::default();
370 let replica_state = pool_meta
371 .load_no_lock_from_replicas_observing(self.pools.clone(), &mut write_state)
372 .await?;
373 write_state.observe_replicas(replica_state);
374 write_state.ensure_missing_metadata_can_initialize()?;
375 write_state.ensure_write_safe("heal format fence failed")?;
376 if pool_meta.pools.len() != self.pools.len()
377 || pool_meta.pools.iter().enumerate().any(|(pool_idx, pool)| {
378 pool.id != pool_idx || pool.cmd_line.is_empty() || pool.cmd_line != self.pools[pool_idx].endpoints.cmd_line
379 })
380 {
381 return Err(heal_format_fence_lost_error());
382 }
383
384 let mut rebalance_meta = RebalanceMeta::new();
385 let rebalance_meta = match rebalance_meta
386 .load_with_opts(
387 metadata_pool,
388 ObjectOptions {
389 no_lock: true,
390 ..Default::default()
391 },
392 )
393 .await
394 {
395 Ok(()) => Some(rebalance_meta),
396 Err(Error::ConfigNotFound) => None,
397 Err(err) => return Err(err),
398 };
399
400 if rebalance_meta
401 .as_ref()
402 .is_some_and(|meta| meta.pool_stats.len() != self.pools.len())
403 {
404 return Err(heal_format_fence_lost_error());
405 }
406
407 write_state.ensure_write_safe("heal format fence failed")?;
408 if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
409 return Err(heal_format_fence_lost_error());
410 }
411
412 Ok((write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta))
413 }
414
415 fn get_pools_for_heal_object(&self, opts: &HealOpts) -> Result<Vec<Arc<Sets>>> {
416 match opts.pool {
417 Some(pool_idx) => Ok(vec![
418 self.pools
419 .get(pool_idx)
420 .cloned()
421 .ok_or_else(|| invalid_heal_pool_index(pool_idx, self.pools.len()))?,
422 ]),
423 None => Ok(self.pools.clone()),
424 }
425 }
426
427 pub fn replacement_pool_metadata_required(&self, pool_index: usize, set_index: usize) -> Result<bool> {
431 let pool = self
432 .pools
433 .get(pool_index)
434 .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
435 let target_set = pool.get_disks_for_heal_object(
436 POOL_META_NAME,
437 &HealOpts {
438 set: Some(set_index),
439 ..Default::default()
440 },
441 )?;
442 Ok(Arc::ptr_eq(&target_set, &pool.get_disks_by_key(POOL_META_NAME)))
443 }
444
445 pub async fn heal_replacement_bucket_metadata(
447 self: &Arc<Self>,
448 bucket: &str,
449 opts: &HealOpts,
450 targets: &[String],
451 ) -> Result<()> {
452 use crate::bucket::metadata::{BUCKET_INCARNATION_FILE, BUCKET_METADATA_FILE};
453 use crate::bucket::metadata_sys::acquire_bucket_metadata_transaction_read_lock_in;
454
455 let (Some(pool_index), Some(set_index)) = (opts.pool, opts.set) else {
456 return Err(Error::PreconditionFailed);
457 };
458 if targets.is_empty() || opts.dry_run || opts.no_lock || check_valid_bucket_name_strict(bucket).is_err() {
459 return Err(Error::PreconditionFailed);
460 }
461 let pool = self
462 .pools
463 .get(pool_index)
464 .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
465 let target_set = pool.get_disks_for_heal_object(BUCKET_METADATA_FILE, opts)?;
466 if targets.iter().any(|target| {
467 target_set
468 .set_endpoints
469 .iter()
470 .filter(|endpoint| endpoint.to_string() == *target)
471 .count()
472 != 1
473 }) {
474 return Err(Error::PreconditionFailed);
475 }
476 let objects: Vec<_> = [BUCKET_METADATA_FILE, BUCKET_INCARNATION_FILE]
477 .into_iter()
478 .map(|file| format!("{}/{bucket}/{file}", disk::BUCKET_META_PREFIX))
479 .filter(|object| Arc::ptr_eq(&target_set, &pool.get_disks_by_key(object)))
480 .collect();
481 if objects.is_empty() {
482 return Ok(());
483 }
484
485 let incarnation = self.bucket_incarnation_id_from_disk(bucket).await?;
488 let targets = targets.to_vec();
489 self.run_bucket_heal_at_incarnation(bucket, incarnation, opts, move |store, bucket, opts| async move {
490 let transaction = acquire_bucket_metadata_transaction_read_lock_in(&store.ctx, &bucket).await?;
494 for object in objects {
495 let scope = bucket_heal_scope(&bucket).ok_or(Error::PreconditionFailed)?;
496 scope.check()?;
497 if transaction.is_lock_lost() {
498 return Err(Error::PreconditionFailed);
499 }
500 if !store.single_pool() {
503 let (owner, _) = store
504 .get_pool_info_for_delete_marker(RUSTFS_META_BUCKET, &object, &ObjectOptions::default())
505 .await?;
506 if owner.index != pool_index {
507 continue;
508 }
509 }
510 let metadata_opts = HealOpts { remove: false, ..opts };
511 let (result, error) = store.heal_object(RUSTFS_META_BUCKET, &object, "", &metadata_opts).await?;
512 if let Some(error) = error {
513 return Err(error);
514 }
515 for target in &targets {
516 let mut drives = result.after.drives.iter().filter(|drive| &drive.endpoint == target);
517 if !drives.next().is_some_and(|drive| drive.state == "ok") || drives.next().is_some() {
518 return Err(Error::PreconditionFailed);
519 }
520 }
521 if !store
522 .replacement_targets_have_version(RUSTFS_META_BUCKET, &object, "", pool_index, set_index, &targets)
523 .await?
524 {
525 return Err(Error::PreconditionFailed);
526 }
527 }
528 if transaction.is_lock_lost() {
529 return Err(Error::PreconditionFailed);
530 }
531 Ok(())
532 })
533 .await
534 }
535
536 pub async fn heal_erasure_set_scopes(&self, opts: &HealOpts) -> Result<Vec<(usize, usize)>> {
538 let pools = self.get_pools_for_heal_object(opts)?;
539 let pool_meta = self.pool_meta.read().await;
540 let mut scopes = Vec::new();
541
542 for pool in pools {
543 let suspended_complete = pool_meta.is_suspended(pool.pool_idx).then(|| {
544 pool_meta
545 .pools
546 .get(pool.pool_idx)
547 .and_then(|status| status.decommission.as_ref())
548 .is_some_and(|decommission| decommission.complete)
549 });
550 if let Some(complete) = suspended_complete {
551 if opts.pool.is_some() {
552 return Err(if complete {
553 StorageError::InvalidArgument(
554 "heal".to_string(),
555 "pool".to_string(),
556 format!("heal pool {} has completed decommission", pool.pool_idx),
557 )
558 } else {
559 Error::SlowDown
560 });
561 }
562 continue;
563 }
564
565 if let Some(set_idx) = opts.set {
566 if set_idx >= pool.disk_set.len() {
567 return Err(StorageError::InvalidArgument(
568 "heal".to_string(),
569 "set".to_string(),
570 format!(
571 "invalid heal set index {set_idx} for pool {} with {} sets",
572 pool.pool_idx,
573 pool.disk_set.len()
574 ),
575 ));
576 }
577 scopes.push((pool.pool_idx, set_idx));
578 } else {
579 scopes.extend((0..pool.disk_set.len()).map(|set_idx| (pool.pool_idx, set_idx)));
580 }
581 }
582
583 if scopes.is_empty() {
584 return Err(Error::SlowDown);
585 }
586
587 Ok(scopes)
588 }
589
590 #[instrument(skip(self))]
591 pub(super) async fn handle_heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
592 let mut r = HealResultItem {
593 heal_item_type: HealItemType::Metadata.to_string(),
594 detail: "disk-format".to_string(),
595 ..Default::default()
596 };
597
598 let mut count_no_heal = 0;
599 let mut count_completed = 0;
600 let mut first_error = None;
601 for (pool_idx, pool) in self.pools.iter().enumerate() {
602 let (mut write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta) =
603 self.acquire_heal_format_fence().await?;
604 if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
605 first_error.get_or_insert(heal_format_fence_lost_error());
606 break;
607 }
608 if let Some(skip) = classify_heal_format_pool(pool_idx, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref())
609 {
610 if matches!(skip, HealFormatPoolSkip::Completed) {
611 count_completed += 1;
612 } else {
613 first_error.get_or_insert(heal_format_pool_skip_error(skip));
614 }
615 continue;
616 }
617
618 let fence_lost = || {
619 let lost = pool_guard.is_lock_lost()
620 || rebalance_guard.is_lock_lost()
621 || write_state.ensure_write_safe("heal format write fence failed").is_err();
622 if lost {
623 write_state.block_writes_after_fence_loss();
624 }
625 lost
626 };
627 let (mut result, err) = pool.heal_format_with_fence(dry_run, fence_lost).await?;
628 if let Some(err) = err {
629 match err {
630 StorageError::NoHealRequired => {
631 count_no_heal += 1;
632 }
633 err => {
634 first_error.get_or_insert(err);
635 }
636 }
637 }
638 r.disk_count += result.disk_count;
639 r.set_count += result.set_count;
640 r.before.drives.append(&mut result.before.drives);
641 r.after.drives.append(&mut result.after.drives);
642
643 let fence_lost = pool_guard.is_lock_lost()
646 || rebalance_guard.is_lock_lost()
647 || write_state.ensure_write_safe("heal format publication fence failed").is_err();
648 if fence_lost {
649 write_state.block_writes_after_fence_loss();
650 first_error.get_or_insert(heal_format_fence_lost_error());
651 break;
652 }
653 }
654 if let Some(err) = first_error {
655 return Ok((r, Some(err)));
656 }
657 if count_no_heal + count_completed == self.pools.len() {
658 info!(
659 event = EVENT_HEAL_FORMAT_COMPLETED,
660 component = LOG_COMPONENT_ECSTORE,
661 subsystem = LOG_SUBSYSTEM_HEAL,
662 dry_run,
663 result = "no_heal_required",
664 pool_count = self.pools.len(),
665 "Heal format completed"
666 );
667 return Ok((r, Some(StorageError::NoHealRequired)));
668 }
669 info!(
670 event = EVENT_HEAL_FORMAT_COMPLETED,
671 component = LOG_COMPONENT_ECSTORE,
672 subsystem = LOG_SUBSYSTEM_HEAL,
673 dry_run,
674 result = "healed_or_inspected",
675 disk_count = r.disk_count,
676 set_count = r.set_count,
677 "Heal format completed"
678 );
679 Ok((r, None))
680 }
681
682 #[instrument(skip(self, targets), fields(pool_index, set_index, target_count = targets.len()))]
683 pub async fn heal_replacement_format(
684 &self,
685 dry_run: bool,
686 pool_index: usize,
687 set_index: usize,
688 targets: &[String],
689 ) -> Result<(HealResultItem, Option<Error>)> {
690 let pool = self
691 .pools
692 .get(pool_index)
693 .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
694 let set = pool.disk_set.get(set_index).cloned().ok_or_else(|| {
695 StorageError::InvalidArgument(
696 "heal".to_string(),
697 "set".to_string(),
698 format!("invalid heal set index {set_index} for pool {pool_index}"),
699 )
700 })?;
701
702 let (mut write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta) = self.acquire_heal_format_fence().await?;
703 if let Some(skip) = classify_heal_format_pool(pool_index, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref()) {
704 return Ok((HealResultItem::default(), Some(heal_format_pool_skip_error(skip))));
705 }
706
707 let fence_lost = || {
708 let lost = pool_guard.is_lock_lost()
709 || rebalance_guard.is_lock_lost()
710 || write_state
711 .ensure_write_safe("replacement format write fence failed")
712 .is_err();
713 if lost {
714 write_state.block_writes_after_fence_loss();
715 }
716 lost
717 };
718 let result = set.heal_replacement_format_with_fence(dry_run, targets, fence_lost).await?;
719 let fence_lost = pool_guard.is_lock_lost()
720 || rebalance_guard.is_lock_lost()
721 || write_state
722 .ensure_write_safe("replacement format publication fence failed")
723 .is_err();
724 if fence_lost {
725 write_state.block_writes_after_fence_loss();
726 return Ok((result.0, Some(heal_format_fence_lost_error())));
727 }
728 Ok(result)
729 }
730
731 #[instrument(skip(self, targets), fields(pool_index, set_index, target_count = targets.len()))]
732 pub async fn replacement_targets_have_version(
733 &self,
734 bucket: &str,
735 object: &str,
736 version_id: &str,
737 pool_index: usize,
738 set_index: usize,
739 targets: &[String],
740 ) -> Result<bool> {
741 let pool = self
742 .pools
743 .get(pool_index)
744 .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
745 let set = pool.disk_set.get(set_index).cloned().ok_or_else(|| {
746 StorageError::InvalidArgument(
747 "heal".to_string(),
748 "set".to_string(),
749 format!("invalid heal set index {set_index} for pool {pool_index}"),
750 )
751 })?;
752
753 set.replacement_targets_have_version(bucket, object, version_id, targets)
754 .await
755 .map_err(Into::into)
756 }
757
758 #[instrument(skip(self))]
759 pub(super) async fn handle_heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
760 let movement_gate = self.ctx.data_movement_operation_gate();
761 let _movement_guard = movement_gate.read().await;
762 let save_guard = self.pool_meta_save_gate.lock().await;
763 save_guard.ensure_write_safe("bucket heal cannot run while pool metadata requires recovery")?;
764 let mut fenced_pools = BTreeSet::new();
765 {
766 let pool_meta = self.pool_meta.read().await;
767 fenced_pools.extend((0..pool_meta.pools.len()).filter(|pool_idx| pool_meta.is_suspended(*pool_idx)));
768 if let Some(pool_idx) = opts.pool {
769 if pool_idx >= pool_meta.pools.len() {
770 return Err(invalid_heal_pool_index(pool_idx, pool_meta.pools.len()));
771 }
772 if pool_meta.is_suspended(pool_idx) {
773 let complete = pool_meta.pools[pool_idx]
774 .decommission
775 .as_ref()
776 .is_some_and(|decommission| decommission.complete);
777 return Err(if complete {
778 StorageError::InvalidArgument(
779 "heal".to_string(),
780 "pool".to_string(),
781 format!("heal pool {pool_idx} has completed decommission"),
782 )
783 } else {
784 Error::SlowDown
785 });
786 }
787 }
788 }
789
790 let dispatch_fenced_pools = fenced_pools.iter().copied().collect::<Vec<_>>();
791 drop(save_guard);
792 let mut res = self
793 .peer_sys
794 .heal_bucket_with_fence_from_movement_guarded_coordinator(bucket, opts, &dispatch_fenced_pools)
795 .await?;
796 {
797 let pool_meta = self.pool_meta.read().await;
798 fenced_pools.extend((0..pool_meta.pools.len()).filter(|pool_idx| pool_meta.is_suspended(*pool_idx)));
799 }
800 if !fenced_pools.is_empty() {
801 let pools = fenced_pools.iter().map(usize::to_string).collect::<Vec<_>>().join(", ");
802 res.detail = format!("skipped: bucket-volume heal fenced on decommission-suspended pool(s): {pools}");
803 }
804
805 Ok(res)
806 }
807
808 #[instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
809 pub(super) async fn handle_heal_object(
810 &self,
811 bucket: &str,
812 object: &str,
813 version_id: &str,
814 opts: &HealOpts,
815 ) -> Result<(HealResultItem, Option<Error>)> {
816 self.handle_heal_object_with_absence(
817 bucket,
818 object,
819 version_id,
820 opts,
821 &mut None,
822 crate::set_disk::AbsenceProofRequest {
823 retirement: None,
824 allow_unversioned: false,
825 },
826 )
827 .await
828 }
829
830 pub async fn heal_object_with_proof(
831 &self,
832 bucket: &str,
833 object: &str,
834 version_id: &str,
835 opts: &HealOpts,
836 ) -> Result<HealObjectStorageResult> {
837 self.heal_object_with_absence_proof(bucket, object, version_id, opts, false)
838 .await
839 }
840
841 async fn heal_object_with_authoritative_absence_proof(
842 &self,
843 bucket: &str,
844 object: &str,
845 version_id: &str,
846 opts: &HealOpts,
847 ) -> Result<HealObjectStorageResult> {
848 self.heal_object_with_absence_proof(bucket, object, version_id, opts, true)
849 .await
850 }
851
852 async fn heal_object_with_absence_proof(
853 &self,
854 bucket: &str,
855 object: &str,
856 version_id: &str,
857 opts: &HealOpts,
858 allow_unversioned_absence: bool,
859 ) -> Result<HealObjectStorageResult> {
860 if opts.dry_run || opts.no_lock || super::utils::is_reserved_or_invalid_bucket(bucket, false) {
864 let (item, error) = self.handle_heal_object(bucket, object, version_id, opts).await?;
865 return Ok(HealObjectStorageResult {
866 item,
867 error,
868 absence: None,
869 });
870 }
871
872 let scope = bucket_heal_scope(bucket).filter(|scope| std::ptr::eq(scope.store.as_ref(), self));
877 let guard = if let Some(scope) = &scope {
878 scope.check().map_err(Error::from)?;
879 None
880 } else {
881 Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
882 };
883 let fence_is_lost = || {
884 scope.as_ref().is_some_and(|scope| scope.fence.is_lock_lost())
885 || guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
886 };
887 let lifecycle_guard = scope
888 .as_ref()
889 .and_then(|scope| scope.fence.namespace_lock_guard())
890 .or(guard.as_ref())
891 .ok_or_else(|| Error::other("bucket heal requires a held lifecycle guard"))?;
892 let retirement = crate::bucket::retirement::MarkerRetirementContext {
893 store: crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await,
894 current_incarnation: if let Some(scope) = &scope {
895 Some(scope.incarnation)
896 } else {
897 self.bucket_incarnation_id_from_disk(bucket)
898 .await
899 .ok()
900 .filter(|id| !id.is_nil())
901 },
902 lifecycle_guard,
903 };
904 let mut proofs = None;
905 let (item, mut error) = self
906 .handle_heal_object_with_absence(
907 bucket,
908 object,
909 version_id,
910 opts,
911 &mut proofs,
912 crate::set_disk::AbsenceProofRequest {
913 retirement: Some(&retirement),
914 allow_unversioned: allow_unversioned_absence,
915 },
916 )
917 .await?;
918 let incarnation = if proofs.is_some() && !fence_is_lost() {
921 if let Some(scope) = &scope {
922 Some(scope.incarnation)
923 } else {
924 self.bucket_incarnation_id_from_disk(bucket)
925 .await
926 .ok()
927 .filter(|id| !id.is_nil())
928 }
929 } else {
930 None
931 };
932 let absence = match (incarnation, proofs) {
933 (Some(incarnation), Some(proofs)) if !fence_is_lost() => Some(HealObjectAbsenceProof {
934 bucket: bucket.to_owned(),
935 object: object.to_owned(),
936 version_id: version_id.to_owned(),
937 bucket_incarnation_id: incarnation,
938 pool_index: opts.pool,
939 set_index: opts.set,
940 removed: proofs.iter().any(|proof| proof.removed),
941 locations: proofs.into_iter().map(|proof| (proof.pool_index, proof.set_index)).collect(),
942 }),
943 _ => None,
944 };
945 if absence.is_some() {
946 error = None;
947 }
948 Ok(HealObjectStorageResult { item, error, absence })
949 }
950
951 async fn handle_heal_object_with_absence(
952 &self,
953 bucket: &str,
954 object: &str,
955 version_id: &str,
956 opts: &HealOpts,
957 absence: &mut Option<Vec<crate::set_disk::HealedObjectAbsence>>,
958 proof: crate::set_disk::AbsenceProofRequest<'_>,
959 ) -> Result<(HealResultItem, Option<Error>)> {
960 trace!(
961 event = EVENT_HEAL_OBJECT_STARTED,
962 component = LOG_COMPONENT_ECSTORE,
963 subsystem = LOG_SUBSYSTEM_HEAL,
964 bucket = %bucket,
965 object = %object,
966 version_id = %version_id,
967 remove = opts.remove,
968 scan_mode = ?opts.scan_mode,
969 "Heal object started"
970 );
971 let object = encode_dir_object(object);
972
973 *absence = None;
974 let pools = self.get_pools_for_heal_object(opts)?;
975 let requested_pool_count = pools.len();
976 if let Some(set_idx) = opts.set {
977 for pool in &pools {
978 if set_idx >= pool.disk_set.len() {
979 let err = StorageError::InvalidArgument(
980 "heal".to_string(),
981 "set".to_string(),
982 format!(
983 "invalid heal set index {set_idx} for pool {} with {} sets",
984 pool.pool_idx,
985 pool.disk_set.len()
986 ),
987 );
988 if opts.pool.is_some() {
989 return Err(err);
990 }
991 return Ok((HealResultItem::default(), Some(err)));
992 }
993 }
994 }
995 #[cfg(test)]
996 let store_id = self.id;
997
998 let mut heal_pools = Vec::with_capacity(pools.len());
999 for pool in pools.iter() {
1000 let suspended_complete = {
1001 let pool_meta = self.pool_meta.read().await;
1002 pool_meta.is_suspended(pool.pool_idx).then(|| {
1003 pool_meta
1004 .pools
1005 .get(pool.pool_idx)
1006 .and_then(|status| status.decommission.as_ref())
1007 .is_some_and(|decommission| decommission.complete)
1008 })
1009 };
1010 if let Some(complete) = suspended_complete {
1011 if opts.pool.is_some() {
1012 let _ = pool.get_disks_for_heal_object(&object, opts)?;
1013 let err = if complete {
1014 StorageError::InvalidArgument(
1015 "heal".to_string(),
1016 "pool".to_string(),
1017 format!("heal pool {} has completed decommission", pool.pool_idx),
1018 )
1019 } else {
1020 Error::SlowDown
1021 };
1022 return Ok((HealResultItem::default(), Some(err)));
1023 }
1024 continue;
1025 }
1026 heal_pools.push(Arc::clone(pool));
1027 }
1028 let results = if is_pool_meta_object(bucket, &object) && !opts.no_lock && !heal_pools.is_empty() {
1029 let target_pool_indices = heal_pools.iter().map(|pool| pool.pool_idx).collect::<Vec<_>>();
1030 match self.acquire_pool_meta_object_heal_fence(&target_pool_indices).await {
1031 Ok((pool_meta_guard, admissions)) => {
1032 let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first()).cloned();
1033 let futures = heal_pools.iter().zip(admissions).map(|(pool, admission)| {
1034 let pool = Arc::clone(pool);
1035 let pool_object = object.clone();
1036 let fixed_set = fixed_set.clone();
1037 let mut opts = *opts;
1038 async move {
1039 admission?;
1040 let fixed_set = fixed_set.ok_or_else(|| Error::other("pool metadata heal requires a fixed set"))?;
1041 let target_set = pool.get_disks_for_heal_object(&pool_object, &opts)?;
1042 opts.no_lock = fixed_set.shares_namespace_lock_domain(&target_set).await;
1043 #[cfg(test)]
1044 if !opts.no_lock {
1045 crate::core::pools::notify_decommission_external_heal_target_lock_attempted();
1046 }
1047 #[cfg(test)]
1048 crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
1049 pool.heal_object_with_absence(bucket, &pool_object, version_id, &opts, proof)
1050 .await
1051 }
1052 });
1053 let results = join_all(futures).await;
1054 drop(pool_meta_guard);
1055 results
1056 }
1057 Err(err) => (0..heal_pools.len()).map(|_| Err(err.clone())).collect(),
1058 }
1059 } else {
1060 let mut futures = Vec::with_capacity(heal_pools.len());
1061 for pool in heal_pools {
1062 let pool_idx = pool.pool_idx;
1063 let pool_object = object.clone();
1064 let opts = *opts;
1065 futures.push(self.run_external_decommission_capacity_heal(
1066 pool_idx,
1067 bucket,
1068 &object,
1069 opts,
1070 move |opts| async move {
1071 #[cfg(test)]
1072 crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
1073 pool.heal_object_with_absence(bucket, &pool_object, version_id, &opts, proof)
1074 .await
1075 },
1076 ));
1077 }
1078 join_all(futures).await
1079 };
1080
1081 let mut proofs = Vec::with_capacity(requested_pool_count);
1082 let mut errs = Vec::with_capacity(self.pools.len());
1083 let mut ress = Vec::with_capacity(self.pools.len());
1084
1085 for res in results.into_iter() {
1086 match res {
1087 Ok((result, err, proof)) => {
1088 if let Some(proof) = proof
1089 && (err.is_none()
1090 || err
1091 .as_ref()
1092 .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)))
1093 {
1094 proofs.push(proof);
1095 }
1096 let mut result = result;
1097 result.object = decode_dir_object(&result.object);
1098 ress.push(result);
1099 errs.push(err);
1100 }
1101 Err(err) => {
1102 errs.push(Some(err));
1103 ress.push(HealResultItem::default());
1104 }
1105 }
1106 }
1107
1108 if requested_pool_count > 0 && proofs.len() == requested_pool_count {
1111 *absence = Some(proofs);
1112 }
1113
1114 for (idx, err) in errs.iter().enumerate() {
1115 if err.is_none() {
1116 return Ok((ress.remove(idx), None));
1117 }
1118 }
1119
1120 for (index, err) in errs.iter().enumerate() {
1122 return match err {
1123 Some(err) => {
1124 if is_err_object_not_found(err) || is_err_version_not_found(err) {
1125 continue;
1126 }
1127 Ok((ress.remove(index), Some(err.clone())))
1128 }
1129 None => Ok((ress.remove(index), None)),
1130 };
1131 }
1132
1133 if !version_id.is_empty() {
1135 return Ok((HealResultItem::default(), Some(Error::FileVersionNotFound)));
1136 }
1137
1138 Ok((HealResultItem::default(), Some(Error::FileNotFound)))
1139 }
1140
1141 #[instrument(skip(self))]
1142 pub(super) async fn handle_check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
1143 let object = encode_dir_object(object);
1144 let pools = self.get_pools_for_heal_object(opts)?;
1145
1146 let mut futures = Vec::with_capacity(pools.len());
1147 for pool in pools.iter() {
1148 futures.push(pool.check_abandoned_parts(bucket, &object, opts));
1149 }
1150
1151 let mut first_error = None;
1152 for result in join_all(futures).await {
1153 if let Err(err) = result
1154 && first_error.is_none()
1155 {
1156 first_error = Some(err);
1157 }
1158 }
1159
1160 if let Some(err) = first_error {
1161 return Err(err);
1162 }
1163
1164 trace!(
1165 event = EVENT_HEAL_ABANDONED_PARTS,
1166 component = LOG_COMPONENT_ECSTORE,
1167 subsystem = LOG_SUBSYSTEM_HEAL,
1168 state = "completed",
1169 result = "ok",
1170 bucket,
1171 object,
1172 dry_run = opts.dry_run,
1173 "Heal abandoned parts completed"
1174 );
1175
1176 Ok(())
1177 }
1178}
1179
1180#[cfg(test)]
1181mod tests {
1182 use super::*;
1183 use crate::bucket::metadata_sys;
1184 use crate::cluster::rpc::PeerS3Client;
1185 use crate::config::com::{delete_config, read_config_no_lock_preserve_empty_with_metadata, save_config};
1186 use crate::core::pools::{
1187 DecommissionCapacityAdmission, DecommissionCapacityLockOrderBarrier, DecommissionErasureLayout,
1188 DecommissionPoolCapacityInfo, POOL_META_IDENTITY_NAME, PoolDecommissionInfo, PoolMetaReplicaState, PoolStatus,
1189 initialized_pool_meta_identity_for_test, set_decommission_capacity_info_overrides_for_test,
1190 };
1191 use crate::core::sets::HealFormatAfterSaveBarrier;
1192 use crate::disk::error::Result as DiskResult;
1193 use crate::disk::{DeleteOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, format::FormatV3, new_disk};
1194 use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
1195 use crate::runtime::instance::InstanceContext;
1196 use crate::services::rebalance::{
1197 RebalanceInfo, RebalanceStats, test_three_pool_stores_with_isolated_node_contexts, test_two_pool_stores,
1198 };
1199 use crate::storage_api_contracts::bucket::{
1200 BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
1201 };
1202 use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations};
1203 use crate::store::init_format::{load_format_erasure, save_format_file};
1204 use crate::store::init_local_disks_with_instance_ctx;
1205 use rustfs_heal_contracts::heal_channel::DriveState;
1206 use tokio_util::sync::CancellationToken;
1207
1208 #[derive(Debug)]
1209 struct BlockingHealPeer {
1210 started: Arc<tokio::sync::Notify>,
1211 release: Arc<tokio::sync::Notify>,
1212 }
1213
1214 #[async_trait::async_trait]
1215 impl PeerS3Client for BlockingHealPeer {
1216 async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> DiskResult<HealResultItem> {
1217 self.started.notify_one();
1218 self.release.notified().await;
1219 Ok(HealResultItem::default())
1220 }
1221
1222 async fn make_bucket(&self, _bucket: &str, _opts: &MakeBucketOptions) -> DiskResult<()> {
1223 Ok(())
1224 }
1225
1226 async fn list_bucket(&self, _opts: &BucketOptions) -> DiskResult<Vec<BucketInfo>> {
1227 Ok(Vec::new())
1228 }
1229
1230 async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> DiskResult<()> {
1231 Ok(())
1232 }
1233
1234 async fn get_bucket_info(&self, _bucket: &str, _opts: &BucketOptions) -> DiskResult<BucketInfo> {
1235 Ok(BucketInfo::default())
1236 }
1237
1238 fn get_pools(&self) -> Option<Vec<usize>> {
1239 Some(vec![0, 1])
1240 }
1241 }
1242
1243 #[derive(Debug)]
1244 struct WriterQueuedLocalHealPeer {
1245 movement_gate: Arc<tokio::sync::RwLock<()>>,
1246 writer_queued: Arc<tokio::sync::Notify>,
1247 writer_acquired: Arc<tokio::sync::Notify>,
1248 }
1249
1250 #[async_trait::async_trait]
1251 impl PeerS3Client for WriterQueuedLocalHealPeer {
1252 async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> DiskResult<HealResultItem> {
1253 let _movement_guard = self
1254 .movement_gate
1255 .try_read()
1256 .map_err(|_| crate::disk::error::DiskError::TooManyOpenFiles)?;
1257 Ok(HealResultItem::default())
1258 }
1259
1260 async fn heal_bucket_with_fence_from_movement_guarded_coordinator(
1261 &self,
1262 _bucket: &str,
1263 _opts: &HealOpts,
1264 _fenced_pools: &[usize],
1265 ) -> DiskResult<HealResultItem> {
1266 Ok(HealResultItem::default())
1267 }
1268
1269 async fn make_bucket(&self, _bucket: &str, _opts: &MakeBucketOptions) -> DiskResult<()> {
1270 Ok(())
1271 }
1272
1273 async fn list_bucket(&self, _opts: &BucketOptions) -> DiskResult<Vec<BucketInfo>> {
1274 Ok(Vec::new())
1275 }
1276
1277 async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> DiskResult<()> {
1278 Ok(())
1279 }
1280
1281 async fn get_bucket_info(&self, _bucket: &str, _opts: &BucketOptions) -> DiskResult<BucketInfo> {
1282 let movement_gate = self.movement_gate.clone();
1283 let writer_acquired = self.writer_acquired.clone();
1284 tokio::spawn(async move {
1285 let _movement_guard = movement_gate.write().await;
1286 writer_acquired.notify_one();
1287 });
1288 while self.movement_gate.try_read().is_ok() {
1289 tokio::task::yield_now().await;
1290 }
1291 self.writer_queued.notify_one();
1292 Ok(BucketInfo::default())
1293 }
1294
1295 fn get_pools(&self) -> Option<Vec<usize>> {
1296 Some(vec![0, 1])
1297 }
1298 }
1299
1300 async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
1301 let format = FormatV3::new(1, 1);
1302 let endpoint_url = format!("http://127.0.0.1:{}/data", 19000 + pool_idx);
1303 let mut endpoint = Endpoint::try_from(endpoint_url.as_str()).expect("endpoint should parse");
1304 endpoint.set_pool_index(pool_idx);
1305 endpoint.set_set_index(0);
1306 endpoint.set_disk_index(0);
1307
1308 Sets::new(
1309 vec![None],
1310 &PoolEndpoints {
1311 legacy: false,
1312 set_count: 1,
1313 drives_per_set: 1,
1314 endpoints: Endpoints::from(vec![endpoint]),
1315 cmd_line: String::new(),
1316 platform: String::new(),
1317 },
1318 &format,
1319 pool_idx,
1320 0,
1321 )
1322 .await
1323 .expect("minimal pool should build")
1324 }
1325
1326 #[tokio::test]
1327 #[serial_test::serial]
1328 async fn bucket_incarnation_heal_preserves_successor_shards_and_allows_fresh_repair() {
1329 let (_root, store, shutdown) = multi_pool_heal_store().await;
1330 let bucket = format!("heal-incarnation-{}", Uuid::new_v4().simple());
1331 store
1332 .make_bucket(&bucket, &MakeBucketOptions::default())
1333 .await
1334 .expect("original bucket");
1335 let old = store
1336 .bucket_incarnation_id_from_disk(&bucket)
1337 .await
1338 .expect("original identity");
1339 store
1340 .delete_bucket(&bucket, &DeleteBucketOptions::default())
1341 .await
1342 .expect("normal bucket deletion");
1343 store
1344 .make_bucket(&bucket, &MakeBucketOptions::default())
1345 .await
1346 .expect("same-name successor");
1347 let new = store
1348 .bucket_incarnation_id_from_disk(&bucket)
1349 .await
1350 .expect("successor identity");
1351 assert_ne!(old, new);
1352 let object = "successor-version";
1353 let version = Uuid::new_v4().to_string();
1354 let set = store.pools[0].get_disks(0);
1355 let mut reader = PutObjReader::from_vec(vec![17; 512 * 1024]);
1356 set.put_object(
1357 &bucket,
1358 object,
1359 &mut reader,
1360 &ObjectOptions {
1361 versioned: true,
1362 version_id: Some(version.clone()),
1363 write_completion: crate::object_api::WriteCompletion::TailDrained,
1365 ..Default::default()
1366 },
1367 )
1368 .await
1369 .expect("write successor version");
1370 let missing = set.disks.read().await[0].clone().expect("missing target");
1371 let healthy = set.disks.read().await[1].clone().expect("healthy target");
1372 missing
1373 .delete(
1374 &bucket,
1375 object,
1376 DeleteOptions {
1377 recursive: true,
1378 immediate: true,
1379 ..Default::default()
1380 },
1381 )
1382 .await
1383 .expect("remove one successor shard");
1384 let baseline = healthy
1385 .read_all(&bucket, &format!("{object}/xl.meta"))
1386 .await
1387 .expect("healthy baseline");
1388 let opts = HealOpts {
1389 pool: Some(0),
1390 set: Some(0),
1391 scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
1392 ..Default::default()
1393 };
1394 let (_, error) = store
1396 .heal_object(&bucket, object, &version, &opts)
1397 .await
1398 .expect("unbound control");
1399 assert!(error.is_none());
1400 assert!(missing.read_xl(&bucket, object, false).await.is_ok());
1401 missing
1402 .delete(
1403 &bucket,
1404 object,
1405 DeleteOptions {
1406 recursive: true,
1407 immediate: true,
1408 ..Default::default()
1409 },
1410 )
1411 .await
1412 .expect("restore the same missing-shard condition");
1413 let fi = healthy
1414 .read_version("", &bucket, object, &version, &Default::default())
1415 .await
1416 .expect("successor metadata");
1417 let disk_ref = healthy.endpoint().to_string();
1418 assert!(
1419 store
1420 .write_local_metadata_at_incarnation(&disk_ref, (&bucket, object), fi.clone(), old)
1421 .await
1422 .is_err()
1423 );
1424 assert!(
1425 store
1426 .delete_local_path_at_incarnation(
1427 &disk_ref,
1428 (&bucket, object),
1429 DeleteOptions {
1430 recursive: true,
1431 immediate: true,
1432 ..Default::default()
1433 },
1434 old
1435 )
1436 .await
1437 .is_err()
1438 );
1439 assert!(
1440 store
1441 .delete_local_version_at_incarnation(
1442 &disk_ref,
1443 (&bucket, object),
1444 fi.clone(),
1445 false,
1446 DeleteOptions::default(),
1447 old
1448 )
1449 .await
1450 .is_err()
1451 );
1452 assert!(
1453 store
1454 .rename_local_data_at_incarnation(&disk_ref, (&bucket, object), &fi, (&bucket, "stale-destination"), old)
1455 .await
1456 .is_err()
1457 );
1458 for file in [".metadata.bin", ".bucket-incarnation"] {
1459 let metadata = format!("buckets/{bucket}/{file}");
1460 assert!(
1461 store
1462 .rename_local_data_at_incarnation(&disk_ref, (&bucket, object), &fi, (RUSTFS_META_BUCKET, &metadata), old)
1463 .await
1464 .is_err(),
1465 "stale remote heal must not replace the successor's {file}"
1466 );
1467 }
1468 assert!(store.heal_bucket_at_incarnation(&bucket, old, &opts).await.is_err());
1469 assert!(
1470 store
1471 .heal_object_at_incarnation(&bucket, object, &version, old, &opts)
1472 .await
1473 .is_err()
1474 );
1475 assert!(
1476 missing.read_xl(&bucket, object, false).await.is_err(),
1477 "obsolete admission must not publish a shard"
1478 );
1479 assert_eq!(
1480 healthy
1481 .read_all(&bucket, &format!("{object}/xl.meta"))
1482 .await
1483 .expect("healthy after stale heal"),
1484 baseline
1485 );
1486 store
1487 .heal_bucket_at_incarnation(&bucket, new, &opts)
1488 .await
1489 .expect("fresh bucket metadata repair");
1490 let result = store
1491 .heal_object_at_incarnation(&bucket, object, &version, new, &opts)
1492 .await
1493 .expect("new admission repair");
1494 assert!(result.error.is_none(), "new admission must repair independently: {:?}", result.error);
1495 assert!(
1496 missing.read_xl(&bucket, object, false).await.is_ok(),
1497 "new admission publishes the missing shard"
1498 );
1499 store
1500 .write_local_metadata_at_incarnation(&disk_ref, (&bucket, object), fi.clone(), new)
1501 .await
1502 .expect("fresh target metadata");
1503 store
1504 .delete_local_version_at_incarnation(&disk_ref, (&bucket, object), fi, false, DeleteOptions::default(), new)
1505 .await
1506 .expect("fresh target version cleanup");
1507 healthy
1508 .write_all(&bucket, "fresh-cleanup/data", bytes::Bytes::from_static(b"temporary"))
1509 .await
1510 .expect("cleanup fixture");
1511 store
1512 .delete_local_path_at_incarnation(
1513 &disk_ref,
1514 (&bucket, "fresh-cleanup"),
1515 DeleteOptions {
1516 recursive: true,
1517 immediate: true,
1518 ..Default::default()
1519 },
1520 new,
1521 )
1522 .await
1523 .expect("fresh target path cleanup");
1524 shutdown.cancel();
1525 }
1526
1527 #[tokio::test]
1528 #[serial_test::serial]
1529 async fn bucket_incarnation_heal_owner_survives_cancelled_waiter_and_queued_writer() {
1530 let (_root, store, shutdown) = multi_pool_heal_store().await;
1531 let bucket = format!("heal-owner-{}", Uuid::new_v4().simple());
1532 store
1533 .make_bucket(&bucket, &MakeBucketOptions::default())
1534 .await
1535 .expect("bucket");
1536 let identity = store.bucket_incarnation_id_from_disk(&bucket).await.expect("identity");
1537 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
1538 let (release_tx, release_rx) = tokio::sync::oneshot::channel();
1539 let (drained_tx, drained_rx) = tokio::sync::oneshot::channel();
1540 let worker_store = store.clone();
1541 let worker_bucket = bucket.clone();
1542 let waiter = tokio::spawn(async move {
1543 worker_store
1544 .run_bucket_heal_at_incarnation(
1545 &worker_bucket,
1546 identity,
1547 &HealOpts::default(),
1548 move |store, bucket, opts| async move {
1549 started_tx.send(()).expect("announce held generation");
1550 release_rx.await.expect("release commit");
1551 let result = store
1552 .heal_object_with_proof(&bucket, "history.txt", &Uuid::new_v4().to_string(), &opts)
1553 .await?;
1554 let proof = result.absence.expect("absence proof must reuse the held lifecycle owner");
1555 assert_eq!(proof.bucket_incarnation_id, identity);
1556 assert_eq!(proof.locations.len(), 2);
1557 drained_tx.send(()).expect("announce drain");
1558 Ok(())
1559 },
1560 )
1561 .await
1562 });
1563 started_rx.await.expect("operation acquired generation fence");
1564 waiter.abort();
1565 assert!(waiter.await.expect_err("waiter aborted").is_cancelled());
1566 let writer_store = store.clone();
1567 let writer_bucket = bucket.clone();
1568 let mut writer = Box::pin(writer_store.acquire_bucket_lifecycle_write_lock(&writer_bucket));
1569 assert!(
1570 futures::poll!(&mut writer).is_pending(),
1571 "cancellation must not release the live operation's generation"
1572 );
1573 release_tx.send(()).expect("finish operation");
1574 tokio::time::timeout(std::time::Duration::from_secs(10), drained_rx)
1575 .await
1576 .expect("proof must not reacquire a lifecycle read lock behind the queued writer")
1577 .expect("physical operation drains");
1578 let guard = tokio::time::timeout(std::time::Duration::from_secs(10), writer)
1579 .await
1580 .expect("writer resumes after drain")
1581 .expect("lifecycle writer");
1582 drop(guard);
1583 shutdown.cancel();
1584 }
1585
1586 async fn minimal_heal_store() -> ECStore {
1587 ECStore {
1588 id: Uuid::new_v4(),
1589 disk_map: HashMap::new(),
1590 pools: vec![minimal_heal_pool(0).await, minimal_heal_pool(1).await],
1591 peer_sys: S3PeerSys {
1592 clients: Vec::new(),
1593 pools_count: 2,
1594 },
1595 pool_meta: RwLock::new(PoolMeta::default()),
1596 rebalance_meta: RwLock::new(None),
1597 decommission_cancelers: RwLock::new(Vec::new()),
1598 start_gate: Mutex::new(()),
1599 pool_meta_save_gate: Mutex::default(),
1600 ctx: crate::runtime::instance::bootstrap_ctx(),
1601 bucket_fence_registry: std::sync::Arc::default(),
1602 }
1603 }
1604
1605 async fn remove_pool_meta_shard(store: &ECStore, pool_idx: usize) -> DiskStore {
1606 let target_set = store.pools[pool_idx].get_disks_by_key(POOL_META_NAME);
1607 let missing_disk = target_set.disks.read().await[0]
1608 .clone()
1609 .expect("pool metadata fixture disk should be online");
1610 missing_disk
1611 .delete(
1612 RUSTFS_META_BUCKET,
1613 POOL_META_NAME,
1614 DeleteOptions {
1615 recursive: true,
1616 immediate: true,
1617 ..Default::default()
1618 },
1619 )
1620 .await
1621 .expect("one pool metadata shard should be removable");
1622 assert!(
1623 missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_err(),
1624 "pool metadata fixture must start with one missing shard"
1625 );
1626 missing_disk
1627 }
1628
1629 #[tokio::test]
1630 async fn heal_erasure_set_scopes_follow_requested_pool_and_set() {
1631 let store = minimal_heal_store().await;
1632
1633 assert_eq!(
1634 store
1635 .heal_erasure_set_scopes(&HealOpts::default())
1636 .await
1637 .expect("unscoped heal should enumerate every live set"),
1638 vec![(0, 0), (1, 0)]
1639 );
1640 assert_eq!(
1641 store
1642 .heal_erasure_set_scopes(&HealOpts {
1643 pool: Some(1),
1644 set: Some(0),
1645 ..Default::default()
1646 })
1647 .await
1648 .expect("scoped heal should enumerate only its requested set"),
1649 vec![(1, 0)]
1650 );
1651
1652 let err = store
1653 .heal_erasure_set_scopes(&HealOpts {
1654 pool: Some(0),
1655 set: Some(1),
1656 ..Default::default()
1657 })
1658 .await
1659 .expect_err("an invalid set scope must fail closed");
1660 assert!(matches!(err, Error::InvalidArgument(..)));
1661 }
1662
1663 fn pool_meta_with_decommission(info: PoolDecommissionInfo) -> PoolMeta {
1664 PoolMeta {
1665 pools: vec![PoolStatus {
1666 id: 0,
1667 cmd_line: "pool-0".to_string(),
1668 last_update: OffsetDateTime::UNIX_EPOCH,
1669 decommission: Some(info),
1670 }],
1671 ..Default::default()
1672 }
1673 }
1674
1675 #[test]
1676 fn heal_format_pool_state_barriers_are_classified() {
1677 let active = pool_meta_with_decommission(PoolDecommissionInfo {
1678 start_time: Some(OffsetDateTime::UNIX_EPOCH),
1679 ..Default::default()
1680 });
1681 assert!(matches!(
1682 classify_heal_format_pool(0, "pool-0", &active, None),
1683 Some(HealFormatPoolSkip::Retryable)
1684 ));
1685
1686 for info in [
1687 PoolDecommissionInfo {
1688 failed: true,
1689 ..Default::default()
1690 },
1691 PoolDecommissionInfo {
1692 canceled: true,
1693 ..Default::default()
1694 },
1695 ] {
1696 assert!(matches!(
1697 classify_heal_format_pool(0, "pool-0", &pool_meta_with_decommission(info), None),
1698 Some(HealFormatPoolSkip::Retryable)
1699 ));
1700 }
1701
1702 let completed = pool_meta_with_decommission(PoolDecommissionInfo {
1703 complete: true,
1704 ..Default::default()
1705 });
1706 assert!(matches!(
1707 classify_heal_format_pool(0, "pool-0", &completed, None),
1708 Some(HealFormatPoolSkip::Completed)
1709 ));
1710 }
1711
1712 #[test]
1713 fn heal_format_pool_rebalance_barriers_and_identity_are_fail_closed() {
1714 let identity_meta = pool_meta_with_decommission(PoolDecommissionInfo::default());
1715 let rebalance = RebalanceMeta {
1716 pool_stats: vec![RebalanceStats {
1717 participating: true,
1718 info: RebalanceInfo {
1719 status: RebalStatus::Started,
1720 ..Default::default()
1721 },
1722 ..Default::default()
1723 }],
1724 ..Default::default()
1725 };
1726 assert!(matches!(
1727 classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&rebalance)),
1728 Some(HealFormatPoolSkip::Retryable)
1729 ));
1730
1731 let stopping = RebalanceMeta {
1732 pool_stats: vec![RebalanceStats {
1733 info: RebalanceInfo {
1734 stopping: true,
1735 ..Default::default()
1736 },
1737 ..Default::default()
1738 }],
1739 ..Default::default()
1740 };
1741 assert!(matches!(
1742 classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping)),
1743 Some(HealFormatPoolSkip::Retryable)
1744 ));
1745
1746 let identity = pool_meta_with_decommission(PoolDecommissionInfo::default());
1747 assert!(matches!(
1748 classify_heal_format_pool(0, "pool-new", &identity, None),
1749 Some(HealFormatPoolSkip::Retryable)
1750 ));
1751
1752 let identity_without_decommission = PoolMeta {
1753 pools: vec![PoolStatus {
1754 id: 0,
1755 cmd_line: "pool-0".to_string(),
1756 last_update: OffsetDateTime::UNIX_EPOCH,
1757 decommission: None,
1758 }],
1759 ..Default::default()
1760 };
1761 assert!(matches!(
1762 classify_heal_format_pool(0, "pool-new", &identity_without_decommission, None),
1763 Some(HealFormatPoolSkip::Retryable)
1764 ));
1765
1766 assert!(matches!(
1767 classify_heal_format_pool(0, "", &identity_meta, None),
1768 Some(HealFormatPoolSkip::Retryable)
1769 ));
1770
1771 assert!(matches!(
1772 classify_heal_format_pool(0, "pool-0", &PoolMeta::default(), None),
1773 Some(HealFormatPoolSkip::Retryable)
1774 ));
1775
1776 let stopped = RebalanceMeta {
1777 stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
1778 pool_stats: vec![RebalanceStats {
1779 participating: true,
1780 info: RebalanceInfo {
1781 status: RebalStatus::Stopped,
1782 ..Default::default()
1783 },
1784 ..Default::default()
1785 }],
1786 ..Default::default()
1787 };
1788 assert!(classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopped)).is_none());
1789
1790 let stopping_after_stop = RebalanceMeta {
1791 stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
1792 pool_stats: vec![RebalanceStats {
1793 participating: true,
1794 info: RebalanceInfo {
1795 status: RebalStatus::Started,
1796 stopping: true,
1797 ..Default::default()
1798 },
1799 ..Default::default()
1800 }],
1801 ..Default::default()
1802 };
1803 assert!(matches!(
1804 classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping_after_stop)),
1805 Some(HealFormatPoolSkip::Retryable)
1806 ));
1807 }
1808
1809 #[test]
1810 fn skipped_heal_format_pool_is_never_reported_as_success() {
1811 assert!(matches!(
1812 heal_format_pool_skip_error(HealFormatPoolSkip::Retryable),
1813 StorageError::SlowDown
1814 ));
1815 assert!(matches!(
1816 heal_format_pool_skip_error(HealFormatPoolSkip::Completed),
1817 StorageError::NoHealRequired
1818 ));
1819 }
1820
1821 async fn multi_pool_heal_store() -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
1822 let temp_dir = tempfile::tempdir().expect("multi-pool heal test directory should be created");
1823 let mut pool_endpoints = Vec::new();
1824 for pool_index in 0..2 {
1825 let mut endpoints = Vec::new();
1826 for disk_index in 0..4 {
1827 let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
1828 tokio::fs::create_dir_all(&disk_path)
1829 .await
1830 .expect("multi-pool heal test disk should be created");
1831 let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8"))
1832 .expect("test endpoint should parse");
1833 endpoint.set_pool_index(pool_index);
1834 endpoint.set_set_index(0);
1835 endpoint.set_disk_index(disk_index);
1836 endpoints.push(endpoint);
1837 }
1838 pool_endpoints.push(PoolEndpoints {
1839 legacy: false,
1840 set_count: 1,
1841 drives_per_set: 4,
1842 endpoints: Endpoints::from(endpoints),
1843 cmd_line: format!("heal-owner-pool-{pool_index}"),
1844 platform: "test".to_string(),
1845 });
1846 }
1847
1848 let endpoint_pools = EndpointServerPools::from(pool_endpoints);
1849 let instance_ctx = Arc::new(InstanceContext::new());
1850 init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
1851 .await
1852 .expect("multi-pool local disks should initialize");
1853 let shutdown = CancellationToken::new();
1854 let store = ECStore::new_with_instance_ctx(
1855 "127.0.0.1:0".parse().expect("test address should parse"),
1856 endpoint_pools,
1857 shutdown.clone(),
1858 instance_ctx,
1859 )
1860 .await
1861 .expect("multi-pool test store should initialize");
1862 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
1863 (temp_dir, store, shutdown)
1864 }
1865
1866 #[test]
1867 fn bucket_metadata_heal_owner_rejects_path_aliases_and_unrelated_records() {
1868 for file in [".metadata.bin", ".bucket-incarnation"] {
1869 assert_eq!(bucket_metadata_owner(&format!("buckets/example/{file}")), Some("example"));
1870 }
1871 for path in [
1872 "/buckets/example/.metadata.bin",
1873 "buckets/../.metadata.bin",
1874 "buckets/example/../.metadata.bin",
1875 "buckets/example/.metadata.bin/extra",
1876 "buckets/example//.metadata.bin",
1877 "buckets/example/.metadata.bin.old",
1878 "config/iam/example/.metadata.bin",
1879 "buckets/example/usage.json",
1880 ] {
1881 assert_eq!(bucket_metadata_owner(path), None, "unexpected bucket authority for {path}");
1882 }
1883 }
1884
1885 #[tokio::test]
1886 async fn replacement_bucket_metadata_respects_pool_ownership_and_required_records() {
1887 let (root, store, shutdown) = multi_pool_heal_store().await;
1888 let bucket = "replacement-bucket-metadata";
1889 for invalid_bucket in ["MixedCase", "../example", RUSTFS_META_BUCKET] {
1890 assert!(matches!(
1891 store
1892 .heal_replacement_bucket_metadata(
1893 invalid_bucket,
1894 &HealOpts {
1895 pool: Some(0),
1896 set: Some(0),
1897 ..Default::default()
1898 },
1899 &[root.path().join("pool0-disk0").to_string_lossy().into_owned()],
1900 )
1901 .await,
1902 Err(Error::PreconditionFailed)
1903 ));
1904 }
1905 store
1906 .make_bucket(bucket, &MakeBucketOptions::default())
1907 .await
1908 .expect("create bucket and its persisted records");
1909 let mut owned = 0;
1910 for pool in 0..2 {
1911 let target = root.path().join(format!("pool{pool}-disk0"));
1912 let records: Vec<_> = [".metadata.bin", ".bucket-incarnation"]
1913 .into_iter()
1914 .map(|file| {
1915 let path = target.join(RUSTFS_META_BUCKET).join("buckets").join(bucket).join(file);
1916 let exists = path.join("xl.meta").exists();
1917 if exists {
1918 std::fs::remove_dir_all(&path).expect("remove replacement metadata shard");
1919 owned += 1;
1920 }
1921 (path, exists)
1922 })
1923 .collect();
1924 store
1925 .heal_replacement_bucket_metadata(
1926 bucket,
1927 &HealOpts {
1928 pool: Some(pool),
1929 set: Some(0),
1930 recreate: true,
1931 ..Default::default()
1932 },
1933 &[target.to_string_lossy().into_owned()],
1934 )
1935 .await
1936 .expect("repair owned records and accept authoritative ownership in the other pool");
1937 for (path, existed) in records {
1938 assert_eq!(path.join("xl.meta").exists(), existed, "wrong placement for {}", path.display());
1939 }
1940 }
1941 assert_eq!(owned, 2, "each required record must have exactly one owning pool");
1942 let object = format!("buckets/{bucket}/.metadata.bin");
1943 let owner = store
1944 .get_pool_idx_existing_with_opts(RUSTFS_META_BUCKET, &object, &ObjectOptions::default())
1945 .await
1946 .expect("persisted metadata owner");
1947 let disk = store.pools[owner].disk_set[0].disks.read().await[0]
1948 .clone()
1949 .expect("metadata target disk");
1950 let fi = disk
1951 .read_version(
1952 "",
1953 RUSTFS_META_BUCKET,
1954 &object,
1955 "",
1956 &disk::ReadOptions {
1957 read_data: true,
1958 ..Default::default()
1959 },
1960 )
1961 .await
1962 .expect("inline metadata to repair through target-side admission");
1963 assert!(fi.data.is_some(), "fixture must contain an inline metadata shard");
1964 let target_path = root.path().join(format!("pool{owner}-disk0"));
1965 std::fs::remove_dir_all(target_path.join(RUSTFS_META_BUCKET).join(&object)).expect("remove remote target shard");
1966 let incarnation = store
1967 .bucket_incarnation_id_from_disk(bucket)
1968 .await
1969 .expect("current incarnation");
1970 store
1971 .rename_local_data_at_incarnation(
1972 &disk.endpoint().to_string(),
1973 (disk::RUSTFS_META_TMP_BUCKET, "metadata-heal-control"),
1974 &fi,
1975 (RUSTFS_META_BUCKET, &object),
1976 incarnation,
1977 )
1978 .await
1979 .expect("the target-side RPC admission must accept current bucket metadata repair");
1980 assert!(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta").exists());
1981 let (configuration, _) = read_config_no_lock_preserve_empty_with_metadata(store.clone(), &object)
1982 .await
1983 .expect("read the current bucket configuration");
1984 save_config(store.clone(), &object, configuration)
1985 .await
1986 .expect("publish a newer config revision in the same bucket incarnation");
1987 let committed =
1988 std::fs::read(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta")).expect("new target configuration");
1989 assert!(
1990 store
1991 .rename_local_data_at_incarnation(
1992 &disk.endpoint().to_string(),
1993 (disk::RUSTFS_META_TMP_BUCKET, "delayed-metadata-heal"),
1994 &fi,
1995 (RUSTFS_META_BUCKET, &object),
1996 incarnation,
1997 )
1998 .await
1999 .is_err(),
2000 "a delayed repair must not overwrite a newer config in the same incarnation"
2001 );
2002 assert_eq!(
2003 std::fs::read(target_path.join(RUSTFS_META_BUCKET).join(&object).join("xl.meta")).expect("retained configuration"),
2004 committed,
2005 "rejected repair must leave the newer configuration unchanged"
2006 );
2007 for pool in 0..2 {
2008 for disk in 0..4 {
2009 let path = root
2010 .path()
2011 .join(format!("pool{pool}-disk{disk}"))
2012 .join(RUSTFS_META_BUCKET)
2013 .join("buckets")
2014 .join(bucket)
2015 .join(".metadata.bin");
2016 if path.exists() {
2017 std::fs::remove_dir_all(path).expect("remove every copy of required bucket configuration");
2018 }
2019 }
2020 }
2021 metadata_sys::remove_bucket_metadata_in(&store.ctx, bucket)
2022 .await
2023 .expect("remove the cached configuration so repair cannot rely on it");
2024 assert!(
2025 store
2026 .heal_replacement_bucket_metadata(
2027 bucket,
2028 &HealOpts {
2029 pool: Some(0),
2030 set: Some(0),
2031 recreate: true,
2032 ..Default::default()
2033 },
2034 &[root.path().join("pool0-disk0").to_string_lossy().into_owned()],
2035 )
2036 .await
2037 .is_err(),
2038 "total metadata loss must not be accepted as ownership in another pool"
2039 );
2040 for pool in 0..2 {
2041 assert!(
2042 !root
2043 .path()
2044 .join(format!("pool{pool}-disk0"))
2045 .join(RUSTFS_META_BUCKET)
2046 .join(&object)
2047 .exists(),
2048 "missing required configuration must not be recreated with defaults"
2049 );
2050 }
2051 shutdown.cancel();
2052 }
2053
2054 #[tokio::test]
2055 #[serial_test::serial]
2056 async fn absence_proof_requires_every_selected_pool() {
2057 let (_temp_dir, store, shutdown) = multi_pool_heal_store().await;
2058 let bucket = format!("absence-scope-{}", Uuid::new_v4().simple());
2059 store
2060 .make_bucket(&bucket, &MakeBucketOptions::default())
2061 .await
2062 .expect("create bucket in both pools");
2063 let version = Uuid::new_v4().to_string();
2064 let result = store
2065 .heal_object_with_proof(&bucket, "history.txt", &version, &HealOpts::default())
2066 .await
2067 .expect("exact absence lookup should complete across both pools");
2068 assert!(result.error.is_none());
2069 let proof = result.absence.expect("all selected pools proved the exact version absent");
2070 assert_eq!(proof.locations, vec![(0, 0), (1, 0)]);
2071 assert_eq!((proof.pool_index, proof.set_index), (None, None));
2072 assert_eq!(proof.bucket, bucket);
2073 assert_eq!(proof.version_id, version);
2074 assert!(!proof.removed, "already absent versions do not count as another cleanup");
2075
2076 store.pool_meta.write().await.pools[1].decommission = Some(PoolDecommissionInfo {
2077 start_time: Some(OffsetDateTime::now_utc()),
2078 ..Default::default()
2079 });
2080 let partial = store
2081 .heal_object_with_proof(&bucket, "history.txt", &version, &HealOpts::default())
2082 .await
2083 .expect("unscoped heal should retain its legacy suspended-pool behavior");
2084 assert!(partial.absence.is_none(), "an uninspected suspended pool prevents global absence proof");
2085 shutdown.cancel();
2086 }
2087
2088 fn heal_test_format_path(temp_dir: &tempfile::TempDir, pool_index: usize, disk_index: usize) -> std::path::PathBuf {
2089 temp_dir
2090 .path()
2091 .join(format!("pool{pool_index}-disk{disk_index}"))
2092 .join(crate::disk::RUSTFS_META_BUCKET)
2093 .join(FORMAT_CONFIG_FILE)
2094 }
2095
2096 async fn remove_heal_test_format(
2097 temp_dir: &tempfile::TempDir,
2098 store: &ECStore,
2099 pool_index: usize,
2100 disk_index: usize,
2101 ) -> String {
2102 let endpoint = store.pools[pool_index].endpoints.endpoints.as_ref()[disk_index].clone();
2103 let target = endpoint.to_string();
2104 let format_path = heal_test_format_path(temp_dir, pool_index, disk_index);
2105 tokio::fs::remove_file(&format_path)
2106 .await
2107 .expect("replacement target format should be removable");
2108 assert!(
2109 !tokio::fs::try_exists(&format_path)
2110 .await
2111 .expect("replacement target format path should be inspectable")
2112 );
2113 let replacement = new_disk(
2114 &endpoint,
2115 &DiskOption {
2116 cleanup: false,
2117 health_check: false,
2118 },
2119 )
2120 .await
2121 .expect("unformatted replacement disk should open");
2122 let set = &store.pools[pool_index].disk_set[0];
2123 let target_slot = set
2124 .set_endpoints
2125 .iter()
2126 .position(|candidate| candidate == &endpoint)
2127 .expect("replacement endpoint should belong to its set");
2128 set.disks.write().await[target_slot] = Some(replacement);
2129 target
2130 }
2131
2132 async fn assert_heal_test_format_missing(temp_dir: &tempfile::TempDir, pool_index: usize, disk_index: usize) {
2133 assert!(
2134 !tokio::fs::try_exists(heal_test_format_path(temp_dir, pool_index, disk_index))
2135 .await
2136 .expect("replacement target format path should be inspectable"),
2137 "format heal must not write the replacement target"
2138 );
2139 }
2140
2141 #[tokio::test]
2142 #[serial_test::serial]
2143 async fn replacement_format_heal_coexists_with_capacity_read_fence() {
2144 let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
2145 let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
2146 let capacity_guard = store
2147 .acquire_external_decommission_capacity_fence(&[0], DecommissionCapacityAdmission::Heal)
2148 .await
2149 .expect("ordinary heal capacity fence should be acquired");
2150
2151 let (_, err) =
2152 tokio::time::timeout(std::time::Duration::from_secs(2), store.heal_replacement_format(false, 0, 0, &[target]))
2153 .await
2154 .expect("replacement format heal must not wait on an existing capacity read fence")
2155 .expect("replacement format heal should complete");
2156
2157 assert!(err.is_none(), "replacement format heal should succeed: {err:?}");
2158 assert!(
2159 tokio::fs::try_exists(heal_test_format_path(&temp_dir, 0, 3))
2160 .await
2161 .expect("replacement target format path should be inspectable"),
2162 "replacement format heal should restore the target while the capacity read fence is held"
2163 );
2164 drop(capacity_guard);
2165 shutdown.cancel();
2166 }
2167
2168 #[tokio::test]
2169 #[serial_test::serial]
2170 async fn replacement_format_heal_remains_blocked_by_pool_metadata_writer() {
2171 let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
2172 let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
2173 let pool_lock = store.pools[0]
2174 .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
2175 .await
2176 .expect("pool metadata lock should be created");
2177 let pool_writer = pool_lock
2178 .get_write_lock(get_lock_acquire_timeout())
2179 .await
2180 .expect("pool metadata writer should be acquired");
2181
2182 assert!(
2183 tokio::time::timeout(
2184 std::time::Duration::from_secs(1),
2185 store.heal_replacement_format(false, 0, 0, std::slice::from_ref(&target)),
2186 )
2187 .await
2188 .is_err(),
2189 "replacement format heal must wait for a pool metadata writer"
2190 );
2191 assert_heal_test_format_missing(&temp_dir, 0, 3).await;
2192
2193 drop(pool_writer);
2194 let (_, err) =
2195 tokio::time::timeout(std::time::Duration::from_secs(30), store.heal_replacement_format(false, 0, 0, &[target]))
2196 .await
2197 .expect("replacement format heal should resume after the writer releases")
2198 .expect("replacement format heal should complete after the writer releases");
2199 assert!(err.is_none(), "replacement format heal should succeed: {err:?}");
2200 assert!(
2201 tokio::fs::try_exists(heal_test_format_path(&temp_dir, 0, 3))
2202 .await
2203 .expect("replacement target format path should be inspectable"),
2204 "replacement format heal should restore the target after the writer releases"
2205 );
2206 shutdown.cancel();
2207 }
2208
2209 #[tokio::test]
2210 #[serial_test::serial]
2211 async fn full_format_heal_preblocked_pool_metadata_never_writes_format() {
2212 let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
2213 remove_heal_test_format(&temp_dir, &store, 0, 3).await;
2214 store.pool_meta_save_gate.lock().await.observe_replicas(PoolMetaReplicaState {
2215 needs_repair: true,
2216 repair_write_safe: false,
2217 });
2218
2219 let err = store
2220 .handle_heal_format(false)
2221 .await
2222 .expect_err("a preblocked pool metadata state must reject full format heal");
2223 assert!(
2224 err.to_string()
2225 .contains("restart after all replicas are readable and consistent")
2226 );
2227 assert_heal_test_format_missing(&temp_dir, 0, 3).await;
2228 store
2229 .ensure_pool_meta_side_effects_safe("preblocked format heal side effect")
2230 .await
2231 .expect_err("the preblocked state must remain sticky");
2232 shutdown.cancel();
2233 }
2234
2235 #[tokio::test]
2236 #[serial_test::serial]
2237 async fn full_format_heal_future_identity_never_writes_format_and_latches() {
2238 let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
2239 remove_heal_test_format(&temp_dir, &store, 0, 3).await;
2240 let (mut future_identity, _) =
2241 read_config_no_lock_preserve_empty_with_metadata(store.pools[1].clone(), POOL_META_IDENTITY_NAME)
2242 .await
2243 .expect("current identity should be readable");
2244 let future_version = u16::from_le_bytes([future_identity[2], future_identity[3]])
2245 .checked_add(1)
2246 .expect("identity version should have a future value");
2247 future_identity[2..4].copy_from_slice(&future_version.to_le_bytes());
2248 save_config(store.pools[1].clone(), POOL_META_IDENTITY_NAME, future_identity)
2249 .await
2250 .expect("future identity should be persisted");
2251
2252 let err = store
2253 .handle_heal_format(false)
2254 .await
2255 .expect_err("a future identity must reject full format heal");
2256 assert!(err.to_string().contains("pool metadata incompatible"));
2257 assert_heal_test_format_missing(&temp_dir, 0, 3).await;
2258 store
2259 .ensure_pool_meta_side_effects_safe("future identity format heal side effect")
2260 .await
2261 .expect_err("the future identity rejection must latch the write gate");
2262 shutdown.cancel();
2263 }
2264
2265 #[tokio::test]
2266 #[serial_test::serial]
2267 async fn replacement_format_heal_epoch_conflict_never_writes_format_and_latches() {
2268 let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
2269 let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
2270 let conflicting_identity =
2271 initialized_pool_meta_identity_for_test(store.id, 2).expect("conflicting identity should encode");
2272 save_config(store.pools[1].clone(), POOL_META_IDENTITY_NAME, conflicting_identity)
2273 .await
2274 .expect("conflicting identity should be persisted");
2275
2276 let err = store
2277 .heal_replacement_format(false, 0, 0, &[target])
2278 .await
2279 .expect_err("an identity epoch conflict must reject replacement format heal");
2280 assert!(
2281 err.to_string()
2282 .contains("identity replicas disagree on cluster identity or epoch")
2283 );
2284 assert_heal_test_format_missing(&temp_dir, 0, 3).await;
2285 store
2286 .ensure_pool_meta_side_effects_safe("epoch conflict replacement format side effect")
2287 .await
2288 .expect_err("the identity epoch conflict must latch the write gate");
2289 shutdown.cancel();
2290 }
2291
2292 #[tokio::test]
2293 #[serial_test::serial]
2294 async fn replacement_format_heal_initialized_identity_without_pool_meta_never_writes_and_latches() {
2295 let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
2296 let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
2297 for pool in &store.pools {
2298 delete_config(pool.clone(), POOL_META_NAME)
2299 .await
2300 .expect("pool metadata replica should be removable");
2301 }
2302
2303 let err = store
2304 .heal_replacement_format(false, 0, 0, &[target])
2305 .await
2306 .expect_err("initialized identity without pool metadata must reject replacement format heal");
2307 assert!(err.to_string().contains("initialized cluster identity exists"));
2308 assert_heal_test_format_missing(&temp_dir, 0, 3).await;
2309 store
2310 .ensure_pool_meta_side_effects_safe("missing pool metadata replacement format side effect")
2311 .await
2312 .expect_err("missing initialized pool metadata must latch the write gate");
2313 shutdown.cancel();
2314 }
2315
2316 #[tokio::test]
2317 #[serial_test::serial]
2318 async fn full_format_heal_lost_after_last_save_does_not_publish_or_renew_and_latches() {
2319 let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
2320 let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
2321 store.pools[0].disk_set[0].disks.write().await[3] = None;
2322 let barrier = HealFormatAfterSaveBarrier::install(&store.pools[0], 3);
2323 let recovery_latch = store.pool_meta_save_gate.lock().await.aborted_transaction_latch_for_test();
2324 let mut heal = tokio::spawn({
2325 let store = Arc::clone(&store);
2326 async move { store.handle_heal_format(false).await }
2327 });
2328
2329 barrier.wait_until_paused().await;
2330 let saved = tokio::fs::read(heal_test_format_path(&temp_dir, 0, 3))
2331 .await
2332 .expect("the last replacement format must be durable before fence loss");
2333 let saved = FormatV3::try_from(saved.as_slice()).expect("the durable replacement format should decode");
2334 assert_eq!(saved.erasure.this, store.pools[0].format.erasure.sets[0][3]);
2335 recovery_latch.store(true, std::sync::atomic::Ordering::SeqCst);
2336 barrier.release();
2337
2338 let (result, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
2339 .await
2340 .expect("format heal should stop after the lost fence")
2341 .expect("format heal task should not panic")
2342 .expect("format heal should return its fenced result");
2343 assert!(matches!(err, Some(StorageError::SlowDown)));
2344 assert!(
2345 result
2346 .after
2347 .drives
2348 .iter()
2349 .any(|drive| drive.endpoint == target && drive.state == DriveState::Missing.to_string()),
2350 "the lost fence must prevent the durable format from being published in the heal result"
2351 );
2352 assert!(
2353 store.pools[0].disk_set[0].disks.read().await[3].is_none(),
2354 "the lost fence must prevent renew_disk from attaching the replacement"
2355 );
2356 recovery_latch.store(false, std::sync::atomic::Ordering::SeqCst);
2357 store
2358 .ensure_pool_meta_side_effects_safe("post-save format fence loss side effect")
2359 .await
2360 .expect_err("post-save format fence loss must remain sticky");
2361 shutdown.cancel();
2362 }
2363
2364 #[tokio::test]
2365 async fn heal_object_pool_scope_selects_only_requested_pool() {
2366 let store = minimal_heal_store().await;
2367 let pools = store
2368 .get_pools_for_heal_object(&HealOpts {
2369 pool: Some(1),
2370 ..Default::default()
2371 })
2372 .expect("requested pool should be selected");
2373
2374 assert_eq!(pools.len(), 1);
2375 assert!(Arc::ptr_eq(&pools[0], &store.pools[1]));
2376 }
2377
2378 #[tokio::test]
2379 async fn heal_object_pool_scope_rejects_invalid_pool() {
2380 let store = minimal_heal_store().await;
2381 let err = store
2382 .get_pools_for_heal_object(&HealOpts {
2383 pool: Some(2),
2384 ..Default::default()
2385 })
2386 .expect_err("out-of-range pool scope must fail closed");
2387
2388 assert!(
2389 matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
2390 if field == "pool" && reason.contains("invalid heal pool index 2 for 2 pools")),
2391 "unexpected invalid pool error: {err:?}"
2392 );
2393 }
2394
2395 #[tokio::test]
2396 #[serial_test::serial]
2397 async fn pool_meta_read_repair_reuses_its_write_fence() {
2398 let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
2399 let missing_disk = remove_pool_meta_shard(&store, 0).await;
2400
2401 let (result, err) = tokio::time::timeout(
2402 std::time::Duration::from_secs(30),
2403 store.handle_heal_object(
2404 RUSTFS_META_BUCKET,
2405 POOL_META_NAME,
2406 "",
2407 &HealOpts {
2408 read_repair: true,
2409 pool: Some(0),
2410 ..Default::default()
2411 },
2412 ),
2413 )
2414 .await
2415 .expect("pool metadata read repair must not wait on its own capacity fence")
2416 .expect("pool metadata read repair should complete");
2417
2418 assert!(err.is_none(), "pool metadata read repair should succeed: {err:?}");
2419 assert_eq!(result.object, POOL_META_NAME);
2420 assert!(
2421 missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
2422 "pool metadata read repair should restore the missing shard"
2423 );
2424 }
2425
2426 #[tokio::test]
2427 #[serial_test::serial]
2428 async fn pool_meta_heal_preserves_typed_lock_timeout() {
2429 let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
2430 let lock = store.pools[0]
2431 .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
2432 .await
2433 .expect("pool metadata lock should be created");
2434 let guard = lock
2435 .get_write_lock(get_lock_acquire_timeout())
2436 .await
2437 .expect("pool metadata lock should be acquired");
2438
2439 let (_, err) = temp_env::async_with_vars(
2440 [(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))],
2441 store.handle_heal_object(
2442 RUSTFS_META_BUCKET,
2443 POOL_META_NAME,
2444 "",
2445 &HealOpts {
2446 pool: Some(0),
2447 ..Default::default()
2448 },
2449 ),
2450 )
2451 .await
2452 .expect("pool metadata lock timeout should be mapped into the heal result");
2453
2454 assert!(
2455 matches!(err, Some(Error::Lock(rustfs_lock::LockError::Timeout { .. }))),
2456 "pool metadata heal must preserve the recoverable lock timeout: {err:?}"
2457 );
2458 drop(guard);
2459 }
2460
2461 #[tokio::test]
2462 #[serial_test::serial]
2463 async fn pool_meta_neighbor_keeps_ordinary_object_locking() {
2464 let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
2465 let object = "pool.bin.backup";
2466 save_config(store.pools[0].clone(), object, b"neighbor metadata".to_vec())
2467 .await
2468 .expect("neighbor metadata fixture should be written");
2469
2470 let target_set = store.pools[0].get_disks_by_key(object);
2471 let lock = target_set
2472 .new_ns_lock(RUSTFS_META_BUCKET, object)
2473 .await
2474 .expect("neighbor metadata lock should be created");
2475 let guard = lock
2476 .get_write_lock(get_lock_acquire_timeout())
2477 .await
2478 .expect("neighbor metadata lock should be acquired");
2479 let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
2480 let heal_store = Arc::clone(&store);
2481 let mut heal = tokio::spawn(async move {
2482 heal_store
2483 .handle_heal_object(
2484 RUSTFS_META_BUCKET,
2485 object,
2486 "",
2487 &HealOpts {
2488 pool: Some(0),
2489 ..Default::default()
2490 },
2491 )
2492 .await
2493 });
2494
2495 barrier.wait_until_external_heal_target_lock_attempted().await;
2496 tokio::task::yield_now().await;
2497 assert!(!heal.is_finished(), "neighbor metadata heal must retain ordinary object locking");
2498 drop(guard);
2499
2500 let (result, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
2501 .await
2502 .expect("neighbor metadata heal should finish after the object lock is released")
2503 .expect("neighbor metadata heal task should not panic")
2504 .expect("neighbor metadata heal should complete");
2505 assert!(err.is_none(), "neighbor metadata heal should succeed: {err:?}");
2506 assert_eq!(result.object, object);
2507 drop(barrier);
2508 }
2509
2510 #[tokio::test]
2511 #[serial_test::serial]
2512 async fn targeted_heal_is_blocked_by_exact_fit_decommission_reservation() {
2513 let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
2514 let bucket = format!("heal-capacity-{}", Uuid::new_v4().simple());
2515 let object = "targeted-heal.bin";
2516 store
2517 .make_bucket(&bucket, &MakeBucketOptions::default())
2518 .await
2519 .expect("heal capacity bucket should be created");
2520
2521 let target_set = store.pools[1].get_disks(0);
2522 let mut reader = PutObjReader::from_vec(b"targeted heal body".to_vec());
2523 target_set
2524 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
2525 .await
2526 .expect("targeted heal fixture should be written");
2527 let missing_disk = target_set.disks.read().await[0]
2528 .clone()
2529 .expect("targeted heal fixture disk should be online");
2530 missing_disk
2531 .delete(
2532 &bucket,
2533 object,
2534 DeleteOptions {
2535 recursive: true,
2536 immediate: true,
2537 ..Default::default()
2538 },
2539 )
2540 .await
2541 .expect("targeted heal fixture shard should be removed");
2542 assert!(
2543 missing_disk.read_xl(&bucket, object, false).await.is_err(),
2544 "targeted heal fixture must start with one missing metadata copy"
2545 );
2546
2547 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2548 set_decommission_capacity_info_overrides_for_test(
2549 store.id,
2550 vec![vec![
2551 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
2552 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
2553 ]],
2554 );
2555 store
2556 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
2557 .await
2558 .expect("the exact-fit decommission reservation should activate");
2559 {
2560 let pool_meta = store.pool_meta.read().await;
2561 let reservation = pool_meta.pools[0]
2562 .decommission
2563 .as_ref()
2564 .and_then(|info| info.capacity_reservation.as_ref())
2565 .expect("the active decommission reservation should be durable");
2566 assert_eq!(reservation.targets.len(), 1);
2567 assert_eq!(reservation.targets[0].pool_index, 1);
2568 assert_eq!(reservation.targets[0].reserved_physical_bytes, 60);
2569 }
2570
2571 let (_, err) = store
2572 .handle_heal_object(
2573 &bucket,
2574 object,
2575 "",
2576 &HealOpts {
2577 pool: Some(1),
2578 set: Some(0),
2579 ..Default::default()
2580 },
2581 )
2582 .await
2583 .expect("targeted heal should return a mapped capacity result");
2584 assert!(matches!(err, Some(Error::SlowDown)), "targeted heal must be capacity-blocked: {err:?}");
2585 assert!(
2586 missing_disk.read_xl(&bucket, object, false).await.is_err(),
2587 "capacity-blocked targeted heal must not rewrite the missing shard"
2588 );
2589
2590 let (_, err) = tokio::time::timeout(
2591 std::time::Duration::from_secs(30),
2592 store.handle_heal_object(
2593 RUSTFS_META_BUCKET,
2594 POOL_META_NAME,
2595 "",
2596 &HealOpts {
2597 pool: Some(1),
2598 ..Default::default()
2599 },
2600 ),
2601 )
2602 .await
2603 .expect("pool metadata admission should not recurse on its namespace lock")
2604 .expect("pool metadata capacity rejection should be mapped");
2605 assert!(
2606 matches!(err, Some(Error::SlowDown)),
2607 "pool metadata heal must preserve target reservation admission: {err:?}"
2608 );
2609 }
2610
2611 #[tokio::test]
2612 #[serial_test::serial]
2613 async fn pool_meta_heal_keeps_per_target_capacity_admission() {
2614 let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
2615 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2616 set_decommission_capacity_info_overrides_for_test(
2617 store.id,
2618 vec![vec![
2619 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
2620 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
2621 DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
2622 ]],
2623 );
2624 store
2625 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
2626 .await
2627 .expect("pool metadata reservation should activate");
2628 {
2629 let pool_meta = store.pool_meta.read().await;
2630 let reservation = pool_meta.pools[0]
2631 .decommission
2632 .as_ref()
2633 .and_then(|info| info.capacity_reservation.as_ref())
2634 .expect("pool metadata reservation should be durable");
2635 assert_eq!(reservation.targets.len(), 1);
2636 assert_eq!(reservation.targets[0].pool_index, 1);
2637 }
2638
2639 let missing_disk = remove_pool_meta_shard(&store, 2).await;
2640
2641 let (result, err) = tokio::time::timeout(
2642 std::time::Duration::from_secs(30),
2643 store.handle_heal_object(RUSTFS_META_BUCKET, POOL_META_NAME, "", &HealOpts::default()),
2644 )
2645 .await
2646 .expect("unscoped pool metadata heal should complete")
2647 .expect("unscoped pool metadata heal should return a mapped result");
2648
2649 assert!(err.is_none(), "an admitted target should let unscoped metadata heal succeed: {err:?}");
2650 assert_eq!(result.object, POOL_META_NAME);
2651 assert!(
2652 missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
2653 "unscoped metadata heal should repair the admitted target while another target is reserved"
2654 );
2655 }
2656
2657 #[tokio::test]
2658 #[serial_test::serial]
2659 async fn targeted_heal_keeps_target_lock_for_different_lock_domain() {
2660 let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
2661 let bucket = format!("heal-lock-domain-{}", Uuid::new_v4().simple());
2662 let object = "different-domain-heal.bin";
2663 store
2664 .make_bucket(&bucket, &MakeBucketOptions::default())
2665 .await
2666 .expect("heal lock-domain bucket should be created");
2667
2668 let target_set = other_store.pools[1].get_disks(0);
2669 let mut reader = PutObjReader::from_vec(b"different lock domain body".to_vec());
2670 target_set
2671 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
2672 .await
2673 .expect("heal lock-domain fixture should be written");
2674 let missing_disk = target_set.disks.read().await[0]
2675 .clone()
2676 .expect("heal lock-domain fixture disk should be online");
2677 missing_disk
2678 .delete(
2679 &bucket,
2680 object,
2681 DeleteOptions {
2682 recursive: true,
2683 immediate: true,
2684 ..Default::default()
2685 },
2686 )
2687 .await
2688 .expect("heal lock-domain fixture shard should be removed");
2689 assert!(
2690 missing_disk.read_xl(&bucket, object, false).await.is_err(),
2691 "heal lock-domain fixture must start with one missing metadata copy"
2692 );
2693
2694 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2695 set_decommission_capacity_info_overrides_for_test(
2696 store.id,
2697 vec![vec![
2698 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
2699 DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100),
2700 DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
2701 ]],
2702 );
2703 store
2704 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
2705 .await
2706 .expect("heal lock-domain reservation should activate");
2707 {
2708 let pool_meta = store.pool_meta.read().await;
2709 let reservation = pool_meta.pools[0]
2710 .decommission
2711 .as_ref()
2712 .and_then(|info| info.capacity_reservation.as_ref())
2713 .expect("heal lock-domain reservation should be durable");
2714 assert_eq!(reservation.targets.len(), 1);
2715 assert_eq!(reservation.targets[0].pool_index, 2);
2716 }
2717 let pool_meta = store.pool_meta.read().await.clone();
2718 *other_store.pool_meta.write().await = pool_meta;
2719
2720 let fixed_set = other_store.pools[0].disk_set[0].clone();
2721 assert!(
2722 !fixed_set.shares_namespace_lock_domain(&target_set).await,
2723 "heal fixture must use different fixed and target lock domains"
2724 );
2725 let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, other_store.id);
2726 let heal_store = Arc::clone(&other_store);
2727 let heal_bucket = bucket.clone();
2728 let heal_object = object.to_string();
2729 let mut heal = tokio::spawn(async move {
2730 heal_store
2731 .handle_heal_object(
2732 &heal_bucket,
2733 &heal_object,
2734 "",
2735 &HealOpts {
2736 pool: Some(1),
2737 set: Some(0),
2738 ..Default::default()
2739 },
2740 )
2741 .await
2742 });
2743
2744 barrier.wait_until_external_capacity_released().await;
2745 barrier.wait_until_external_heal_target_lock_attempted().await;
2746
2747 let (_, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
2748 .await
2749 .expect("targeted heal should finish after target lock release")
2750 .expect("targeted heal task should not panic")
2751 .expect("targeted heal should complete");
2752 assert!(err.is_none(), "targeted heal should repair after target lock release: {err:?}");
2753 assert!(
2754 missing_disk.read_xl(&bucket, object, false).await.is_ok(),
2755 "targeted heal should rewrite the missing shard after the target lock is released"
2756 );
2757 drop(barrier);
2758 }
2759
2760 #[tokio::test]
2761 #[serial_test::serial]
2762 async fn targeted_heal_reuses_shared_target_lock_without_reentrant_lock() {
2763 let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
2764 let bucket = format!("heal-shared-lock-domain-{}", Uuid::new_v4().simple());
2765 let object = "shared-domain-heal.bin";
2766 store
2767 .make_bucket(&bucket, &MakeBucketOptions::default())
2768 .await
2769 .expect("shared heal lock-domain bucket should be created");
2770
2771 let target_set = store.pools[0].get_disks(0);
2772 let mut reader = PutObjReader::from_vec(b"shared lock domain body".to_vec());
2773 target_set
2774 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
2775 .await
2776 .expect("shared heal lock-domain fixture should be written");
2777 let missing_disk = target_set.disks.read().await[0]
2778 .clone()
2779 .expect("shared heal lock-domain fixture disk should be online");
2780 missing_disk
2781 .delete(
2782 &bucket,
2783 object,
2784 DeleteOptions {
2785 recursive: true,
2786 immediate: true,
2787 ..Default::default()
2788 },
2789 )
2790 .await
2791 .expect("shared heal lock-domain fixture shard should be removed");
2792 assert!(
2793 missing_disk.read_xl(&bucket, object, false).await.is_err(),
2794 "shared heal lock-domain fixture must start with one missing metadata copy"
2795 );
2796
2797 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2798 set_decommission_capacity_info_overrides_for_test(
2799 store.id,
2800 vec![vec![
2801 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
2802 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
2803 DecommissionPoolCapacityInfo::for_test(2, layout, 0, 30, 30),
2804 ]],
2805 );
2806 store
2807 .save_current_pool_meta_for_decommission_start(&[2], Vec::new())
2808 .await
2809 .expect("shared heal lock-domain reservation should activate");
2810 {
2811 let pool_meta = store.pool_meta.read().await;
2812 let reservation = pool_meta.pools[2]
2813 .decommission
2814 .as_ref()
2815 .and_then(|info| info.capacity_reservation.as_ref())
2816 .expect("shared heal lock-domain reservation should be durable");
2817 assert_eq!(reservation.targets.len(), 1);
2818 assert_eq!(reservation.targets[0].pool_index, 1);
2819 assert_eq!(reservation.targets[0].reserved_physical_bytes, 60);
2820 }
2821
2822 let fixed_set = store.pools[0].disk_set[0].clone();
2823 assert!(
2824 fixed_set.shares_namespace_lock_domain(&target_set).await,
2825 "shared heal fixture must use one fixed and target lock domain"
2826 );
2827
2828 let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
2829 let heal_store = Arc::clone(&store);
2830 let heal_bucket = bucket.clone();
2831 let heal_object = object.to_string();
2832 let mut heal = tokio::spawn(async move {
2833 heal_store
2834 .handle_heal_object(
2835 &heal_bucket,
2836 &heal_object,
2837 "",
2838 &HealOpts {
2839 pool: Some(0),
2840 set: Some(0),
2841 ..Default::default()
2842 },
2843 )
2844 .await
2845 });
2846
2847 barrier.wait_until_external_capacity_released().await;
2848 barrier.wait_until_external_heal_operation_started().await;
2849 let (_, err) = tokio::time::timeout(std::time::Duration::from_secs(5), &mut heal)
2850 .await
2851 .expect("shared-domain heal must not reenter the fixed namespace lock")
2852 .expect("shared-domain heal task should not panic")
2853 .expect("shared-domain heal should complete");
2854 assert!(err.is_none(), "shared-domain heal should repair after admission: {err:?}");
2855 assert!(
2856 missing_disk.read_xl(&bucket, object, false).await.is_ok(),
2857 "shared-domain heal should rewrite the missing shard"
2858 );
2859 drop(barrier);
2860 }
2861
2862 #[tokio::test]
2863 async fn scoped_heal_object_defers_when_requested_pool_is_suspended() {
2864 let mut store = minimal_heal_store().await;
2865 store.pool_meta = RwLock::new(PoolMeta {
2866 pools: vec![
2867 PoolStatus {
2868 id: 0,
2869 cmd_line: "pool-0".to_string(),
2870 last_update: OffsetDateTime::UNIX_EPOCH,
2871 decommission: None,
2872 },
2873 PoolStatus {
2874 id: 1,
2875 cmd_line: "pool-1".to_string(),
2876 last_update: OffsetDateTime::UNIX_EPOCH,
2877 decommission: Some(PoolDecommissionInfo {
2878 start_time: Some(OffsetDateTime::UNIX_EPOCH),
2879 ..Default::default()
2880 }),
2881 },
2882 ],
2883 ..Default::default()
2884 });
2885
2886 let (_, err) = store
2887 .handle_heal_object(
2888 "bucket",
2889 "object",
2890 "",
2891 &HealOpts {
2892 pool: Some(1),
2893 set: Some(0),
2894 ..Default::default()
2895 },
2896 )
2897 .await
2898 .expect("suspended pool should return a deferred heal result");
2899
2900 assert!(matches!(err, Some(StorageError::SlowDown)));
2901
2902 let (_, err) = store
2903 .handle_heal_object(
2904 "bucket",
2905 "object",
2906 "",
2907 &HealOpts {
2908 set: Some(1),
2909 ..Default::default()
2910 },
2911 )
2912 .await
2913 .expect("unscoped heal should return the active pool result");
2914
2915 assert!(matches!(err, Some(StorageError::InvalidArgument(_, ref field, _)) if field == "set"));
2916
2917 let err = store
2918 .handle_heal_object(
2919 "bucket",
2920 "object",
2921 "",
2922 &HealOpts {
2923 pool: Some(1),
2924 set: Some(1),
2925 ..Default::default()
2926 },
2927 )
2928 .await
2929 .expect_err("invalid set scope should fail before suspended pool deferral");
2930
2931 assert!(matches!(err, StorageError::InvalidArgument(_, ref field, _) if field == "set"));
2932
2933 {
2934 let mut pool_meta = store.pool_meta.write().await;
2935 let decommission = pool_meta.pools[1]
2936 .decommission
2937 .as_mut()
2938 .expect("test pool should have decommission state");
2939 decommission.complete = true;
2940 }
2941 let (_, err) = store
2942 .handle_heal_object(
2943 "bucket",
2944 "object",
2945 "",
2946 &HealOpts {
2947 pool: Some(1),
2948 set: Some(0),
2949 ..Default::default()
2950 },
2951 )
2952 .await
2953 .expect("completed pool should return a terminal heal result");
2954
2955 assert!(matches!(
2956 err,
2957 Some(StorageError::InvalidArgument(_, ref field, ref reason))
2958 if field == "pool" && reason.contains("completed decommission")
2959 ));
2960
2961 for canceled in [false, true] {
2962 {
2963 let mut pool_meta = store.pool_meta.write().await;
2964 let decommission = pool_meta.pools[1]
2965 .decommission
2966 .as_mut()
2967 .expect("test pool should have decommission state");
2968 decommission.complete = false;
2969 decommission.failed = !canceled;
2970 decommission.canceled = canceled;
2971 }
2972 let (_, err) = store
2973 .handle_heal_object(
2974 "bucket",
2975 "object",
2976 "",
2977 &HealOpts {
2978 pool: Some(1),
2979 set: Some(0),
2980 ..Default::default()
2981 },
2982 )
2983 .await
2984 .expect("clearable terminal pool should return a deferred heal result");
2985
2986 assert!(matches!(err, Some(StorageError::SlowDown)));
2987 }
2988 }
2989
2990 #[tokio::test]
2991 async fn scoped_heal_bucket_blocks_before_dispatch_when_pool_is_suspended() {
2992 let mut store = minimal_heal_store().await;
2993 store.pool_meta = RwLock::new(PoolMeta {
2994 pools: vec![
2995 PoolStatus {
2996 id: 0,
2997 cmd_line: "pool-0".to_string(),
2998 last_update: OffsetDateTime::UNIX_EPOCH,
2999 decommission: None,
3000 },
3001 PoolStatus {
3002 id: 1,
3003 cmd_line: "pool-1".to_string(),
3004 last_update: OffsetDateTime::UNIX_EPOCH,
3005 decommission: Some(PoolDecommissionInfo {
3006 start_time: Some(OffsetDateTime::UNIX_EPOCH),
3007 ..Default::default()
3008 }),
3009 },
3010 ],
3011 ..Default::default()
3012 });
3013
3014 let err = store
3015 .handle_heal_bucket(
3016 "bucket",
3017 &HealOpts {
3018 pool: Some(1),
3019 ..Default::default()
3020 },
3021 )
3022 .await
3023 .expect_err("suspended pool must be blocked before bucket-heal fan-out");
3024 assert_eq!(err, Error::SlowDown);
3025
3026 store.pool_meta.write().await.pools[1]
3027 .decommission
3028 .as_mut()
3029 .expect("decommission state should exist")
3030 .complete = true;
3031 let err = store
3032 .handle_heal_bucket(
3033 "bucket",
3034 &HealOpts {
3035 pool: Some(1),
3036 ..Default::default()
3037 },
3038 )
3039 .await
3040 .expect_err("completed pool must remain fenced from bucket heal");
3041 assert!(
3042 matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
3043 if field == "pool" && reason.contains("completed decommission")),
3044 "unexpected completed-pool error: {err:?}"
3045 );
3046 }
3047
3048 #[tokio::test]
3049 async fn bucket_heal_blocks_before_dispatch_after_unreadable_pool_meta_replica() {
3050 let store = minimal_heal_store().await;
3051 store.pool_meta_save_gate.lock().await.observe_replicas(PoolMetaReplicaState {
3052 needs_repair: true,
3053 repair_write_safe: false,
3054 });
3055
3056 let err = store
3057 .handle_heal_bucket("bucket", &HealOpts::default())
3058 .await
3059 .expect_err("bucket heal must stay blocked until restart after an unreadable replica");
3060
3061 assert!(
3062 err.to_string()
3063 .contains("restart after all replicas are readable and consistent")
3064 );
3065 }
3066
3067 #[tokio::test]
3068 async fn bucket_heal_releases_save_gate_before_peer_dispatch_and_holds_movement_snapshot() {
3069 let mut store = minimal_heal_store().await;
3070 let started = Arc::new(tokio::sync::Notify::new());
3071 let release = Arc::new(tokio::sync::Notify::new());
3072 let peer: Box<dyn PeerS3Client> = Box::new(BlockingHealPeer {
3073 started: started.clone(),
3074 release: release.clone(),
3075 });
3076 store.peer_sys.clients = vec![Arc::new(peer)];
3077 let store = Arc::new(store);
3078 let movement_gate = store.ctx.data_movement_operation_gate();
3079 let mut heal = tokio::spawn({
3080 let store = store.clone();
3081 async move { store.handle_heal_bucket("bucket", &HealOpts::default()).await }
3082 });
3083
3084 tokio::time::timeout(std::time::Duration::from_secs(1), started.notified())
3085 .await
3086 .expect("peer dispatch should start");
3087 assert!(
3088 store.pool_meta_save_gate.try_lock().is_ok(),
3089 "coordinator must release its local save gate before waiting for peers"
3090 );
3091 assert!(
3092 movement_gate.try_write().is_err(),
3093 "bucket heal must hold the movement snapshot through peer dispatch"
3094 );
3095
3096 release.notify_one();
3097 tokio::time::timeout(std::time::Duration::from_secs(1), &mut heal)
3098 .await
3099 .expect("bucket heal should finish after peer release")
3100 .expect("bucket heal task should not panic")
3101 .expect("bucket heal should succeed");
3102 }
3103
3104 #[tokio::test]
3105 async fn bucket_heal_local_fanout_does_not_reenter_movement_read_behind_queued_writer() {
3106 let mut store = minimal_heal_store().await;
3107 let movement_gate = store.ctx.data_movement_operation_gate();
3108 let writer_queued = Arc::new(tokio::sync::Notify::new());
3109 let writer_acquired = Arc::new(tokio::sync::Notify::new());
3110 let peer: Box<dyn PeerS3Client> = Box::new(WriterQueuedLocalHealPeer {
3111 movement_gate: movement_gate.clone(),
3112 writer_queued: writer_queued.clone(),
3113 writer_acquired: writer_acquired.clone(),
3114 });
3115 store.peer_sys.clients = vec![Arc::new(peer)];
3116 let store = Arc::new(store);
3117 let mut heal = tokio::spawn({
3118 let store = store.clone();
3119 async move { store.handle_heal_bucket("bucket", &HealOpts::default()).await }
3120 });
3121
3122 tokio::time::timeout(std::time::Duration::from_secs(1), writer_queued.notified())
3123 .await
3124 .expect("movement writer should queue during local peer lookup");
3125 tokio::time::timeout(std::time::Duration::from_secs(1), &mut heal)
3126 .await
3127 .expect("local fan-out must not reenter movement read behind the queued writer")
3128 .expect("bucket heal task should not panic")
3129 .expect("bucket heal should succeed");
3130 tokio::time::timeout(std::time::Duration::from_secs(1), writer_acquired.notified())
3131 .await
3132 .expect("queued movement writer should proceed after bucket heal releases its read guard");
3133 }
3134
3135 #[tokio::test]
3136 #[serial_test::serial]
3137 async fn unscoped_heal_object_suspended_owner_semantics() {
3138 let (_temp_dir, store, shutdown) = multi_pool_heal_store().await;
3139 let bucket = format!("heal-owner-{}", Uuid::new_v4().simple());
3140 let active_object = "active-owner";
3141 let suspended_only_object = "suspended-only";
3142 let duplicate_object = "duplicate-owner";
3143 let marker_object = "marker-owner";
3144 let quorum_object = "quorum-owner";
3145 store
3146 .make_bucket(&bucket, &MakeBucketOptions::default())
3147 .await
3148 .expect("bucket should be created in all pools");
3149
3150 let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec());
3151 store.pools[0]
3152 .put_object(
3153 &bucket,
3154 active_object,
3155 &mut active_reader,
3156 &ObjectOptions {
3157 write_completion: crate::object_api::WriteCompletion::TailDrained,
3159 ..Default::default()
3160 },
3161 )
3162 .await
3163 .expect("active owner object should be written");
3164 let active_disks = store.pools[0].disk_set[0].disks.read().await.clone();
3165 let missing_active_disk = active_disks[0].clone().expect("active disk should be online");
3166 missing_active_disk
3167 .delete(
3168 &bucket,
3169 active_object,
3170 DeleteOptions {
3171 recursive: true,
3172 immediate: true,
3173 ..Default::default()
3174 },
3175 )
3176 .await
3177 .expect("active owner shard should be removed for repair");
3178 assert!(
3179 missing_active_disk.read_xl(&bucket, active_object, false).await.is_err(),
3180 "the active owner fixture must start with one missing metadata copy"
3181 );
3182
3183 let mut suspended_reader = PutObjReader::from_vec(b"suspended owner".to_vec());
3184 store.pools[1]
3185 .put_object(&bucket, suspended_only_object, &mut suspended_reader, &ObjectOptions::default())
3186 .await
3187 .expect("suspended owner object should be written");
3188 for (pool_index, mod_time) in [1_i64, 2_i64].into_iter().enumerate() {
3189 let mut duplicate_reader = PutObjReader::from_vec(format!("duplicate-pool-{pool_index}").into_bytes());
3190 store.pools[pool_index]
3191 .put_object(
3192 &bucket,
3193 duplicate_object,
3194 &mut duplicate_reader,
3195 &ObjectOptions {
3196 mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)),
3197 write_completion: crate::object_api::WriteCompletion::TailDrained,
3198 ..Default::default()
3199 },
3200 )
3201 .await
3202 .expect("duplicate owner object should be written");
3203 }
3204 let duplicate_missing_disk = store.pools[0].disk_set[0].disks.read().await[0]
3205 .clone()
3206 .expect("duplicate active owner disk should be online");
3207 duplicate_missing_disk
3208 .delete(
3209 &bucket,
3210 duplicate_object,
3211 DeleteOptions {
3212 recursive: true,
3213 immediate: true,
3214 ..Default::default()
3215 },
3216 )
3217 .await
3218 .expect("duplicate active owner shard should be removed for repair");
3219 let history_version = Uuid::new_v4();
3220 let mut history_reader = PutObjReader::from_vec(b"marker history".to_vec());
3221 store.pools[0]
3222 .put_object(
3223 &bucket,
3224 marker_object,
3225 &mut history_reader,
3226 &ObjectOptions {
3227 versioned: true,
3228 version_id: Some(history_version.to_string()),
3229 mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
3230 ..Default::default()
3231 },
3232 )
3233 .await
3234 .expect("versioned marker history should be written");
3235 store.pools[0]
3236 .delete_object(
3237 &bucket,
3238 marker_object,
3239 ObjectOptions {
3240 versioned: true,
3241 mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
3242 ..Default::default()
3243 },
3244 )
3245 .await
3246 .expect("delete marker should be written");
3247 let mut quorum_reader = PutObjReader::from_vec(b"quorum boundary".to_vec());
3248 store.pools[0]
3249 .put_object(&bucket, quorum_object, &mut quorum_reader, &ObjectOptions::default())
3250 .await
3251 .expect("quorum boundary object should be written");
3252 {
3253 let mut pool_meta = store.pool_meta.write().await;
3254 let mut next = PoolMeta::new(&store.pools, &pool_meta);
3255 next.pools[1].decommission = Some(PoolDecommissionInfo {
3256 start_time: Some(OffsetDateTime::UNIX_EPOCH),
3257 ..Default::default()
3258 });
3259 *pool_meta = next;
3260 }
3261
3262 let (_, duplicate_owner) = store
3263 .get_latest_object_info_with_idx(&bucket, duplicate_object, &ObjectOptions::default())
3264 .await
3265 .expect("duplicate owner should resolve");
3266 assert_eq!(duplicate_owner, 1, "latest duplicate must win when all pools are eligible");
3267 let (_, active_duplicate_owner) = store
3268 .get_latest_object_info_with_idx(
3269 &bucket,
3270 duplicate_object,
3271 &ObjectOptions {
3272 skip_decommissioned: true,
3273 ..Default::default()
3274 },
3275 )
3276 .await
3277 .expect("active duplicate owner should resolve");
3278 assert_eq!(
3279 active_duplicate_owner, 0,
3280 "suspended duplicate must be excluded from active owner selection"
3281 );
3282 let (duplicate_result, duplicate_err) = store
3283 .handle_heal_object(&bucket, duplicate_object, "", &HealOpts::default())
3284 .await
3285 .expect("duplicate owner heal should complete through the production path");
3286 assert_eq!(duplicate_result.object, duplicate_object);
3287 assert!(duplicate_err.is_none(), "active duplicate should be repaired: {duplicate_err:?}");
3288 assert!(
3289 duplicate_missing_disk.read_xl(&bucket, duplicate_object, false).await.is_ok(),
3290 "production heal must repair the active duplicate owner rather than the suspended owner"
3291 );
3292 let (marker_info, marker_owner) = store
3293 .get_latest_object_info_with_idx(
3294 &bucket,
3295 marker_object,
3296 &ObjectOptions {
3297 skip_decommissioned: true,
3298 versioned: true,
3299 ..Default::default()
3300 },
3301 )
3302 .await
3303 .expect("latest delete marker should resolve");
3304 assert_eq!(marker_owner, 0);
3305 assert!(marker_info.delete_marker, "latest version must preserve delete-marker semantics");
3306
3307 let (active_result, active_err) = store
3308 .handle_heal_object(&bucket, active_object, "", &HealOpts::default())
3309 .await
3310 .expect("unscoped active-owner heal should complete");
3311 assert_eq!(active_result.object, active_object);
3312 assert!(active_err.is_none(), "active owner must be selected even with a suspended pool");
3313 assert!(
3314 missing_active_disk.read_xl(&bucket, active_object, false).await.is_ok(),
3315 "active owner heal must write the missing disk metadata: result={active_result:?}, err={active_err:?}"
3316 );
3317 assert!(
3318 store.pools[1]
3319 .get_object_info(&bucket, active_object, &ObjectOptions::default())
3320 .await
3321 .is_err(),
3322 "the suspended pool must not be written for an active-owner object"
3323 );
3324
3325 let (suspended_result, suspended_err) = store
3326 .handle_heal_object(&bucket, suspended_only_object, "", &HealOpts::default())
3327 .await
3328 .expect("unscoped suspended-only heal should return a terminal result");
3329 assert!(suspended_result.object.is_empty());
3330 assert!(matches!(suspended_err, Some(Error::FileNotFound)));
3331 assert!(
3332 store.pools[1]
3333 .get_object_info(&bucket, suspended_only_object, &ObjectOptions::default())
3334 .await
3335 .is_ok(),
3336 "suspended-only data must remain untouched when unscoped heal reports absent"
3337 );
3338
3339 let (_, explicit_err) = store
3340 .handle_heal_object(
3341 &bucket,
3342 suspended_only_object,
3343 "",
3344 &HealOpts {
3345 pool: Some(1),
3346 ..Default::default()
3347 },
3348 )
3349 .await
3350 .expect("explicit suspended-owner heal should return a mapped error");
3351 assert!(matches!(explicit_err, Some(Error::SlowDown)));
3352
3353 let original_quorum_disks = store.pools[0].disk_set[0].disks.read().await.clone();
3354 let surviving_quorum_disk = original_quorum_disks[3].clone();
3355 *store.pools[0].disk_set[0].disks.write().await = vec![None, None, None, surviving_quorum_disk];
3356 let (_, quorum_err) = store
3357 .handle_heal_object(&bucket, quorum_object, "", &HealOpts::default())
3358 .await
3359 .expect("quorum boundary heal should return a mapped result");
3360 *store.pools[0].disk_set[0].disks.write().await = original_quorum_disks;
3361 let quorum_err = quorum_err
3362 .as_ref()
3363 .expect("heal must fail closed when capacity admission cannot verify pool metadata");
3364 let quorum_failure = quorum_err
3365 .pool_metadata_failure()
3366 .expect("capacity admission failure should preserve typed pool metadata context");
3367 assert_eq!(
3368 quorum_failure.kind,
3369 crate::error::PoolMetadataFailure::ReadUnavailable,
3370 "read-only capacity admission failure must remain retryable"
3371 );
3372 assert_eq!(quorum_failure.operation, "target capacity admission failed");
3373 assert_eq!(quorum_failure.phase, "pool_read");
3374 assert!(
3375 store.pool_meta_writes_ready().await,
3376 "read-only capacity admission failure must not latch the pool metadata writer"
3377 );
3378 shutdown.cancel();
3379 }
3380
3381 #[tokio::test]
3382 async fn handle_heal_format_continues_after_a_pool_error() {
3383 let canonical_format = FormatV3::new(1, 3);
3384 let mut foreign_format = canonical_format.clone();
3385 foreign_format.id = Uuid::new_v4();
3386 let mut temp_dirs = Vec::new();
3387 let mut endpoints = Vec::new();
3388 let mut disks = Vec::new();
3389
3390 for disk_index in 0..3 {
3391 let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
3392 let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
3393 .expect("temporary endpoint should parse");
3394 endpoint.set_pool_index(0);
3395 endpoint.set_set_index(0);
3396 endpoint.set_disk_index(disk_index);
3397 let disk = new_disk(
3398 &endpoint,
3399 &DiskOption {
3400 cleanup: false,
3401 health_check: false,
3402 },
3403 )
3404 .await
3405 .expect("temporary disk should open");
3406 let mut disk_format = foreign_format.clone();
3407 disk_format.erasure.this = foreign_format.erasure.sets[0][disk_index];
3408 save_format_file(&Some(disk.clone()), &Some(disk_format))
3409 .await
3410 .expect("foreign format should be written");
3411 temp_dirs.push(temp_dir);
3412 endpoints.push(endpoint);
3413 disks.push(Some(disk));
3414 }
3415
3416 let pool_endpoints = PoolEndpoints {
3417 legacy: false,
3418 set_count: 1,
3419 drives_per_set: 3,
3420 endpoints: Endpoints::from(endpoints),
3421 cmd_line: "foreign-format-majority-test".to_string(),
3422 platform: "test".to_string(),
3423 };
3424 let pool = Sets::new(disks, &pool_endpoints, &canonical_format, 0, 1)
3425 .await
3426 .expect("test pool should build around the cached canonical format");
3427
3428 let mut recoverable_format = FormatV3::new(1, 3);
3429 recoverable_format.id = canonical_format.id;
3430 let mut recoverable_temp_dirs = Vec::new();
3431 let mut recoverable_endpoints = Vec::new();
3432 let mut recoverable_disks = Vec::new();
3433 let mut unformatted_disk = None;
3434 for disk_index in 0..3 {
3435 let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
3436 let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
3437 .expect("temporary endpoint should parse");
3438 endpoint.set_pool_index(1);
3439 endpoint.set_set_index(0);
3440 endpoint.set_disk_index(disk_index);
3441 let disk = new_disk(
3442 &endpoint,
3443 &DiskOption {
3444 cleanup: false,
3445 health_check: false,
3446 },
3447 )
3448 .await
3449 .expect("temporary disk should open");
3450 if disk_index < 2 {
3451 let mut disk_format = recoverable_format.clone();
3452 disk_format.erasure.this = recoverable_format.erasure.sets[0][disk_index];
3453 save_format_file(&Some(disk.clone()), &Some(disk_format))
3454 .await
3455 .expect("recoverable format should be written");
3456 } else {
3457 unformatted_disk = Some(disk.clone());
3458 }
3459 recoverable_temp_dirs.push(temp_dir);
3460 recoverable_endpoints.push(endpoint);
3461 recoverable_disks.push(Some(disk));
3462 }
3463 let recoverable_pool_endpoints = PoolEndpoints {
3464 legacy: false,
3465 set_count: 1,
3466 drives_per_set: 3,
3467 endpoints: Endpoints::from(recoverable_endpoints),
3468 cmd_line: "recoverable-format-test".to_string(),
3469 platform: "test".to_string(),
3470 };
3471 let recoverable_pool = Sets::new(recoverable_disks, &recoverable_pool_endpoints, &recoverable_format, 1, 1)
3472 .await
3473 .expect("recoverable test pool should build");
3474
3475 let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints.clone(), recoverable_pool_endpoints.clone()]);
3476 let store = ECStore {
3477 id: canonical_format.id,
3478 disk_map: HashMap::new(),
3479 pools: vec![pool, recoverable_pool],
3480 peer_sys: S3PeerSys::new(&endpoint_pools),
3481 pool_meta: RwLock::new(PoolMeta::default()),
3482 rebalance_meta: RwLock::new(None),
3483 decommission_cancelers: RwLock::new(Vec::new()),
3484 start_gate: Mutex::new(()),
3485 pool_meta_save_gate: Mutex::default(),
3486 ctx: crate::runtime::instance::bootstrap_ctx(),
3487 bucket_fence_registry: std::sync::Arc::default(),
3488 };
3489
3490 let err = store
3491 .handle_heal_format(false)
3492 .await
3493 .expect_err("missing pool metadata must fail closed before format writes");
3494 assert!(err.to_string().contains("no durable bootstrap identity or pool.bin replica"));
3495 store
3496 .ensure_pool_meta_side_effects_safe("missing format-heal metadata side effect")
3497 .await
3498 .expect_err("missing metadata must latch the format-heal write gate");
3499
3500 let pool_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
3501 pool_meta
3502 .save_for_startup(store.pools.clone())
3503 .await
3504 .expect("pool metadata should be persisted before format heal");
3505 *store.pool_meta_save_gate.lock().await = PoolMetaWriteState::default();
3506
3507 let (result, err) = store
3508 .handle_heal_format(false)
3509 .await
3510 .expect("format heal should return the typed pool error");
3511 assert!(
3512 matches!(err, Some(StorageError::CorruptedFormat)),
3513 "foreign format majority must not be downgraded to a successful heal: {err:?}"
3514 );
3515 assert_eq!(result.disk_count, 3, "the recoverable pool should still be inspected");
3516 let healed = load_format_erasure(&unformatted_disk.expect("the unformatted disk handle should be retained"), true)
3517 .await
3518 .expect("the later pool should be healed despite the first pool error");
3519 assert_eq!(healed.erasure.this, recoverable_format.erasure.sets[0][2]);
3520
3521 let mut completed_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
3522 for status in &mut completed_meta.pools {
3523 status.decommission = Some(PoolDecommissionInfo {
3524 complete: true,
3525 ..Default::default()
3526 });
3527 }
3528 completed_meta
3529 .save(store.pools.clone())
3530 .await
3531 .expect("completed pool metadata should be persisted");
3532 let (_, err) = store
3533 .handle_heal_format(false)
3534 .await
3535 .expect("completed pools should be reported as a no-op");
3536 assert!(matches!(err, Some(StorageError::NoHealRequired)));
3537 }
3538}