1use super::*;
16use crate::core::pools::{POOL_META_NAME, load_pool_meta_identity_observing};
17use crate::disk::{self, error::DiskError};
18use crate::services::rebalance::{REBAL_META_NAME, RebalStatus};
19use crate::set_disk::get_lock_acquire_timeout;
20use crate::storage_api_contracts::heal::HealOperations as _;
21use crate::storage_api_contracts::namespace::NamespaceLocking as _;
22use rustfs_common::mrf_channel::MrfDeleteMarkerPurge;
23use rustfs_lock::NamespaceLockGuard;
24use std::collections::BTreeSet;
25use tracing::trace;
26
27const LOG_COMPONENT_ECSTORE: &str = "ecstore";
28const LOG_SUBSYSTEM_HEAL: &str = "heal";
29const EVENT_HEAL_ABANDONED_PARTS: &str = "heal_abandoned_parts";
30const EVENT_HEAL_FORMAT_COMPLETED: &str = "heal_format_completed";
31const EVENT_HEAL_OBJECT_STARTED: &str = "heal_object_started";
32
33pub(crate) struct BucketHealScope {
35 pub(crate) bucket: String,
36 pub(crate) incarnation: uuid::Uuid,
37 pub(crate) store: Arc<ECStore>,
38 fence: super::BucketIncarnationFenceGuard,
39}
40
41impl BucketHealScope {
42 pub(crate) fn check(&self) -> disk::error::Result<()> {
43 if self.fence.is_lock_lost() {
44 return Err(DiskError::other("bucket heal incarnation fence was lost"));
45 }
46 Ok(())
47 }
48}
49
50tokio::task_local! {
51 static BUCKET_HEAL_SCOPE: Arc<BucketHealScope>;
52}
53
54pub(crate) fn bucket_heal_scope(bucket: &str) -> Option<Arc<BucketHealScope>> {
55 BUCKET_HEAL_SCOPE
56 .try_with(|scope| (scope.bucket == bucket).then(|| scope.clone()))
57 .ok()
58 .flatten()
59}
60
61#[derive(Debug)]
64pub struct HealObjectAbsenceProof {
65 pub bucket: String,
66 pub object: String,
67 pub version_id: String,
68 pub bucket_incarnation_id: Uuid,
69 pub pool_index: Option<usize>,
70 pub set_index: Option<usize>,
71 pub locations: Vec<(usize, usize)>,
72 pub removed: bool,
73}
74
75#[derive(Debug)]
76pub struct HealObjectStorageResult {
77 pub item: HealResultItem,
78 pub error: Option<Error>,
79 pub absence: Option<HealObjectAbsenceProof>,
80}
81
82fn invalid_heal_pool_index(pool_idx: usize, pool_count: usize) -> Error {
83 StorageError::InvalidArgument(
84 "heal".to_string(),
85 "pool".to_string(),
86 format!("invalid heal pool index {pool_idx} for {pool_count} pools"),
87 )
88}
89
90fn is_pool_meta_object(bucket: &str, object: &str) -> bool {
91 bucket == RUSTFS_META_BUCKET && object == POOL_META_NAME
92}
93
94#[derive(Debug, Clone, Copy)]
95enum HealFormatPoolSkip {
96 Completed,
97 Retryable,
98}
99
100fn classify_heal_format_pool(
101 pool_idx: usize,
102 pool_cmd_line: &str,
103 pool_meta: &PoolMeta,
104 rebalance_meta: Option<&RebalanceMeta>,
105) -> Option<HealFormatPoolSkip> {
106 let Some(pool) = pool_meta.pools.get(pool_idx) else {
107 return Some(HealFormatPoolSkip::Retryable);
108 };
109
110 if pool.id != pool_idx || pool_cmd_line.is_empty() || pool.cmd_line.is_empty() || pool.cmd_line != pool_cmd_line {
111 return Some(HealFormatPoolSkip::Retryable);
112 }
113
114 if let Some(decommission) = pool.decommission.as_ref() {
115 if decommission.complete {
116 return Some(HealFormatPoolSkip::Completed);
117 }
118 if decommission.failed || decommission.canceled || decommission.queued || pool_meta.is_suspended(pool_idx) {
119 return Some(HealFormatPoolSkip::Retryable);
120 }
121 }
122
123 if let Some(meta) = rebalance_meta {
124 let Some(pool_stats) = meta.pool_stats.get(pool_idx) else {
125 return Some(HealFormatPoolSkip::Retryable);
126 };
127 if pool_stats.info.stopping || (pool_stats.participating && pool_stats.info.status == RebalStatus::Started) {
128 return Some(HealFormatPoolSkip::Retryable);
129 }
130 }
131
132 None
133}
134
135fn heal_format_pool_skip_error(skip: HealFormatPoolSkip) -> Error {
136 match skip {
137 HealFormatPoolSkip::Completed => StorageError::NoHealRequired,
138 HealFormatPoolSkip::Retryable => StorageError::SlowDown,
139 }
140}
141
142fn heal_format_fence_lost_error() -> Error {
143 StorageError::SlowDown
144}
145
146impl ECStore {
147 pub async fn purge_delete_marker_with_proof(
148 self: &Arc<Self>,
149 bucket: &str,
150 object: &str,
151 version_id: &str,
152 purge: &MrfDeleteMarkerPurge,
153 opts: &HealOpts,
154 ) -> Result<bool> {
155 if opts.dry_run || opts.no_lock || !purge.is_valid() {
156 return Err(StorageError::PreconditionFailed);
157 }
158 let version = Uuid::parse_str(version_id)
159 .ok()
160 .filter(|version| !version.is_nil())
161 .ok_or(StorageError::PreconditionFailed)?;
162 let (Some(pool_index), Some(set_index)) = (opts.pool, opts.set) else {
163 return Err(StorageError::PreconditionFailed);
164 };
165 let mut marker = rustfs_filemeta::MetaDeleteMarker::default();
166 let consumed = marker.unmarshal_msg(&purge.marker)?;
167 if usize::try_from(consumed).ok() != Some(purge.marker.len())
168 || marker.version_id != Some(version)
169 || marker.stable_identity() != purge.marker_identity
170 {
171 return Err(StorageError::PreconditionFailed);
172 }
173 let marker_info = marker.clone().into_fileinfo(bucket, object, false)?;
174 if !marker_info.is_canonical_delete_marker()
175 || marker_info.delete_marker_incarnation() != Some(purge.marker_incarnation_id)
176 {
177 return Err(StorageError::PreconditionFailed);
178 }
179
180 let current = self.bucket_incarnation_id_from_disk(bucket).await?;
181 if current.is_nil() {
182 return Err(StorageError::BucketNotFound(bucket.to_owned()));
183 }
184 let original = purge.bucket_incarnation_id;
185 let object = object.to_owned();
186 self.run_bucket_heal_at_incarnation(bucket, current, opts, move |store, bucket, _opts| async move {
187 let scope = bucket_heal_scope(&bucket).ok_or(StorageError::PreconditionFailed)?;
188 scope.check()?;
189 if original != current {
190 let retirement_store = crate::bucket::metadata_sys::object_store_if_initialized_in(&store.ctx)
191 .await
192 .ok_or(StorageError::PreconditionFailed)?;
193 if !crate::bucket::retirement::is_retired(retirement_store, &bucket, original).await? {
194 return Err(StorageError::PreconditionFailed);
195 }
196 }
197 let pool = store
198 .pools
199 .get(pool_index)
200 .ok_or_else(|| invalid_heal_pool_index(pool_index, store.pools.len()))?;
201 let set = pool.disk_set.get(set_index).cloned().ok_or_else(|| {
202 StorageError::InvalidArgument(
203 "heal".to_string(),
204 "set".to_string(),
205 format!("invalid heal set index {set_index} for pool {pool_index}"),
206 )
207 })?;
208 set.purge_delete_marker_exact(&bucket, &object, version, marker).await
209 })
210 .await
211 }
212
213 pub(super) async fn run_bucket_heal_at_incarnation<T, F, Fut>(
214 self: &Arc<Self>,
215 bucket: &str,
216 expected: uuid::Uuid,
217 opts: &HealOpts,
218 operation: F,
219 ) -> Result<T>
220 where
221 T: Send + 'static,
222 F: FnOnce(Arc<Self>, String, HealOpts) -> Fut + Send + 'static,
223 Fut: std::future::Future<Output = Result<T>> + Send + 'static,
224 {
225 if expected.is_nil() || self.ctx.lock_manager().is_disabled() {
226 return Err(Error::other(
227 "incarnation-bound bucket heal requires a valid identity and namespace locking",
228 ));
229 }
230 let fence = self.acquire_bucket_incarnation_fence(bucket, expected).await?;
232 let scope = Arc::new(BucketHealScope {
233 bucket: bucket.to_owned(),
234 incarnation: expected,
235 store: self.clone(),
236 fence,
237 });
238 let store = self.clone();
239 let bucket = bucket.to_owned();
240 let opts = *opts;
241 tokio::spawn(BUCKET_HEAL_SCOPE.scope(scope.clone(), async move {
244 scope.check().map_err(Error::from)?;
245 let result = operation(store, bucket, opts).await;
246 scope.check().map_err(Error::from)?;
247 result
248 }))
249 .await
250 .map_err(|error| Error::other_with_context("bucket heal owner task failed", error))?
251 }
252
253 pub async fn heal_bucket_at_incarnation(
254 self: &Arc<Self>,
255 bucket: &str,
256 expected: uuid::Uuid,
257 opts: &HealOpts,
258 ) -> Result<HealResultItem> {
259 self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
260 store.heal_bucket(&bucket, &opts).await
261 })
262 .await
263 }
264
265 pub async fn heal_local_bucket_at_incarnation(
266 self: &Arc<Self>,
267 bucket: &str,
268 expected: uuid::Uuid,
269 opts: &HealOpts,
270 fenced_pools: Vec<usize>,
271 pools: Option<Vec<usize>>,
272 ) -> Result<HealResultItem> {
273 self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
274 let peer =
275 crate::cluster::rpc::peer_s3_client::LocalPeerS3Client::new_with_instance_ctx(None, pools, store.ctx.clone());
276 crate::cluster::rpc::peer_s3_client::PeerS3Client::heal_bucket_with_fence(&peer, &bucket, &opts, &fenced_pools)
277 .await
278 .map_err(Error::from)
279 })
280 .await
281 }
282
283 pub async fn heal_object_at_incarnation(
284 self: &Arc<Self>,
285 bucket: &str,
286 object: &str,
287 version_id: &str,
288 expected: uuid::Uuid,
289 opts: &HealOpts,
290 ) -> Result<HealObjectStorageResult> {
291 let object = object.to_owned();
292 let version_id = version_id.to_owned();
293 self.run_bucket_heal_at_incarnation(bucket, expected, opts, |store, bucket, opts| async move {
294 store.heal_object_with_proof(&bucket, &object, &version_id, &opts).await
295 })
296 .await
297 }
298
299 async fn acquire_heal_format_fence(
300 &self,
301 ) -> Result<(
302 tokio::sync::MutexGuard<'_, PoolMetaWriteState>,
303 NamespaceLockGuard,
304 NamespaceLockGuard,
305 PoolMeta,
306 Option<RebalanceMeta>,
307 )> {
308 let metadata_pool = self
309 .pools
310 .first()
311 .cloned()
312 .ok_or_else(|| Error::other("heal format requires at least one storage pool"))?;
313 let mut write_state = self.pool_meta_save_gate.lock().await;
314 write_state.ensure_write_safe("heal format fence failed")?;
315
316 let pool_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
322 let pool_guard = pool_lock.get_read_lock(get_lock_acquire_timeout()).await?;
323 let rebalance_lock = metadata_pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
324 let rebalance_guard = rebalance_lock.get_write_lock(get_lock_acquire_timeout()).await?;
325
326 if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
327 return Err(heal_format_fence_lost_error());
328 }
329
330 load_pool_meta_identity_observing(self.pools.clone(), &mut write_state).await?;
331 let mut pool_meta = PoolMeta::default();
332 let replica_state = pool_meta
333 .load_no_lock_from_replicas_observing(self.pools.clone(), &mut write_state)
334 .await?;
335 write_state.observe_replicas(replica_state);
336 write_state.ensure_missing_metadata_can_initialize()?;
337 write_state.ensure_write_safe("heal format fence failed")?;
338 if pool_meta.pools.len() != self.pools.len()
339 || pool_meta.pools.iter().enumerate().any(|(pool_idx, pool)| {
340 pool.id != pool_idx || pool.cmd_line.is_empty() || pool.cmd_line != self.pools[pool_idx].endpoints.cmd_line
341 })
342 {
343 return Err(heal_format_fence_lost_error());
344 }
345
346 let mut rebalance_meta = RebalanceMeta::new();
347 let rebalance_meta = match rebalance_meta
348 .load_with_opts(
349 metadata_pool,
350 ObjectOptions {
351 no_lock: true,
352 ..Default::default()
353 },
354 )
355 .await
356 {
357 Ok(()) => Some(rebalance_meta),
358 Err(Error::ConfigNotFound) => None,
359 Err(err) => return Err(err),
360 };
361
362 if rebalance_meta
363 .as_ref()
364 .is_some_and(|meta| meta.pool_stats.len() != self.pools.len())
365 {
366 return Err(heal_format_fence_lost_error());
367 }
368
369 write_state.ensure_write_safe("heal format fence failed")?;
370 if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
371 return Err(heal_format_fence_lost_error());
372 }
373
374 Ok((write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta))
375 }
376
377 fn get_pools_for_heal_object(&self, opts: &HealOpts) -> Result<Vec<Arc<Sets>>> {
378 match opts.pool {
379 Some(pool_idx) => Ok(vec![
380 self.pools
381 .get(pool_idx)
382 .cloned()
383 .ok_or_else(|| invalid_heal_pool_index(pool_idx, self.pools.len()))?,
384 ]),
385 None => Ok(self.pools.clone()),
386 }
387 }
388
389 pub fn replacement_pool_metadata_required(&self, pool_index: usize, set_index: usize) -> Result<bool> {
393 let pool = self
394 .pools
395 .get(pool_index)
396 .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
397 let target_set = pool.get_disks_for_heal_object(
398 POOL_META_NAME,
399 &HealOpts {
400 set: Some(set_index),
401 ..Default::default()
402 },
403 )?;
404 Ok(Arc::ptr_eq(&target_set, &pool.get_disks_by_key(POOL_META_NAME)))
405 }
406
407 pub async fn heal_erasure_set_scopes(&self, opts: &HealOpts) -> Result<Vec<(usize, usize)>> {
409 let pools = self.get_pools_for_heal_object(opts)?;
410 let pool_meta = self.pool_meta.read().await;
411 let mut scopes = Vec::new();
412
413 for pool in pools {
414 let suspended_complete = pool_meta.is_suspended(pool.pool_idx).then(|| {
415 pool_meta
416 .pools
417 .get(pool.pool_idx)
418 .and_then(|status| status.decommission.as_ref())
419 .is_some_and(|decommission| decommission.complete)
420 });
421 if let Some(complete) = suspended_complete {
422 if opts.pool.is_some() {
423 return Err(if complete {
424 StorageError::InvalidArgument(
425 "heal".to_string(),
426 "pool".to_string(),
427 format!("heal pool {} has completed decommission", pool.pool_idx),
428 )
429 } else {
430 Error::SlowDown
431 });
432 }
433 continue;
434 }
435
436 if let Some(set_idx) = opts.set {
437 if set_idx >= pool.disk_set.len() {
438 return Err(StorageError::InvalidArgument(
439 "heal".to_string(),
440 "set".to_string(),
441 format!(
442 "invalid heal set index {set_idx} for pool {} with {} sets",
443 pool.pool_idx,
444 pool.disk_set.len()
445 ),
446 ));
447 }
448 scopes.push((pool.pool_idx, set_idx));
449 } else {
450 scopes.extend((0..pool.disk_set.len()).map(|set_idx| (pool.pool_idx, set_idx)));
451 }
452 }
453
454 if scopes.is_empty() {
455 return Err(Error::SlowDown);
456 }
457
458 Ok(scopes)
459 }
460
461 #[instrument(skip(self))]
462 pub(super) async fn handle_heal_format(&self, dry_run: bool) -> Result<(HealResultItem, Option<Error>)> {
463 let mut r = HealResultItem {
464 heal_item_type: HealItemType::Metadata.to_string(),
465 detail: "disk-format".to_string(),
466 ..Default::default()
467 };
468
469 let mut count_no_heal = 0;
470 let mut count_completed = 0;
471 let mut first_error = None;
472 for (pool_idx, pool) in self.pools.iter().enumerate() {
473 let (mut write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta) =
474 self.acquire_heal_format_fence().await?;
475 if pool_guard.is_lock_lost() || rebalance_guard.is_lock_lost() {
476 first_error.get_or_insert(heal_format_fence_lost_error());
477 break;
478 }
479 if let Some(skip) = classify_heal_format_pool(pool_idx, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref())
480 {
481 if matches!(skip, HealFormatPoolSkip::Completed) {
482 count_completed += 1;
483 } else {
484 first_error.get_or_insert(heal_format_pool_skip_error(skip));
485 }
486 continue;
487 }
488
489 let fence_lost = || {
490 let lost = pool_guard.is_lock_lost()
491 || rebalance_guard.is_lock_lost()
492 || write_state.ensure_write_safe("heal format write fence failed").is_err();
493 if lost {
494 write_state.block_writes_after_fence_loss();
495 }
496 lost
497 };
498 let (mut result, err) = pool.heal_format_with_fence(dry_run, fence_lost).await?;
499 if let Some(err) = err {
500 match err {
501 StorageError::NoHealRequired => {
502 count_no_heal += 1;
503 }
504 err => {
505 first_error.get_or_insert(err);
506 }
507 }
508 }
509 r.disk_count += result.disk_count;
510 r.set_count += result.set_count;
511 r.before.drives.append(&mut result.before.drives);
512 r.after.drives.append(&mut result.after.drives);
513
514 let fence_lost = pool_guard.is_lock_lost()
517 || rebalance_guard.is_lock_lost()
518 || write_state.ensure_write_safe("heal format publication fence failed").is_err();
519 if fence_lost {
520 write_state.block_writes_after_fence_loss();
521 first_error.get_or_insert(heal_format_fence_lost_error());
522 break;
523 }
524 }
525 if let Some(err) = first_error {
526 return Ok((r, Some(err)));
527 }
528 if count_no_heal + count_completed == self.pools.len() {
529 info!(
530 event = EVENT_HEAL_FORMAT_COMPLETED,
531 component = LOG_COMPONENT_ECSTORE,
532 subsystem = LOG_SUBSYSTEM_HEAL,
533 dry_run,
534 result = "no_heal_required",
535 pool_count = self.pools.len(),
536 "Heal format completed"
537 );
538 return Ok((r, Some(StorageError::NoHealRequired)));
539 }
540 info!(
541 event = EVENT_HEAL_FORMAT_COMPLETED,
542 component = LOG_COMPONENT_ECSTORE,
543 subsystem = LOG_SUBSYSTEM_HEAL,
544 dry_run,
545 result = "healed_or_inspected",
546 disk_count = r.disk_count,
547 set_count = r.set_count,
548 "Heal format completed"
549 );
550 Ok((r, None))
551 }
552
553 #[instrument(skip(self, targets), fields(pool_index, set_index, target_count = targets.len()))]
554 pub async fn heal_replacement_format(
555 &self,
556 dry_run: bool,
557 pool_index: usize,
558 set_index: usize,
559 targets: &[String],
560 ) -> Result<(HealResultItem, Option<Error>)> {
561 let pool = self
562 .pools
563 .get(pool_index)
564 .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
565 let set = pool.disk_set.get(set_index).cloned().ok_or_else(|| {
566 StorageError::InvalidArgument(
567 "heal".to_string(),
568 "set".to_string(),
569 format!("invalid heal set index {set_index} for pool {pool_index}"),
570 )
571 })?;
572
573 let (mut write_state, pool_guard, rebalance_guard, pool_meta, rebalance_meta) = self.acquire_heal_format_fence().await?;
574 if let Some(skip) = classify_heal_format_pool(pool_index, &pool.endpoints.cmd_line, &pool_meta, rebalance_meta.as_ref()) {
575 return Ok((HealResultItem::default(), Some(heal_format_pool_skip_error(skip))));
576 }
577
578 let fence_lost = || {
579 let lost = pool_guard.is_lock_lost()
580 || rebalance_guard.is_lock_lost()
581 || write_state
582 .ensure_write_safe("replacement format write fence failed")
583 .is_err();
584 if lost {
585 write_state.block_writes_after_fence_loss();
586 }
587 lost
588 };
589 let result = set.heal_replacement_format_with_fence(dry_run, targets, fence_lost).await?;
590 let fence_lost = pool_guard.is_lock_lost()
591 || rebalance_guard.is_lock_lost()
592 || write_state
593 .ensure_write_safe("replacement format publication fence failed")
594 .is_err();
595 if fence_lost {
596 write_state.block_writes_after_fence_loss();
597 return Ok((result.0, Some(heal_format_fence_lost_error())));
598 }
599 Ok(result)
600 }
601
602 #[instrument(skip(self, targets), fields(pool_index, set_index, target_count = targets.len()))]
603 pub async fn replacement_targets_have_version(
604 &self,
605 bucket: &str,
606 object: &str,
607 version_id: &str,
608 pool_index: usize,
609 set_index: usize,
610 targets: &[String],
611 ) -> Result<bool> {
612 let pool = self
613 .pools
614 .get(pool_index)
615 .ok_or_else(|| invalid_heal_pool_index(pool_index, self.pools.len()))?;
616 let set = pool.disk_set.get(set_index).cloned().ok_or_else(|| {
617 StorageError::InvalidArgument(
618 "heal".to_string(),
619 "set".to_string(),
620 format!("invalid heal set index {set_index} for pool {pool_index}"),
621 )
622 })?;
623
624 set.replacement_targets_have_version(bucket, object, version_id, targets)
625 .await
626 .map_err(Into::into)
627 }
628
629 #[instrument(skip(self))]
630 pub(super) async fn handle_heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
631 let movement_gate = self.ctx.data_movement_operation_gate();
632 let _movement_guard = movement_gate.read().await;
633 let save_guard = self.pool_meta_save_gate.lock().await;
634 save_guard.ensure_write_safe("bucket heal cannot run while pool metadata requires recovery")?;
635 let mut fenced_pools = BTreeSet::new();
636 {
637 let pool_meta = self.pool_meta.read().await;
638 fenced_pools.extend((0..pool_meta.pools.len()).filter(|pool_idx| pool_meta.is_suspended(*pool_idx)));
639 if let Some(pool_idx) = opts.pool {
640 if pool_idx >= pool_meta.pools.len() {
641 return Err(invalid_heal_pool_index(pool_idx, pool_meta.pools.len()));
642 }
643 if pool_meta.is_suspended(pool_idx) {
644 let complete = pool_meta.pools[pool_idx]
645 .decommission
646 .as_ref()
647 .is_some_and(|decommission| decommission.complete);
648 return Err(if complete {
649 StorageError::InvalidArgument(
650 "heal".to_string(),
651 "pool".to_string(),
652 format!("heal pool {pool_idx} has completed decommission"),
653 )
654 } else {
655 Error::SlowDown
656 });
657 }
658 }
659 }
660
661 let dispatch_fenced_pools = fenced_pools.iter().copied().collect::<Vec<_>>();
662 drop(save_guard);
663 let mut res = self
664 .peer_sys
665 .heal_bucket_with_fence_from_movement_guarded_coordinator(bucket, opts, &dispatch_fenced_pools)
666 .await?;
667 {
668 let pool_meta = self.pool_meta.read().await;
669 fenced_pools.extend((0..pool_meta.pools.len()).filter(|pool_idx| pool_meta.is_suspended(*pool_idx)));
670 }
671 if !fenced_pools.is_empty() {
672 let pools = fenced_pools.iter().map(usize::to_string).collect::<Vec<_>>().join(", ");
673 res.detail = format!("skipped: bucket-volume heal fenced on decommission-suspended pool(s): {pools}");
674 }
675
676 Ok(res)
677 }
678
679 #[instrument(level = "trace", skip(self, opts), fields(bucket = %bucket, object = %object, version_id = %version_id))]
680 pub(super) async fn handle_heal_object(
681 &self,
682 bucket: &str,
683 object: &str,
684 version_id: &str,
685 opts: &HealOpts,
686 ) -> Result<(HealResultItem, Option<Error>)> {
687 self.handle_heal_object_with_absence(bucket, object, version_id, opts, &mut None, None)
688 .await
689 }
690
691 pub async fn heal_object_with_proof(
692 &self,
693 bucket: &str,
694 object: &str,
695 version_id: &str,
696 opts: &HealOpts,
697 ) -> Result<HealObjectStorageResult> {
698 if opts.dry_run || opts.no_lock || version_id.is_empty() || super::utils::is_reserved_or_invalid_bucket(bucket, false) {
699 let (item, error) = self.handle_heal_object(bucket, object, version_id, opts).await?;
700 return Ok(HealObjectStorageResult {
701 item,
702 error,
703 absence: None,
704 });
705 }
706
707 let scope = bucket_heal_scope(bucket).filter(|scope| std::ptr::eq(scope.store.as_ref(), self));
712 let guard = if let Some(scope) = &scope {
713 scope.check().map_err(Error::from)?;
714 None
715 } else {
716 Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
717 };
718 let fence_is_lost = || {
719 scope.as_ref().is_some_and(|scope| scope.fence.is_lock_lost())
720 || guard.as_ref().is_some_and(NamespaceLockGuard::is_lock_lost)
721 };
722 let lifecycle_guard = scope
723 .as_ref()
724 .and_then(|scope| scope.fence.namespace_lock_guard())
725 .or(guard.as_ref())
726 .ok_or_else(|| Error::other("bucket heal requires a held lifecycle guard"))?;
727 let retirement = crate::bucket::retirement::MarkerRetirementContext {
728 store: crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await,
729 current_incarnation: if let Some(scope) = &scope {
730 Some(scope.incarnation)
731 } else {
732 self.bucket_incarnation_id_from_disk(bucket)
733 .await
734 .ok()
735 .filter(|id| !id.is_nil())
736 },
737 lifecycle_guard,
738 };
739 let mut proofs = None;
740 let (item, mut error) = self
741 .handle_heal_object_with_absence(bucket, object, version_id, opts, &mut proofs, Some(&retirement))
742 .await?;
743 let incarnation = if proofs.is_some() && !fence_is_lost() {
746 if let Some(scope) = &scope {
747 Some(scope.incarnation)
748 } else {
749 self.bucket_incarnation_id_from_disk(bucket)
750 .await
751 .ok()
752 .filter(|id| !id.is_nil())
753 }
754 } else {
755 None
756 };
757 let absence = match (incarnation, proofs) {
758 (Some(incarnation), Some(proofs)) if !fence_is_lost() => Some(HealObjectAbsenceProof {
759 bucket: bucket.to_owned(),
760 object: object.to_owned(),
761 version_id: version_id.to_owned(),
762 bucket_incarnation_id: incarnation,
763 pool_index: opts.pool,
764 set_index: opts.set,
765 removed: proofs.iter().any(|proof| proof.removed),
766 locations: proofs.into_iter().map(|proof| (proof.pool_index, proof.set_index)).collect(),
767 }),
768 _ => None,
769 };
770 if absence.is_some() {
771 error = None;
772 }
773 Ok(HealObjectStorageResult { item, error, absence })
774 }
775
776 async fn handle_heal_object_with_absence(
777 &self,
778 bucket: &str,
779 object: &str,
780 version_id: &str,
781 opts: &HealOpts,
782 absence: &mut Option<Vec<crate::set_disk::HealedObjectAbsence>>,
783 retirement: Option<&crate::bucket::retirement::MarkerRetirementContext<'_>>,
784 ) -> Result<(HealResultItem, Option<Error>)> {
785 trace!(
786 event = EVENT_HEAL_OBJECT_STARTED,
787 component = LOG_COMPONENT_ECSTORE,
788 subsystem = LOG_SUBSYSTEM_HEAL,
789 bucket = %bucket,
790 object = %object,
791 version_id = %version_id,
792 remove = opts.remove,
793 scan_mode = ?opts.scan_mode,
794 "Heal object started"
795 );
796 let object = encode_dir_object(object);
797
798 *absence = None;
799 let pools = self.get_pools_for_heal_object(opts)?;
800 let requested_pool_count = pools.len();
801 if let Some(set_idx) = opts.set {
802 for pool in &pools {
803 if set_idx >= pool.disk_set.len() {
804 let err = StorageError::InvalidArgument(
805 "heal".to_string(),
806 "set".to_string(),
807 format!(
808 "invalid heal set index {set_idx} for pool {} with {} sets",
809 pool.pool_idx,
810 pool.disk_set.len()
811 ),
812 );
813 if opts.pool.is_some() {
814 return Err(err);
815 }
816 return Ok((HealResultItem::default(), Some(err)));
817 }
818 }
819 }
820 #[cfg(test)]
821 let store_id = self.id;
822
823 let mut heal_pools = Vec::with_capacity(pools.len());
824 for pool in pools.iter() {
825 let suspended_complete = {
826 let pool_meta = self.pool_meta.read().await;
827 pool_meta.is_suspended(pool.pool_idx).then(|| {
828 pool_meta
829 .pools
830 .get(pool.pool_idx)
831 .and_then(|status| status.decommission.as_ref())
832 .is_some_and(|decommission| decommission.complete)
833 })
834 };
835 if let Some(complete) = suspended_complete {
836 if opts.pool.is_some() {
837 let _ = pool.get_disks_for_heal_object(&object, opts)?;
838 let err = if complete {
839 StorageError::InvalidArgument(
840 "heal".to_string(),
841 "pool".to_string(),
842 format!("heal pool {} has completed decommission", pool.pool_idx),
843 )
844 } else {
845 Error::SlowDown
846 };
847 return Ok((HealResultItem::default(), Some(err)));
848 }
849 continue;
850 }
851 heal_pools.push(Arc::clone(pool));
852 }
853 let results = if is_pool_meta_object(bucket, &object) && !opts.no_lock && !heal_pools.is_empty() {
854 let target_pool_indices = heal_pools.iter().map(|pool| pool.pool_idx).collect::<Vec<_>>();
855 match self.acquire_pool_meta_object_heal_fence(&target_pool_indices).await {
856 Ok((pool_meta_guard, admissions)) => {
857 let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first()).cloned();
858 let futures = heal_pools.iter().zip(admissions).map(|(pool, admission)| {
859 let pool = Arc::clone(pool);
860 let pool_object = object.clone();
861 let fixed_set = fixed_set.clone();
862 let mut opts = *opts;
863 async move {
864 admission?;
865 let fixed_set = fixed_set.ok_or_else(|| Error::other("pool metadata heal requires a fixed set"))?;
866 let target_set = pool.get_disks_for_heal_object(&pool_object, &opts)?;
867 opts.no_lock = fixed_set.shares_namespace_lock_domain(&target_set).await;
868 #[cfg(test)]
869 if !opts.no_lock {
870 crate::core::pools::notify_decommission_external_heal_target_lock_attempted();
871 }
872 #[cfg(test)]
873 crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
874 pool.heal_object_with_absence(bucket, &pool_object, version_id, &opts, retirement)
875 .await
876 }
877 });
878 let results = join_all(futures).await;
879 drop(pool_meta_guard);
880 results
881 }
882 Err(err) => (0..heal_pools.len()).map(|_| Err(err.clone())).collect(),
883 }
884 } else {
885 let mut futures = Vec::with_capacity(heal_pools.len());
886 for pool in heal_pools {
887 let pool_idx = pool.pool_idx;
888 let pool_object = object.clone();
889 let opts = *opts;
890 futures.push(self.run_external_decommission_capacity_heal(
891 pool_idx,
892 bucket,
893 &object,
894 opts,
895 move |opts| async move {
896 #[cfg(test)]
897 crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
898 pool.heal_object_with_absence(bucket, &pool_object, version_id, &opts, retirement)
899 .await
900 },
901 ));
902 }
903 join_all(futures).await
904 };
905
906 let mut proofs = Vec::with_capacity(requested_pool_count);
907 let mut errs = Vec::with_capacity(self.pools.len());
908 let mut ress = Vec::with_capacity(self.pools.len());
909
910 for res in results.into_iter() {
911 match res {
912 Ok((result, err, proof)) => {
913 if let Some(proof) = proof
914 && (err.is_none()
915 || err
916 .as_ref()
917 .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)))
918 {
919 proofs.push(proof);
920 }
921 let mut result = result;
922 result.object = decode_dir_object(&result.object);
923 ress.push(result);
924 errs.push(err);
925 }
926 Err(err) => {
927 errs.push(Some(err));
928 ress.push(HealResultItem::default());
929 }
930 }
931 }
932
933 if requested_pool_count > 0 && proofs.len() == requested_pool_count {
936 *absence = Some(proofs);
937 }
938
939 for (idx, err) in errs.iter().enumerate() {
940 if err.is_none() {
941 return Ok((ress.remove(idx), None));
942 }
943 }
944
945 for (index, err) in errs.iter().enumerate() {
947 return match err {
948 Some(err) => {
949 if is_err_object_not_found(err) || is_err_version_not_found(err) {
950 continue;
951 }
952 Ok((ress.remove(index), Some(err.clone())))
953 }
954 None => Ok((ress.remove(index), None)),
955 };
956 }
957
958 if !version_id.is_empty() {
960 return Ok((HealResultItem::default(), Some(Error::FileVersionNotFound)));
961 }
962
963 Ok((HealResultItem::default(), Some(Error::FileNotFound)))
964 }
965
966 #[instrument(skip(self))]
967 pub(super) async fn handle_check_abandoned_parts(&self, bucket: &str, object: &str, opts: &HealOpts) -> Result<()> {
968 let object = encode_dir_object(object);
969 let pools = self.get_pools_for_heal_object(opts)?;
970
971 let mut futures = Vec::with_capacity(pools.len());
972 for pool in pools.iter() {
973 futures.push(pool.check_abandoned_parts(bucket, &object, opts));
974 }
975
976 let mut first_error = None;
977 for result in join_all(futures).await {
978 if let Err(err) = result
979 && first_error.is_none()
980 {
981 first_error = Some(err);
982 }
983 }
984
985 if let Some(err) = first_error {
986 return Err(err);
987 }
988
989 trace!(
990 event = EVENT_HEAL_ABANDONED_PARTS,
991 component = LOG_COMPONENT_ECSTORE,
992 subsystem = LOG_SUBSYSTEM_HEAL,
993 state = "completed",
994 result = "ok",
995 bucket,
996 object,
997 dry_run = opts.dry_run,
998 "Heal abandoned parts completed"
999 );
1000
1001 Ok(())
1002 }
1003}
1004
1005#[cfg(test)]
1006mod tests {
1007 use super::*;
1008 use crate::bucket::metadata_sys;
1009 use crate::cluster::rpc::PeerS3Client;
1010 use crate::config::com::{delete_config, read_config_no_lock_preserve_empty_with_metadata, save_config};
1011 use crate::core::pools::{
1012 DecommissionCapacityAdmission, DecommissionCapacityLockOrderBarrier, DecommissionErasureLayout,
1013 DecommissionPoolCapacityInfo, POOL_META_IDENTITY_NAME, PoolDecommissionInfo, PoolMetaReplicaState, PoolStatus,
1014 initialized_pool_meta_identity_for_test, set_decommission_capacity_info_overrides_for_test,
1015 };
1016 use crate::core::sets::HealFormatAfterSaveBarrier;
1017 use crate::disk::error::Result as DiskResult;
1018 use crate::disk::{DeleteOptions, DiskOption, DiskStore, FORMAT_CONFIG_FILE, format::FormatV3, new_disk};
1019 use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
1020 use crate::runtime::instance::InstanceContext;
1021 use crate::services::rebalance::{
1022 RebalanceInfo, RebalanceStats, test_three_pool_stores_with_isolated_node_contexts, test_two_pool_stores,
1023 };
1024 use crate::storage_api_contracts::bucket::{
1025 BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
1026 };
1027 use crate::storage_api_contracts::object::{ObjectIO as _, ObjectOperations};
1028 use crate::store::init_format::{load_format_erasure, save_format_file};
1029 use crate::store::init_local_disks_with_instance_ctx;
1030 use rustfs_heal_contracts::heal_channel::DriveState;
1031 use tokio_util::sync::CancellationToken;
1032
1033 #[derive(Debug)]
1034 struct BlockingHealPeer {
1035 started: Arc<tokio::sync::Notify>,
1036 release: Arc<tokio::sync::Notify>,
1037 }
1038
1039 #[async_trait::async_trait]
1040 impl PeerS3Client for BlockingHealPeer {
1041 async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> DiskResult<HealResultItem> {
1042 self.started.notify_one();
1043 self.release.notified().await;
1044 Ok(HealResultItem::default())
1045 }
1046
1047 async fn make_bucket(&self, _bucket: &str, _opts: &MakeBucketOptions) -> DiskResult<()> {
1048 Ok(())
1049 }
1050
1051 async fn list_bucket(&self, _opts: &BucketOptions) -> DiskResult<Vec<BucketInfo>> {
1052 Ok(Vec::new())
1053 }
1054
1055 async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> DiskResult<()> {
1056 Ok(())
1057 }
1058
1059 async fn get_bucket_info(&self, _bucket: &str, _opts: &BucketOptions) -> DiskResult<BucketInfo> {
1060 Ok(BucketInfo::default())
1061 }
1062
1063 fn get_pools(&self) -> Option<Vec<usize>> {
1064 Some(vec![0, 1])
1065 }
1066 }
1067
1068 #[derive(Debug)]
1069 struct WriterQueuedLocalHealPeer {
1070 movement_gate: Arc<tokio::sync::RwLock<()>>,
1071 writer_queued: Arc<tokio::sync::Notify>,
1072 writer_acquired: Arc<tokio::sync::Notify>,
1073 }
1074
1075 #[async_trait::async_trait]
1076 impl PeerS3Client for WriterQueuedLocalHealPeer {
1077 async fn heal_bucket(&self, _bucket: &str, _opts: &HealOpts) -> DiskResult<HealResultItem> {
1078 let _movement_guard = self
1079 .movement_gate
1080 .try_read()
1081 .map_err(|_| crate::disk::error::DiskError::TooManyOpenFiles)?;
1082 Ok(HealResultItem::default())
1083 }
1084
1085 async fn heal_bucket_with_fence_from_movement_guarded_coordinator(
1086 &self,
1087 _bucket: &str,
1088 _opts: &HealOpts,
1089 _fenced_pools: &[usize],
1090 ) -> DiskResult<HealResultItem> {
1091 Ok(HealResultItem::default())
1092 }
1093
1094 async fn make_bucket(&self, _bucket: &str, _opts: &MakeBucketOptions) -> DiskResult<()> {
1095 Ok(())
1096 }
1097
1098 async fn list_bucket(&self, _opts: &BucketOptions) -> DiskResult<Vec<BucketInfo>> {
1099 Ok(Vec::new())
1100 }
1101
1102 async fn delete_bucket(&self, _bucket: &str, _opts: &DeleteBucketOptions) -> DiskResult<()> {
1103 Ok(())
1104 }
1105
1106 async fn get_bucket_info(&self, _bucket: &str, _opts: &BucketOptions) -> DiskResult<BucketInfo> {
1107 let movement_gate = self.movement_gate.clone();
1108 let writer_acquired = self.writer_acquired.clone();
1109 tokio::spawn(async move {
1110 let _movement_guard = movement_gate.write().await;
1111 writer_acquired.notify_one();
1112 });
1113 while self.movement_gate.try_read().is_ok() {
1114 tokio::task::yield_now().await;
1115 }
1116 self.writer_queued.notify_one();
1117 Ok(BucketInfo::default())
1118 }
1119
1120 fn get_pools(&self) -> Option<Vec<usize>> {
1121 Some(vec![0, 1])
1122 }
1123 }
1124
1125 async fn minimal_heal_pool(pool_idx: usize) -> Arc<Sets> {
1126 let format = FormatV3::new(1, 1);
1127 let endpoint_url = format!("http://127.0.0.1:{}/data", 19000 + pool_idx);
1128 let mut endpoint = Endpoint::try_from(endpoint_url.as_str()).expect("endpoint should parse");
1129 endpoint.set_pool_index(pool_idx);
1130 endpoint.set_set_index(0);
1131 endpoint.set_disk_index(0);
1132
1133 Sets::new(
1134 vec![None],
1135 &PoolEndpoints {
1136 legacy: false,
1137 set_count: 1,
1138 drives_per_set: 1,
1139 endpoints: Endpoints::from(vec![endpoint]),
1140 cmd_line: String::new(),
1141 platform: String::new(),
1142 },
1143 &format,
1144 pool_idx,
1145 0,
1146 )
1147 .await
1148 .expect("minimal pool should build")
1149 }
1150
1151 #[tokio::test]
1152 #[serial_test::serial]
1153 async fn bucket_incarnation_heal_preserves_successor_shards_and_allows_fresh_repair() {
1154 let (_root, store, shutdown) = multi_pool_heal_store().await;
1155 let bucket = format!("heal-incarnation-{}", Uuid::new_v4().simple());
1156 store
1157 .make_bucket(&bucket, &MakeBucketOptions::default())
1158 .await
1159 .expect("original bucket");
1160 let old = store
1161 .bucket_incarnation_id_from_disk(&bucket)
1162 .await
1163 .expect("original identity");
1164 store
1165 .delete_bucket(&bucket, &DeleteBucketOptions::default())
1166 .await
1167 .expect("normal bucket deletion");
1168 store
1169 .make_bucket(&bucket, &MakeBucketOptions::default())
1170 .await
1171 .expect("same-name successor");
1172 let new = store
1173 .bucket_incarnation_id_from_disk(&bucket)
1174 .await
1175 .expect("successor identity");
1176 assert_ne!(old, new);
1177 let object = "successor-version";
1178 let version = Uuid::new_v4().to_string();
1179 let set = store.pools[0].get_disks(0);
1180 let mut reader = PutObjReader::from_vec(vec![17; 512 * 1024]);
1181 set.put_object(
1182 &bucket,
1183 object,
1184 &mut reader,
1185 &ObjectOptions {
1186 versioned: true,
1187 version_id: Some(version.clone()),
1188 write_completion: crate::object_api::WriteCompletion::TailDrained,
1190 ..Default::default()
1191 },
1192 )
1193 .await
1194 .expect("write successor version");
1195 let missing = set.disks.read().await[0].clone().expect("missing target");
1196 let healthy = set.disks.read().await[1].clone().expect("healthy target");
1197 missing
1198 .delete(
1199 &bucket,
1200 object,
1201 DeleteOptions {
1202 recursive: true,
1203 immediate: true,
1204 ..Default::default()
1205 },
1206 )
1207 .await
1208 .expect("remove one successor shard");
1209 let baseline = healthy
1210 .read_all(&bucket, &format!("{object}/xl.meta"))
1211 .await
1212 .expect("healthy baseline");
1213 let opts = HealOpts {
1214 pool: Some(0),
1215 set: Some(0),
1216 scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
1217 ..Default::default()
1218 };
1219 let (_, error) = store
1221 .heal_object(&bucket, object, &version, &opts)
1222 .await
1223 .expect("unbound control");
1224 assert!(error.is_none());
1225 assert!(missing.read_xl(&bucket, object, false).await.is_ok());
1226 missing
1227 .delete(
1228 &bucket,
1229 object,
1230 DeleteOptions {
1231 recursive: true,
1232 immediate: true,
1233 ..Default::default()
1234 },
1235 )
1236 .await
1237 .expect("restore the same missing-shard condition");
1238 let fi = healthy
1239 .read_version("", &bucket, object, &version, &Default::default())
1240 .await
1241 .expect("successor metadata");
1242 let disk_ref = healthy.endpoint().to_string();
1243 assert!(
1244 store
1245 .write_local_metadata_at_incarnation(&disk_ref, (&bucket, object), fi.clone(), old)
1246 .await
1247 .is_err()
1248 );
1249 assert!(
1250 store
1251 .delete_local_path_at_incarnation(
1252 &disk_ref,
1253 (&bucket, object),
1254 DeleteOptions {
1255 recursive: true,
1256 immediate: true,
1257 ..Default::default()
1258 },
1259 old
1260 )
1261 .await
1262 .is_err()
1263 );
1264 assert!(
1265 store
1266 .delete_local_version_at_incarnation(
1267 &disk_ref,
1268 (&bucket, object),
1269 fi.clone(),
1270 false,
1271 DeleteOptions::default(),
1272 old
1273 )
1274 .await
1275 .is_err()
1276 );
1277 assert!(
1278 store
1279 .rename_local_data_at_incarnation(&disk_ref, (&bucket, object), &fi, (&bucket, "stale-destination"), old)
1280 .await
1281 .is_err()
1282 );
1283 assert!(store.heal_bucket_at_incarnation(&bucket, old, &opts).await.is_err());
1284 assert!(
1285 store
1286 .heal_object_at_incarnation(&bucket, object, &version, old, &opts)
1287 .await
1288 .is_err()
1289 );
1290 assert!(
1291 missing.read_xl(&bucket, object, false).await.is_err(),
1292 "obsolete admission must not publish a shard"
1293 );
1294 assert_eq!(
1295 healthy
1296 .read_all(&bucket, &format!("{object}/xl.meta"))
1297 .await
1298 .expect("healthy after stale heal"),
1299 baseline
1300 );
1301 store
1302 .heal_bucket_at_incarnation(&bucket, new, &opts)
1303 .await
1304 .expect("fresh bucket metadata repair");
1305 let result = store
1306 .heal_object_at_incarnation(&bucket, object, &version, new, &opts)
1307 .await
1308 .expect("new admission repair");
1309 assert!(result.error.is_none(), "new admission must repair independently: {:?}", result.error);
1310 assert!(
1311 missing.read_xl(&bucket, object, false).await.is_ok(),
1312 "new admission publishes the missing shard"
1313 );
1314 store
1315 .write_local_metadata_at_incarnation(&disk_ref, (&bucket, object), fi.clone(), new)
1316 .await
1317 .expect("fresh target metadata");
1318 store
1319 .delete_local_version_at_incarnation(&disk_ref, (&bucket, object), fi, false, DeleteOptions::default(), new)
1320 .await
1321 .expect("fresh target version cleanup");
1322 healthy
1323 .write_all(&bucket, "fresh-cleanup/data", bytes::Bytes::from_static(b"temporary"))
1324 .await
1325 .expect("cleanup fixture");
1326 store
1327 .delete_local_path_at_incarnation(
1328 &disk_ref,
1329 (&bucket, "fresh-cleanup"),
1330 DeleteOptions {
1331 recursive: true,
1332 immediate: true,
1333 ..Default::default()
1334 },
1335 new,
1336 )
1337 .await
1338 .expect("fresh target path cleanup");
1339 shutdown.cancel();
1340 }
1341
1342 #[tokio::test]
1343 #[serial_test::serial]
1344 async fn bucket_incarnation_heal_owner_survives_cancelled_waiter_and_queued_writer() {
1345 let (_root, store, shutdown) = multi_pool_heal_store().await;
1346 let bucket = format!("heal-owner-{}", Uuid::new_v4().simple());
1347 store
1348 .make_bucket(&bucket, &MakeBucketOptions::default())
1349 .await
1350 .expect("bucket");
1351 let identity = store.bucket_incarnation_id_from_disk(&bucket).await.expect("identity");
1352 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
1353 let (release_tx, release_rx) = tokio::sync::oneshot::channel();
1354 let (drained_tx, drained_rx) = tokio::sync::oneshot::channel();
1355 let worker_store = store.clone();
1356 let worker_bucket = bucket.clone();
1357 let waiter = tokio::spawn(async move {
1358 worker_store
1359 .run_bucket_heal_at_incarnation(
1360 &worker_bucket,
1361 identity,
1362 &HealOpts::default(),
1363 move |store, bucket, opts| async move {
1364 started_tx.send(()).expect("announce held generation");
1365 release_rx.await.expect("release commit");
1366 let result = store
1367 .heal_object_with_proof(&bucket, "history.txt", &Uuid::new_v4().to_string(), &opts)
1368 .await?;
1369 let proof = result.absence.expect("absence proof must reuse the held lifecycle owner");
1370 assert_eq!(proof.bucket_incarnation_id, identity);
1371 assert_eq!(proof.locations.len(), 2);
1372 drained_tx.send(()).expect("announce drain");
1373 Ok(())
1374 },
1375 )
1376 .await
1377 });
1378 started_rx.await.expect("operation acquired generation fence");
1379 waiter.abort();
1380 assert!(waiter.await.expect_err("waiter aborted").is_cancelled());
1381 let writer_store = store.clone();
1382 let writer_bucket = bucket.clone();
1383 let mut writer = Box::pin(writer_store.acquire_bucket_lifecycle_write_lock(&writer_bucket));
1384 assert!(
1385 futures::poll!(&mut writer).is_pending(),
1386 "cancellation must not release the live operation's generation"
1387 );
1388 release_tx.send(()).expect("finish operation");
1389 tokio::time::timeout(std::time::Duration::from_secs(10), drained_rx)
1390 .await
1391 .expect("proof must not reacquire a lifecycle read lock behind the queued writer")
1392 .expect("physical operation drains");
1393 let guard = tokio::time::timeout(std::time::Duration::from_secs(10), writer)
1394 .await
1395 .expect("writer resumes after drain")
1396 .expect("lifecycle writer");
1397 drop(guard);
1398 shutdown.cancel();
1399 }
1400
1401 async fn minimal_heal_store() -> ECStore {
1402 ECStore {
1403 id: Uuid::new_v4(),
1404 disk_map: HashMap::new(),
1405 pools: vec![minimal_heal_pool(0).await, minimal_heal_pool(1).await],
1406 peer_sys: S3PeerSys {
1407 clients: Vec::new(),
1408 pools_count: 2,
1409 },
1410 pool_meta: RwLock::new(PoolMeta::default()),
1411 rebalance_meta: RwLock::new(None),
1412 decommission_cancelers: RwLock::new(Vec::new()),
1413 start_gate: Mutex::new(()),
1414 pool_meta_save_gate: Mutex::default(),
1415 ctx: crate::runtime::instance::bootstrap_ctx(),
1416 bucket_fence_registry: std::sync::Arc::default(),
1417 }
1418 }
1419
1420 async fn remove_pool_meta_shard(store: &ECStore, pool_idx: usize) -> DiskStore {
1421 let target_set = store.pools[pool_idx].get_disks_by_key(POOL_META_NAME);
1422 let missing_disk = target_set.disks.read().await[0]
1423 .clone()
1424 .expect("pool metadata fixture disk should be online");
1425 missing_disk
1426 .delete(
1427 RUSTFS_META_BUCKET,
1428 POOL_META_NAME,
1429 DeleteOptions {
1430 recursive: true,
1431 immediate: true,
1432 ..Default::default()
1433 },
1434 )
1435 .await
1436 .expect("one pool metadata shard should be removable");
1437 assert!(
1438 missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_err(),
1439 "pool metadata fixture must start with one missing shard"
1440 );
1441 missing_disk
1442 }
1443
1444 #[tokio::test]
1445 async fn heal_erasure_set_scopes_follow_requested_pool_and_set() {
1446 let store = minimal_heal_store().await;
1447
1448 assert_eq!(
1449 store
1450 .heal_erasure_set_scopes(&HealOpts::default())
1451 .await
1452 .expect("unscoped heal should enumerate every live set"),
1453 vec![(0, 0), (1, 0)]
1454 );
1455 assert_eq!(
1456 store
1457 .heal_erasure_set_scopes(&HealOpts {
1458 pool: Some(1),
1459 set: Some(0),
1460 ..Default::default()
1461 })
1462 .await
1463 .expect("scoped heal should enumerate only its requested set"),
1464 vec![(1, 0)]
1465 );
1466
1467 let err = store
1468 .heal_erasure_set_scopes(&HealOpts {
1469 pool: Some(0),
1470 set: Some(1),
1471 ..Default::default()
1472 })
1473 .await
1474 .expect_err("an invalid set scope must fail closed");
1475 assert!(matches!(err, Error::InvalidArgument(..)));
1476 }
1477
1478 fn pool_meta_with_decommission(info: PoolDecommissionInfo) -> PoolMeta {
1479 PoolMeta {
1480 pools: vec![PoolStatus {
1481 id: 0,
1482 cmd_line: "pool-0".to_string(),
1483 last_update: OffsetDateTime::UNIX_EPOCH,
1484 decommission: Some(info),
1485 }],
1486 ..Default::default()
1487 }
1488 }
1489
1490 #[test]
1491 fn heal_format_pool_state_barriers_are_classified() {
1492 let active = pool_meta_with_decommission(PoolDecommissionInfo {
1493 start_time: Some(OffsetDateTime::UNIX_EPOCH),
1494 ..Default::default()
1495 });
1496 assert!(matches!(
1497 classify_heal_format_pool(0, "pool-0", &active, None),
1498 Some(HealFormatPoolSkip::Retryable)
1499 ));
1500
1501 for info in [
1502 PoolDecommissionInfo {
1503 failed: true,
1504 ..Default::default()
1505 },
1506 PoolDecommissionInfo {
1507 canceled: true,
1508 ..Default::default()
1509 },
1510 ] {
1511 assert!(matches!(
1512 classify_heal_format_pool(0, "pool-0", &pool_meta_with_decommission(info), None),
1513 Some(HealFormatPoolSkip::Retryable)
1514 ));
1515 }
1516
1517 let completed = pool_meta_with_decommission(PoolDecommissionInfo {
1518 complete: true,
1519 ..Default::default()
1520 });
1521 assert!(matches!(
1522 classify_heal_format_pool(0, "pool-0", &completed, None),
1523 Some(HealFormatPoolSkip::Completed)
1524 ));
1525 }
1526
1527 #[test]
1528 fn heal_format_pool_rebalance_barriers_and_identity_are_fail_closed() {
1529 let identity_meta = pool_meta_with_decommission(PoolDecommissionInfo::default());
1530 let rebalance = RebalanceMeta {
1531 pool_stats: vec![RebalanceStats {
1532 participating: true,
1533 info: RebalanceInfo {
1534 status: RebalStatus::Started,
1535 ..Default::default()
1536 },
1537 ..Default::default()
1538 }],
1539 ..Default::default()
1540 };
1541 assert!(matches!(
1542 classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&rebalance)),
1543 Some(HealFormatPoolSkip::Retryable)
1544 ));
1545
1546 let stopping = RebalanceMeta {
1547 pool_stats: vec![RebalanceStats {
1548 info: RebalanceInfo {
1549 stopping: true,
1550 ..Default::default()
1551 },
1552 ..Default::default()
1553 }],
1554 ..Default::default()
1555 };
1556 assert!(matches!(
1557 classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping)),
1558 Some(HealFormatPoolSkip::Retryable)
1559 ));
1560
1561 let identity = pool_meta_with_decommission(PoolDecommissionInfo::default());
1562 assert!(matches!(
1563 classify_heal_format_pool(0, "pool-new", &identity, None),
1564 Some(HealFormatPoolSkip::Retryable)
1565 ));
1566
1567 let identity_without_decommission = PoolMeta {
1568 pools: vec![PoolStatus {
1569 id: 0,
1570 cmd_line: "pool-0".to_string(),
1571 last_update: OffsetDateTime::UNIX_EPOCH,
1572 decommission: None,
1573 }],
1574 ..Default::default()
1575 };
1576 assert!(matches!(
1577 classify_heal_format_pool(0, "pool-new", &identity_without_decommission, None),
1578 Some(HealFormatPoolSkip::Retryable)
1579 ));
1580
1581 assert!(matches!(
1582 classify_heal_format_pool(0, "", &identity_meta, None),
1583 Some(HealFormatPoolSkip::Retryable)
1584 ));
1585
1586 assert!(matches!(
1587 classify_heal_format_pool(0, "pool-0", &PoolMeta::default(), None),
1588 Some(HealFormatPoolSkip::Retryable)
1589 ));
1590
1591 let stopped = RebalanceMeta {
1592 stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
1593 pool_stats: vec![RebalanceStats {
1594 participating: true,
1595 info: RebalanceInfo {
1596 status: RebalStatus::Stopped,
1597 ..Default::default()
1598 },
1599 ..Default::default()
1600 }],
1601 ..Default::default()
1602 };
1603 assert!(classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopped)).is_none());
1604
1605 let stopping_after_stop = RebalanceMeta {
1606 stopped_at: Some(OffsetDateTime::UNIX_EPOCH),
1607 pool_stats: vec![RebalanceStats {
1608 participating: true,
1609 info: RebalanceInfo {
1610 status: RebalStatus::Started,
1611 stopping: true,
1612 ..Default::default()
1613 },
1614 ..Default::default()
1615 }],
1616 ..Default::default()
1617 };
1618 assert!(matches!(
1619 classify_heal_format_pool(0, "pool-0", &identity_meta, Some(&stopping_after_stop)),
1620 Some(HealFormatPoolSkip::Retryable)
1621 ));
1622 }
1623
1624 #[test]
1625 fn skipped_heal_format_pool_is_never_reported_as_success() {
1626 assert!(matches!(
1627 heal_format_pool_skip_error(HealFormatPoolSkip::Retryable),
1628 StorageError::SlowDown
1629 ));
1630 assert!(matches!(
1631 heal_format_pool_skip_error(HealFormatPoolSkip::Completed),
1632 StorageError::NoHealRequired
1633 ));
1634 }
1635
1636 async fn multi_pool_heal_store() -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
1637 let temp_dir = tempfile::tempdir().expect("multi-pool heal test directory should be created");
1638 let mut pool_endpoints = Vec::new();
1639 for pool_index in 0..2 {
1640 let mut endpoints = Vec::new();
1641 for disk_index in 0..4 {
1642 let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
1643 tokio::fs::create_dir_all(&disk_path)
1644 .await
1645 .expect("multi-pool heal test disk should be created");
1646 let mut endpoint = Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8"))
1647 .expect("test endpoint should parse");
1648 endpoint.set_pool_index(pool_index);
1649 endpoint.set_set_index(0);
1650 endpoint.set_disk_index(disk_index);
1651 endpoints.push(endpoint);
1652 }
1653 pool_endpoints.push(PoolEndpoints {
1654 legacy: false,
1655 set_count: 1,
1656 drives_per_set: 4,
1657 endpoints: Endpoints::from(endpoints),
1658 cmd_line: format!("heal-owner-pool-{pool_index}"),
1659 platform: "test".to_string(),
1660 });
1661 }
1662
1663 let endpoint_pools = EndpointServerPools::from(pool_endpoints);
1664 let instance_ctx = Arc::new(InstanceContext::new());
1665 init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
1666 .await
1667 .expect("multi-pool local disks should initialize");
1668 let shutdown = CancellationToken::new();
1669 let store = ECStore::new_with_instance_ctx(
1670 "127.0.0.1:0".parse().expect("test address should parse"),
1671 endpoint_pools,
1672 shutdown.clone(),
1673 instance_ctx,
1674 )
1675 .await
1676 .expect("multi-pool test store should initialize");
1677 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
1678 (temp_dir, store, shutdown)
1679 }
1680
1681 #[tokio::test]
1682 #[serial_test::serial]
1683 async fn absence_proof_requires_every_selected_pool() {
1684 let (_temp_dir, store, shutdown) = multi_pool_heal_store().await;
1685 let bucket = format!("absence-scope-{}", Uuid::new_v4().simple());
1686 store
1687 .make_bucket(&bucket, &MakeBucketOptions::default())
1688 .await
1689 .expect("create bucket in both pools");
1690 let version = Uuid::new_v4().to_string();
1691 let result = store
1692 .heal_object_with_proof(&bucket, "history.txt", &version, &HealOpts::default())
1693 .await
1694 .expect("exact absence lookup should complete across both pools");
1695 assert!(result.error.is_none());
1696 let proof = result.absence.expect("all selected pools proved the exact version absent");
1697 assert_eq!(proof.locations, vec![(0, 0), (1, 0)]);
1698 assert_eq!((proof.pool_index, proof.set_index), (None, None));
1699 assert_eq!(proof.bucket, bucket);
1700 assert_eq!(proof.version_id, version);
1701 assert!(!proof.removed, "already absent versions do not count as another cleanup");
1702
1703 store.pool_meta.write().await.pools[1].decommission = Some(PoolDecommissionInfo {
1704 start_time: Some(OffsetDateTime::now_utc()),
1705 ..Default::default()
1706 });
1707 let partial = store
1708 .heal_object_with_proof(&bucket, "history.txt", &version, &HealOpts::default())
1709 .await
1710 .expect("unscoped heal should retain its legacy suspended-pool behavior");
1711 assert!(partial.absence.is_none(), "an uninspected suspended pool prevents global absence proof");
1712 shutdown.cancel();
1713 }
1714
1715 fn heal_test_format_path(temp_dir: &tempfile::TempDir, pool_index: usize, disk_index: usize) -> std::path::PathBuf {
1716 temp_dir
1717 .path()
1718 .join(format!("pool{pool_index}-disk{disk_index}"))
1719 .join(crate::disk::RUSTFS_META_BUCKET)
1720 .join(FORMAT_CONFIG_FILE)
1721 }
1722
1723 async fn remove_heal_test_format(
1724 temp_dir: &tempfile::TempDir,
1725 store: &ECStore,
1726 pool_index: usize,
1727 disk_index: usize,
1728 ) -> String {
1729 let endpoint = store.pools[pool_index].endpoints.endpoints.as_ref()[disk_index].clone();
1730 let target = endpoint.to_string();
1731 let format_path = heal_test_format_path(temp_dir, pool_index, disk_index);
1732 tokio::fs::remove_file(&format_path)
1733 .await
1734 .expect("replacement target format should be removable");
1735 assert!(
1736 !tokio::fs::try_exists(&format_path)
1737 .await
1738 .expect("replacement target format path should be inspectable")
1739 );
1740 let replacement = new_disk(
1741 &endpoint,
1742 &DiskOption {
1743 cleanup: false,
1744 health_check: false,
1745 },
1746 )
1747 .await
1748 .expect("unformatted replacement disk should open");
1749 let set = &store.pools[pool_index].disk_set[0];
1750 let target_slot = set
1751 .set_endpoints
1752 .iter()
1753 .position(|candidate| candidate == &endpoint)
1754 .expect("replacement endpoint should belong to its set");
1755 set.disks.write().await[target_slot] = Some(replacement);
1756 target
1757 }
1758
1759 async fn assert_heal_test_format_missing(temp_dir: &tempfile::TempDir, pool_index: usize, disk_index: usize) {
1760 assert!(
1761 !tokio::fs::try_exists(heal_test_format_path(temp_dir, pool_index, disk_index))
1762 .await
1763 .expect("replacement target format path should be inspectable"),
1764 "format heal must not write the replacement target"
1765 );
1766 }
1767
1768 #[tokio::test]
1769 #[serial_test::serial]
1770 async fn replacement_format_heal_coexists_with_capacity_read_fence() {
1771 let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
1772 let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
1773 let capacity_guard = store
1774 .acquire_external_decommission_capacity_fence(&[0], DecommissionCapacityAdmission::Heal)
1775 .await
1776 .expect("ordinary heal capacity fence should be acquired");
1777
1778 let (_, err) =
1779 tokio::time::timeout(std::time::Duration::from_secs(2), store.heal_replacement_format(false, 0, 0, &[target]))
1780 .await
1781 .expect("replacement format heal must not wait on an existing capacity read fence")
1782 .expect("replacement format heal should complete");
1783
1784 assert!(err.is_none(), "replacement format heal should succeed: {err:?}");
1785 assert!(
1786 tokio::fs::try_exists(heal_test_format_path(&temp_dir, 0, 3))
1787 .await
1788 .expect("replacement target format path should be inspectable"),
1789 "replacement format heal should restore the target while the capacity read fence is held"
1790 );
1791 drop(capacity_guard);
1792 shutdown.cancel();
1793 }
1794
1795 #[tokio::test]
1796 #[serial_test::serial]
1797 async fn replacement_format_heal_remains_blocked_by_pool_metadata_writer() {
1798 let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
1799 let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
1800 let pool_lock = store.pools[0]
1801 .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
1802 .await
1803 .expect("pool metadata lock should be created");
1804 let pool_writer = pool_lock
1805 .get_write_lock(get_lock_acquire_timeout())
1806 .await
1807 .expect("pool metadata writer should be acquired");
1808
1809 assert!(
1810 tokio::time::timeout(
1811 std::time::Duration::from_secs(1),
1812 store.heal_replacement_format(false, 0, 0, std::slice::from_ref(&target)),
1813 )
1814 .await
1815 .is_err(),
1816 "replacement format heal must wait for a pool metadata writer"
1817 );
1818 assert_heal_test_format_missing(&temp_dir, 0, 3).await;
1819
1820 drop(pool_writer);
1821 let (_, err) =
1822 tokio::time::timeout(std::time::Duration::from_secs(30), store.heal_replacement_format(false, 0, 0, &[target]))
1823 .await
1824 .expect("replacement format heal should resume after the writer releases")
1825 .expect("replacement format heal should complete after the writer releases");
1826 assert!(err.is_none(), "replacement format heal should succeed: {err:?}");
1827 assert!(
1828 tokio::fs::try_exists(heal_test_format_path(&temp_dir, 0, 3))
1829 .await
1830 .expect("replacement target format path should be inspectable"),
1831 "replacement format heal should restore the target after the writer releases"
1832 );
1833 shutdown.cancel();
1834 }
1835
1836 #[tokio::test]
1837 #[serial_test::serial]
1838 async fn full_format_heal_preblocked_pool_metadata_never_writes_format() {
1839 let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
1840 remove_heal_test_format(&temp_dir, &store, 0, 3).await;
1841 store.pool_meta_save_gate.lock().await.observe_replicas(PoolMetaReplicaState {
1842 needs_repair: true,
1843 repair_write_safe: false,
1844 });
1845
1846 let err = store
1847 .handle_heal_format(false)
1848 .await
1849 .expect_err("a preblocked pool metadata state must reject full format heal");
1850 assert!(
1851 err.to_string()
1852 .contains("restart after all replicas are readable and consistent")
1853 );
1854 assert_heal_test_format_missing(&temp_dir, 0, 3).await;
1855 store
1856 .ensure_pool_meta_side_effects_safe("preblocked format heal side effect")
1857 .await
1858 .expect_err("the preblocked state must remain sticky");
1859 shutdown.cancel();
1860 }
1861
1862 #[tokio::test]
1863 #[serial_test::serial]
1864 async fn full_format_heal_future_identity_never_writes_format_and_latches() {
1865 let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
1866 remove_heal_test_format(&temp_dir, &store, 0, 3).await;
1867 let (mut future_identity, _) =
1868 read_config_no_lock_preserve_empty_with_metadata(store.pools[1].clone(), POOL_META_IDENTITY_NAME)
1869 .await
1870 .expect("current identity should be readable");
1871 let future_version = u16::from_le_bytes([future_identity[2], future_identity[3]])
1872 .checked_add(1)
1873 .expect("identity version should have a future value");
1874 future_identity[2..4].copy_from_slice(&future_version.to_le_bytes());
1875 save_config(store.pools[1].clone(), POOL_META_IDENTITY_NAME, future_identity)
1876 .await
1877 .expect("future identity should be persisted");
1878
1879 let err = store
1880 .handle_heal_format(false)
1881 .await
1882 .expect_err("a future identity must reject full format heal");
1883 assert!(err.to_string().contains("pool metadata incompatible"));
1884 assert_heal_test_format_missing(&temp_dir, 0, 3).await;
1885 store
1886 .ensure_pool_meta_side_effects_safe("future identity format heal side effect")
1887 .await
1888 .expect_err("the future identity rejection must latch the write gate");
1889 shutdown.cancel();
1890 }
1891
1892 #[tokio::test]
1893 #[serial_test::serial]
1894 async fn replacement_format_heal_epoch_conflict_never_writes_format_and_latches() {
1895 let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
1896 let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
1897 let conflicting_identity =
1898 initialized_pool_meta_identity_for_test(store.id, 2).expect("conflicting identity should encode");
1899 save_config(store.pools[1].clone(), POOL_META_IDENTITY_NAME, conflicting_identity)
1900 .await
1901 .expect("conflicting identity should be persisted");
1902
1903 let err = store
1904 .heal_replacement_format(false, 0, 0, &[target])
1905 .await
1906 .expect_err("an identity epoch conflict must reject replacement format heal");
1907 assert!(
1908 err.to_string()
1909 .contains("identity replicas disagree on cluster identity or epoch")
1910 );
1911 assert_heal_test_format_missing(&temp_dir, 0, 3).await;
1912 store
1913 .ensure_pool_meta_side_effects_safe("epoch conflict replacement format side effect")
1914 .await
1915 .expect_err("the identity epoch conflict must latch the write gate");
1916 shutdown.cancel();
1917 }
1918
1919 #[tokio::test]
1920 #[serial_test::serial]
1921 async fn replacement_format_heal_initialized_identity_without_pool_meta_never_writes_and_latches() {
1922 let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
1923 let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
1924 for pool in &store.pools {
1925 delete_config(pool.clone(), POOL_META_NAME)
1926 .await
1927 .expect("pool metadata replica should be removable");
1928 }
1929
1930 let err = store
1931 .heal_replacement_format(false, 0, 0, &[target])
1932 .await
1933 .expect_err("initialized identity without pool metadata must reject replacement format heal");
1934 assert!(err.to_string().contains("initialized cluster identity exists"));
1935 assert_heal_test_format_missing(&temp_dir, 0, 3).await;
1936 store
1937 .ensure_pool_meta_side_effects_safe("missing pool metadata replacement format side effect")
1938 .await
1939 .expect_err("missing initialized pool metadata must latch the write gate");
1940 shutdown.cancel();
1941 }
1942
1943 #[tokio::test]
1944 #[serial_test::serial]
1945 async fn full_format_heal_lost_after_last_save_does_not_publish_or_renew_and_latches() {
1946 let (temp_dir, store, shutdown) = multi_pool_heal_store().await;
1947 let target = remove_heal_test_format(&temp_dir, &store, 0, 3).await;
1948 store.pools[0].disk_set[0].disks.write().await[3] = None;
1949 let barrier = HealFormatAfterSaveBarrier::install(&store.pools[0], 3);
1950 let recovery_latch = store.pool_meta_save_gate.lock().await.aborted_transaction_latch_for_test();
1951 let mut heal = tokio::spawn({
1952 let store = Arc::clone(&store);
1953 async move { store.handle_heal_format(false).await }
1954 });
1955
1956 barrier.wait_until_paused().await;
1957 let saved = tokio::fs::read(heal_test_format_path(&temp_dir, 0, 3))
1958 .await
1959 .expect("the last replacement format must be durable before fence loss");
1960 let saved = FormatV3::try_from(saved.as_slice()).expect("the durable replacement format should decode");
1961 assert_eq!(saved.erasure.this, store.pools[0].format.erasure.sets[0][3]);
1962 recovery_latch.store(true, std::sync::atomic::Ordering::SeqCst);
1963 barrier.release();
1964
1965 let (result, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
1966 .await
1967 .expect("format heal should stop after the lost fence")
1968 .expect("format heal task should not panic")
1969 .expect("format heal should return its fenced result");
1970 assert!(matches!(err, Some(StorageError::SlowDown)));
1971 assert!(
1972 result
1973 .after
1974 .drives
1975 .iter()
1976 .any(|drive| drive.endpoint == target && drive.state == DriveState::Missing.to_string()),
1977 "the lost fence must prevent the durable format from being published in the heal result"
1978 );
1979 assert!(
1980 store.pools[0].disk_set[0].disks.read().await[3].is_none(),
1981 "the lost fence must prevent renew_disk from attaching the replacement"
1982 );
1983 recovery_latch.store(false, std::sync::atomic::Ordering::SeqCst);
1984 store
1985 .ensure_pool_meta_side_effects_safe("post-save format fence loss side effect")
1986 .await
1987 .expect_err("post-save format fence loss must remain sticky");
1988 shutdown.cancel();
1989 }
1990
1991 #[tokio::test]
1992 async fn heal_object_pool_scope_selects_only_requested_pool() {
1993 let store = minimal_heal_store().await;
1994 let pools = store
1995 .get_pools_for_heal_object(&HealOpts {
1996 pool: Some(1),
1997 ..Default::default()
1998 })
1999 .expect("requested pool should be selected");
2000
2001 assert_eq!(pools.len(), 1);
2002 assert!(Arc::ptr_eq(&pools[0], &store.pools[1]));
2003 }
2004
2005 #[tokio::test]
2006 async fn heal_object_pool_scope_rejects_invalid_pool() {
2007 let store = minimal_heal_store().await;
2008 let err = store
2009 .get_pools_for_heal_object(&HealOpts {
2010 pool: Some(2),
2011 ..Default::default()
2012 })
2013 .expect_err("out-of-range pool scope must fail closed");
2014
2015 assert!(
2016 matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
2017 if field == "pool" && reason.contains("invalid heal pool index 2 for 2 pools")),
2018 "unexpected invalid pool error: {err:?}"
2019 );
2020 }
2021
2022 #[tokio::test]
2023 #[serial_test::serial]
2024 async fn pool_meta_read_repair_reuses_its_write_fence() {
2025 let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
2026 let missing_disk = remove_pool_meta_shard(&store, 0).await;
2027
2028 let (result, err) = tokio::time::timeout(
2029 std::time::Duration::from_secs(30),
2030 store.handle_heal_object(
2031 RUSTFS_META_BUCKET,
2032 POOL_META_NAME,
2033 "",
2034 &HealOpts {
2035 read_repair: true,
2036 pool: Some(0),
2037 ..Default::default()
2038 },
2039 ),
2040 )
2041 .await
2042 .expect("pool metadata read repair must not wait on its own capacity fence")
2043 .expect("pool metadata read repair should complete");
2044
2045 assert!(err.is_none(), "pool metadata read repair should succeed: {err:?}");
2046 assert_eq!(result.object, POOL_META_NAME);
2047 assert!(
2048 missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
2049 "pool metadata read repair should restore the missing shard"
2050 );
2051 }
2052
2053 #[tokio::test]
2054 #[serial_test::serial]
2055 async fn pool_meta_heal_preserves_typed_lock_timeout() {
2056 let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
2057 let lock = store.pools[0]
2058 .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
2059 .await
2060 .expect("pool metadata lock should be created");
2061 let guard = lock
2062 .get_write_lock(get_lock_acquire_timeout())
2063 .await
2064 .expect("pool metadata lock should be acquired");
2065
2066 let (_, err) = temp_env::async_with_vars(
2067 [(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))],
2068 store.handle_heal_object(
2069 RUSTFS_META_BUCKET,
2070 POOL_META_NAME,
2071 "",
2072 &HealOpts {
2073 pool: Some(0),
2074 ..Default::default()
2075 },
2076 ),
2077 )
2078 .await
2079 .expect("pool metadata lock timeout should be mapped into the heal result");
2080
2081 assert!(
2082 matches!(err, Some(Error::Lock(rustfs_lock::LockError::Timeout { .. }))),
2083 "pool metadata heal must preserve the recoverable lock timeout: {err:?}"
2084 );
2085 drop(guard);
2086 }
2087
2088 #[tokio::test]
2089 #[serial_test::serial]
2090 async fn pool_meta_neighbor_keeps_ordinary_object_locking() {
2091 let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
2092 let object = "pool.bin.backup";
2093 save_config(store.pools[0].clone(), object, b"neighbor metadata".to_vec())
2094 .await
2095 .expect("neighbor metadata fixture should be written");
2096
2097 let target_set = store.pools[0].get_disks_by_key(object);
2098 let lock = target_set
2099 .new_ns_lock(RUSTFS_META_BUCKET, object)
2100 .await
2101 .expect("neighbor metadata lock should be created");
2102 let guard = lock
2103 .get_write_lock(get_lock_acquire_timeout())
2104 .await
2105 .expect("neighbor metadata lock should be acquired");
2106 let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
2107 let heal_store = Arc::clone(&store);
2108 let mut heal = tokio::spawn(async move {
2109 heal_store
2110 .handle_heal_object(
2111 RUSTFS_META_BUCKET,
2112 object,
2113 "",
2114 &HealOpts {
2115 pool: Some(0),
2116 ..Default::default()
2117 },
2118 )
2119 .await
2120 });
2121
2122 barrier.wait_until_external_heal_target_lock_attempted().await;
2123 tokio::task::yield_now().await;
2124 assert!(!heal.is_finished(), "neighbor metadata heal must retain ordinary object locking");
2125 drop(guard);
2126
2127 let (result, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
2128 .await
2129 .expect("neighbor metadata heal should finish after the object lock is released")
2130 .expect("neighbor metadata heal task should not panic")
2131 .expect("neighbor metadata heal should complete");
2132 assert!(err.is_none(), "neighbor metadata heal should succeed: {err:?}");
2133 assert_eq!(result.object, object);
2134 drop(barrier);
2135 }
2136
2137 #[tokio::test]
2138 #[serial_test::serial]
2139 async fn targeted_heal_is_blocked_by_exact_fit_decommission_reservation() {
2140 let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
2141 let bucket = format!("heal-capacity-{}", Uuid::new_v4().simple());
2142 let object = "targeted-heal.bin";
2143 store
2144 .make_bucket(&bucket, &MakeBucketOptions::default())
2145 .await
2146 .expect("heal capacity bucket should be created");
2147
2148 let target_set = store.pools[1].get_disks(0);
2149 let mut reader = PutObjReader::from_vec(b"targeted heal body".to_vec());
2150 target_set
2151 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
2152 .await
2153 .expect("targeted heal fixture should be written");
2154 let missing_disk = target_set.disks.read().await[0]
2155 .clone()
2156 .expect("targeted heal fixture disk should be online");
2157 missing_disk
2158 .delete(
2159 &bucket,
2160 object,
2161 DeleteOptions {
2162 recursive: true,
2163 immediate: true,
2164 ..Default::default()
2165 },
2166 )
2167 .await
2168 .expect("targeted heal fixture shard should be removed");
2169 assert!(
2170 missing_disk.read_xl(&bucket, object, false).await.is_err(),
2171 "targeted heal fixture must start with one missing metadata copy"
2172 );
2173
2174 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2175 set_decommission_capacity_info_overrides_for_test(
2176 store.id,
2177 vec![vec![
2178 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
2179 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
2180 ]],
2181 );
2182 store
2183 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
2184 .await
2185 .expect("the exact-fit decommission reservation should activate");
2186 {
2187 let pool_meta = store.pool_meta.read().await;
2188 let reservation = pool_meta.pools[0]
2189 .decommission
2190 .as_ref()
2191 .and_then(|info| info.capacity_reservation.as_ref())
2192 .expect("the active decommission reservation should be durable");
2193 assert_eq!(reservation.targets.len(), 1);
2194 assert_eq!(reservation.targets[0].pool_index, 1);
2195 assert_eq!(reservation.targets[0].reserved_physical_bytes, 60);
2196 }
2197
2198 let (_, err) = store
2199 .handle_heal_object(
2200 &bucket,
2201 object,
2202 "",
2203 &HealOpts {
2204 pool: Some(1),
2205 set: Some(0),
2206 ..Default::default()
2207 },
2208 )
2209 .await
2210 .expect("targeted heal should return a mapped capacity result");
2211 assert!(matches!(err, Some(Error::SlowDown)), "targeted heal must be capacity-blocked: {err:?}");
2212 assert!(
2213 missing_disk.read_xl(&bucket, object, false).await.is_err(),
2214 "capacity-blocked targeted heal must not rewrite the missing shard"
2215 );
2216
2217 let (_, err) = tokio::time::timeout(
2218 std::time::Duration::from_secs(30),
2219 store.handle_heal_object(
2220 RUSTFS_META_BUCKET,
2221 POOL_META_NAME,
2222 "",
2223 &HealOpts {
2224 pool: Some(1),
2225 ..Default::default()
2226 },
2227 ),
2228 )
2229 .await
2230 .expect("pool metadata admission should not recurse on its namespace lock")
2231 .expect("pool metadata capacity rejection should be mapped");
2232 assert!(
2233 matches!(err, Some(Error::SlowDown)),
2234 "pool metadata heal must preserve target reservation admission: {err:?}"
2235 );
2236 }
2237
2238 #[tokio::test]
2239 #[serial_test::serial]
2240 async fn pool_meta_heal_keeps_per_target_capacity_admission() {
2241 let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
2242 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2243 set_decommission_capacity_info_overrides_for_test(
2244 store.id,
2245 vec![vec![
2246 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
2247 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
2248 DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
2249 ]],
2250 );
2251 store
2252 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
2253 .await
2254 .expect("pool metadata reservation should activate");
2255 {
2256 let pool_meta = store.pool_meta.read().await;
2257 let reservation = pool_meta.pools[0]
2258 .decommission
2259 .as_ref()
2260 .and_then(|info| info.capacity_reservation.as_ref())
2261 .expect("pool metadata reservation should be durable");
2262 assert_eq!(reservation.targets.len(), 1);
2263 assert_eq!(reservation.targets[0].pool_index, 1);
2264 }
2265
2266 let missing_disk = remove_pool_meta_shard(&store, 2).await;
2267
2268 let (result, err) = tokio::time::timeout(
2269 std::time::Duration::from_secs(30),
2270 store.handle_heal_object(RUSTFS_META_BUCKET, POOL_META_NAME, "", &HealOpts::default()),
2271 )
2272 .await
2273 .expect("unscoped pool metadata heal should complete")
2274 .expect("unscoped pool metadata heal should return a mapped result");
2275
2276 assert!(err.is_none(), "an admitted target should let unscoped metadata heal succeed: {err:?}");
2277 assert_eq!(result.object, POOL_META_NAME);
2278 assert!(
2279 missing_disk.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false).await.is_ok(),
2280 "unscoped metadata heal should repair the admitted target while another target is reserved"
2281 );
2282 }
2283
2284 #[tokio::test]
2285 #[serial_test::serial]
2286 async fn targeted_heal_keeps_target_lock_for_different_lock_domain() {
2287 let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
2288 let bucket = format!("heal-lock-domain-{}", Uuid::new_v4().simple());
2289 let object = "different-domain-heal.bin";
2290 store
2291 .make_bucket(&bucket, &MakeBucketOptions::default())
2292 .await
2293 .expect("heal lock-domain bucket should be created");
2294
2295 let target_set = other_store.pools[1].get_disks(0);
2296 let mut reader = PutObjReader::from_vec(b"different lock domain body".to_vec());
2297 target_set
2298 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
2299 .await
2300 .expect("heal lock-domain fixture should be written");
2301 let missing_disk = target_set.disks.read().await[0]
2302 .clone()
2303 .expect("heal lock-domain fixture disk should be online");
2304 missing_disk
2305 .delete(
2306 &bucket,
2307 object,
2308 DeleteOptions {
2309 recursive: true,
2310 immediate: true,
2311 ..Default::default()
2312 },
2313 )
2314 .await
2315 .expect("heal lock-domain fixture shard should be removed");
2316 assert!(
2317 missing_disk.read_xl(&bucket, object, false).await.is_err(),
2318 "heal lock-domain fixture must start with one missing metadata copy"
2319 );
2320
2321 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2322 set_decommission_capacity_info_overrides_for_test(
2323 store.id,
2324 vec![vec![
2325 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
2326 DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100),
2327 DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
2328 ]],
2329 );
2330 store
2331 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
2332 .await
2333 .expect("heal lock-domain reservation should activate");
2334 {
2335 let pool_meta = store.pool_meta.read().await;
2336 let reservation = pool_meta.pools[0]
2337 .decommission
2338 .as_ref()
2339 .and_then(|info| info.capacity_reservation.as_ref())
2340 .expect("heal lock-domain reservation should be durable");
2341 assert_eq!(reservation.targets.len(), 1);
2342 assert_eq!(reservation.targets[0].pool_index, 2);
2343 }
2344 let pool_meta = store.pool_meta.read().await.clone();
2345 *other_store.pool_meta.write().await = pool_meta;
2346
2347 let fixed_set = other_store.pools[0].disk_set[0].clone();
2348 assert!(
2349 !fixed_set.shares_namespace_lock_domain(&target_set).await,
2350 "heal fixture must use different fixed and target lock domains"
2351 );
2352 let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, other_store.id);
2353 let heal_store = Arc::clone(&other_store);
2354 let heal_bucket = bucket.clone();
2355 let heal_object = object.to_string();
2356 let mut heal = tokio::spawn(async move {
2357 heal_store
2358 .handle_heal_object(
2359 &heal_bucket,
2360 &heal_object,
2361 "",
2362 &HealOpts {
2363 pool: Some(1),
2364 set: Some(0),
2365 ..Default::default()
2366 },
2367 )
2368 .await
2369 });
2370
2371 barrier.wait_until_external_capacity_released().await;
2372 barrier.wait_until_external_heal_target_lock_attempted().await;
2373
2374 let (_, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
2375 .await
2376 .expect("targeted heal should finish after target lock release")
2377 .expect("targeted heal task should not panic")
2378 .expect("targeted heal should complete");
2379 assert!(err.is_none(), "targeted heal should repair after target lock release: {err:?}");
2380 assert!(
2381 missing_disk.read_xl(&bucket, object, false).await.is_ok(),
2382 "targeted heal should rewrite the missing shard after the target lock is released"
2383 );
2384 drop(barrier);
2385 }
2386
2387 #[tokio::test]
2388 #[serial_test::serial]
2389 async fn targeted_heal_reuses_shared_target_lock_without_reentrant_lock() {
2390 let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
2391 let bucket = format!("heal-shared-lock-domain-{}", Uuid::new_v4().simple());
2392 let object = "shared-domain-heal.bin";
2393 store
2394 .make_bucket(&bucket, &MakeBucketOptions::default())
2395 .await
2396 .expect("shared heal lock-domain bucket should be created");
2397
2398 let target_set = store.pools[0].get_disks(0);
2399 let mut reader = PutObjReader::from_vec(b"shared lock domain body".to_vec());
2400 target_set
2401 .put_object(&bucket, object, &mut reader, &ObjectOptions::default())
2402 .await
2403 .expect("shared heal lock-domain fixture should be written");
2404 let missing_disk = target_set.disks.read().await[0]
2405 .clone()
2406 .expect("shared heal lock-domain fixture disk should be online");
2407 missing_disk
2408 .delete(
2409 &bucket,
2410 object,
2411 DeleteOptions {
2412 recursive: true,
2413 immediate: true,
2414 ..Default::default()
2415 },
2416 )
2417 .await
2418 .expect("shared heal lock-domain fixture shard should be removed");
2419 assert!(
2420 missing_disk.read_xl(&bucket, object, false).await.is_err(),
2421 "shared heal lock-domain fixture must start with one missing metadata copy"
2422 );
2423
2424 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2425 set_decommission_capacity_info_overrides_for_test(
2426 store.id,
2427 vec![vec![
2428 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
2429 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
2430 DecommissionPoolCapacityInfo::for_test(2, layout, 0, 30, 30),
2431 ]],
2432 );
2433 store
2434 .save_current_pool_meta_for_decommission_start(&[2], Vec::new())
2435 .await
2436 .expect("shared heal lock-domain reservation should activate");
2437 {
2438 let pool_meta = store.pool_meta.read().await;
2439 let reservation = pool_meta.pools[2]
2440 .decommission
2441 .as_ref()
2442 .and_then(|info| info.capacity_reservation.as_ref())
2443 .expect("shared heal lock-domain reservation should be durable");
2444 assert_eq!(reservation.targets.len(), 1);
2445 assert_eq!(reservation.targets[0].pool_index, 1);
2446 assert_eq!(reservation.targets[0].reserved_physical_bytes, 60);
2447 }
2448
2449 let fixed_set = store.pools[0].disk_set[0].clone();
2450 assert!(
2451 fixed_set.shares_namespace_lock_domain(&target_set).await,
2452 "shared heal fixture must use one fixed and target lock domain"
2453 );
2454
2455 let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
2456 let heal_store = Arc::clone(&store);
2457 let heal_bucket = bucket.clone();
2458 let heal_object = object.to_string();
2459 let mut heal = tokio::spawn(async move {
2460 heal_store
2461 .handle_heal_object(
2462 &heal_bucket,
2463 &heal_object,
2464 "",
2465 &HealOpts {
2466 pool: Some(0),
2467 set: Some(0),
2468 ..Default::default()
2469 },
2470 )
2471 .await
2472 });
2473
2474 barrier.wait_until_external_capacity_released().await;
2475 barrier.wait_until_external_heal_operation_started().await;
2476 let (_, err) = tokio::time::timeout(std::time::Duration::from_secs(5), &mut heal)
2477 .await
2478 .expect("shared-domain heal must not reenter the fixed namespace lock")
2479 .expect("shared-domain heal task should not panic")
2480 .expect("shared-domain heal should complete");
2481 assert!(err.is_none(), "shared-domain heal should repair after admission: {err:?}");
2482 assert!(
2483 missing_disk.read_xl(&bucket, object, false).await.is_ok(),
2484 "shared-domain heal should rewrite the missing shard"
2485 );
2486 drop(barrier);
2487 }
2488
2489 #[tokio::test]
2490 async fn scoped_heal_object_defers_when_requested_pool_is_suspended() {
2491 let mut store = minimal_heal_store().await;
2492 store.pool_meta = RwLock::new(PoolMeta {
2493 pools: vec![
2494 PoolStatus {
2495 id: 0,
2496 cmd_line: "pool-0".to_string(),
2497 last_update: OffsetDateTime::UNIX_EPOCH,
2498 decommission: None,
2499 },
2500 PoolStatus {
2501 id: 1,
2502 cmd_line: "pool-1".to_string(),
2503 last_update: OffsetDateTime::UNIX_EPOCH,
2504 decommission: Some(PoolDecommissionInfo {
2505 start_time: Some(OffsetDateTime::UNIX_EPOCH),
2506 ..Default::default()
2507 }),
2508 },
2509 ],
2510 ..Default::default()
2511 });
2512
2513 let (_, err) = store
2514 .handle_heal_object(
2515 "bucket",
2516 "object",
2517 "",
2518 &HealOpts {
2519 pool: Some(1),
2520 set: Some(0),
2521 ..Default::default()
2522 },
2523 )
2524 .await
2525 .expect("suspended pool should return a deferred heal result");
2526
2527 assert!(matches!(err, Some(StorageError::SlowDown)));
2528
2529 let (_, err) = store
2530 .handle_heal_object(
2531 "bucket",
2532 "object",
2533 "",
2534 &HealOpts {
2535 set: Some(1),
2536 ..Default::default()
2537 },
2538 )
2539 .await
2540 .expect("unscoped heal should return the active pool result");
2541
2542 assert!(matches!(err, Some(StorageError::InvalidArgument(_, ref field, _)) if field == "set"));
2543
2544 let err = store
2545 .handle_heal_object(
2546 "bucket",
2547 "object",
2548 "",
2549 &HealOpts {
2550 pool: Some(1),
2551 set: Some(1),
2552 ..Default::default()
2553 },
2554 )
2555 .await
2556 .expect_err("invalid set scope should fail before suspended pool deferral");
2557
2558 assert!(matches!(err, StorageError::InvalidArgument(_, ref field, _) if field == "set"));
2559
2560 {
2561 let mut pool_meta = store.pool_meta.write().await;
2562 let decommission = pool_meta.pools[1]
2563 .decommission
2564 .as_mut()
2565 .expect("test pool should have decommission state");
2566 decommission.complete = true;
2567 }
2568 let (_, err) = store
2569 .handle_heal_object(
2570 "bucket",
2571 "object",
2572 "",
2573 &HealOpts {
2574 pool: Some(1),
2575 set: Some(0),
2576 ..Default::default()
2577 },
2578 )
2579 .await
2580 .expect("completed pool should return a terminal heal result");
2581
2582 assert!(matches!(
2583 err,
2584 Some(StorageError::InvalidArgument(_, ref field, ref reason))
2585 if field == "pool" && reason.contains("completed decommission")
2586 ));
2587
2588 for canceled in [false, true] {
2589 {
2590 let mut pool_meta = store.pool_meta.write().await;
2591 let decommission = pool_meta.pools[1]
2592 .decommission
2593 .as_mut()
2594 .expect("test pool should have decommission state");
2595 decommission.complete = false;
2596 decommission.failed = !canceled;
2597 decommission.canceled = canceled;
2598 }
2599 let (_, err) = store
2600 .handle_heal_object(
2601 "bucket",
2602 "object",
2603 "",
2604 &HealOpts {
2605 pool: Some(1),
2606 set: Some(0),
2607 ..Default::default()
2608 },
2609 )
2610 .await
2611 .expect("clearable terminal pool should return a deferred heal result");
2612
2613 assert!(matches!(err, Some(StorageError::SlowDown)));
2614 }
2615 }
2616
2617 #[tokio::test]
2618 async fn scoped_heal_bucket_blocks_before_dispatch_when_pool_is_suspended() {
2619 let mut store = minimal_heal_store().await;
2620 store.pool_meta = RwLock::new(PoolMeta {
2621 pools: vec![
2622 PoolStatus {
2623 id: 0,
2624 cmd_line: "pool-0".to_string(),
2625 last_update: OffsetDateTime::UNIX_EPOCH,
2626 decommission: None,
2627 },
2628 PoolStatus {
2629 id: 1,
2630 cmd_line: "pool-1".to_string(),
2631 last_update: OffsetDateTime::UNIX_EPOCH,
2632 decommission: Some(PoolDecommissionInfo {
2633 start_time: Some(OffsetDateTime::UNIX_EPOCH),
2634 ..Default::default()
2635 }),
2636 },
2637 ],
2638 ..Default::default()
2639 });
2640
2641 let err = store
2642 .handle_heal_bucket(
2643 "bucket",
2644 &HealOpts {
2645 pool: Some(1),
2646 ..Default::default()
2647 },
2648 )
2649 .await
2650 .expect_err("suspended pool must be blocked before bucket-heal fan-out");
2651 assert_eq!(err, Error::SlowDown);
2652
2653 store.pool_meta.write().await.pools[1]
2654 .decommission
2655 .as_mut()
2656 .expect("decommission state should exist")
2657 .complete = true;
2658 let err = store
2659 .handle_heal_bucket(
2660 "bucket",
2661 &HealOpts {
2662 pool: Some(1),
2663 ..Default::default()
2664 },
2665 )
2666 .await
2667 .expect_err("completed pool must remain fenced from bucket heal");
2668 assert!(
2669 matches!(err, StorageError::InvalidArgument(_, ref field, ref reason)
2670 if field == "pool" && reason.contains("completed decommission")),
2671 "unexpected completed-pool error: {err:?}"
2672 );
2673 }
2674
2675 #[tokio::test]
2676 async fn bucket_heal_blocks_before_dispatch_after_unreadable_pool_meta_replica() {
2677 let store = minimal_heal_store().await;
2678 store.pool_meta_save_gate.lock().await.observe_replicas(PoolMetaReplicaState {
2679 needs_repair: true,
2680 repair_write_safe: false,
2681 });
2682
2683 let err = store
2684 .handle_heal_bucket("bucket", &HealOpts::default())
2685 .await
2686 .expect_err("bucket heal must stay blocked until restart after an unreadable replica");
2687
2688 assert!(
2689 err.to_string()
2690 .contains("restart after all replicas are readable and consistent")
2691 );
2692 }
2693
2694 #[tokio::test]
2695 async fn bucket_heal_releases_save_gate_before_peer_dispatch_and_holds_movement_snapshot() {
2696 let mut store = minimal_heal_store().await;
2697 let started = Arc::new(tokio::sync::Notify::new());
2698 let release = Arc::new(tokio::sync::Notify::new());
2699 let peer: Box<dyn PeerS3Client> = Box::new(BlockingHealPeer {
2700 started: started.clone(),
2701 release: release.clone(),
2702 });
2703 store.peer_sys.clients = vec![Arc::new(peer)];
2704 let store = Arc::new(store);
2705 let movement_gate = store.ctx.data_movement_operation_gate();
2706 let mut heal = tokio::spawn({
2707 let store = store.clone();
2708 async move { store.handle_heal_bucket("bucket", &HealOpts::default()).await }
2709 });
2710
2711 tokio::time::timeout(std::time::Duration::from_secs(1), started.notified())
2712 .await
2713 .expect("peer dispatch should start");
2714 assert!(
2715 store.pool_meta_save_gate.try_lock().is_ok(),
2716 "coordinator must release its local save gate before waiting for peers"
2717 );
2718 assert!(
2719 movement_gate.try_write().is_err(),
2720 "bucket heal must hold the movement snapshot through peer dispatch"
2721 );
2722
2723 release.notify_one();
2724 tokio::time::timeout(std::time::Duration::from_secs(1), &mut heal)
2725 .await
2726 .expect("bucket heal should finish after peer release")
2727 .expect("bucket heal task should not panic")
2728 .expect("bucket heal should succeed");
2729 }
2730
2731 #[tokio::test]
2732 async fn bucket_heal_local_fanout_does_not_reenter_movement_read_behind_queued_writer() {
2733 let mut store = minimal_heal_store().await;
2734 let movement_gate = store.ctx.data_movement_operation_gate();
2735 let writer_queued = Arc::new(tokio::sync::Notify::new());
2736 let writer_acquired = Arc::new(tokio::sync::Notify::new());
2737 let peer: Box<dyn PeerS3Client> = Box::new(WriterQueuedLocalHealPeer {
2738 movement_gate: movement_gate.clone(),
2739 writer_queued: writer_queued.clone(),
2740 writer_acquired: writer_acquired.clone(),
2741 });
2742 store.peer_sys.clients = vec![Arc::new(peer)];
2743 let store = Arc::new(store);
2744 let mut heal = tokio::spawn({
2745 let store = store.clone();
2746 async move { store.handle_heal_bucket("bucket", &HealOpts::default()).await }
2747 });
2748
2749 tokio::time::timeout(std::time::Duration::from_secs(1), writer_queued.notified())
2750 .await
2751 .expect("movement writer should queue during local peer lookup");
2752 tokio::time::timeout(std::time::Duration::from_secs(1), &mut heal)
2753 .await
2754 .expect("local fan-out must not reenter movement read behind the queued writer")
2755 .expect("bucket heal task should not panic")
2756 .expect("bucket heal should succeed");
2757 tokio::time::timeout(std::time::Duration::from_secs(1), writer_acquired.notified())
2758 .await
2759 .expect("queued movement writer should proceed after bucket heal releases its read guard");
2760 }
2761
2762 #[tokio::test]
2763 #[serial_test::serial]
2764 async fn unscoped_heal_object_suspended_owner_semantics() {
2765 let (_temp_dir, store, shutdown) = multi_pool_heal_store().await;
2766 let bucket = format!("heal-owner-{}", Uuid::new_v4().simple());
2767 let active_object = "active-owner";
2768 let suspended_only_object = "suspended-only";
2769 let duplicate_object = "duplicate-owner";
2770 let marker_object = "marker-owner";
2771 let quorum_object = "quorum-owner";
2772 store
2773 .make_bucket(&bucket, &MakeBucketOptions::default())
2774 .await
2775 .expect("bucket should be created in all pools");
2776
2777 let mut active_reader = PutObjReader::from_vec(b"active owner".to_vec());
2778 store.pools[0]
2779 .put_object(&bucket, active_object, &mut active_reader, &ObjectOptions::default())
2780 .await
2781 .expect("active owner object should be written");
2782 let active_disks = store.pools[0].disk_set[0].disks.read().await.clone();
2783 let missing_active_disk = active_disks[0].clone().expect("active disk should be online");
2784 missing_active_disk
2785 .delete(
2786 &bucket,
2787 active_object,
2788 DeleteOptions {
2789 recursive: true,
2790 immediate: true,
2791 ..Default::default()
2792 },
2793 )
2794 .await
2795 .expect("active owner shard should be removed for repair");
2796 assert!(
2797 missing_active_disk.read_xl(&bucket, active_object, false).await.is_err(),
2798 "the active owner fixture must start with one missing metadata copy"
2799 );
2800
2801 let mut suspended_reader = PutObjReader::from_vec(b"suspended owner".to_vec());
2802 store.pools[1]
2803 .put_object(&bucket, suspended_only_object, &mut suspended_reader, &ObjectOptions::default())
2804 .await
2805 .expect("suspended owner object should be written");
2806 for (pool_index, mod_time) in [1_i64, 2_i64].into_iter().enumerate() {
2807 let mut duplicate_reader = PutObjReader::from_vec(format!("duplicate-pool-{pool_index}").into_bytes());
2808 store.pools[pool_index]
2809 .put_object(
2810 &bucket,
2811 duplicate_object,
2812 &mut duplicate_reader,
2813 &ObjectOptions {
2814 mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(mod_time)),
2815 ..Default::default()
2816 },
2817 )
2818 .await
2819 .expect("duplicate owner object should be written");
2820 }
2821 let duplicate_missing_disk = store.pools[0].disk_set[0].disks.read().await[0]
2822 .clone()
2823 .expect("duplicate active owner disk should be online");
2824 duplicate_missing_disk
2825 .delete(
2826 &bucket,
2827 duplicate_object,
2828 DeleteOptions {
2829 recursive: true,
2830 immediate: true,
2831 ..Default::default()
2832 },
2833 )
2834 .await
2835 .expect("duplicate active owner shard should be removed for repair");
2836 let history_version = Uuid::new_v4();
2837 let mut history_reader = PutObjReader::from_vec(b"marker history".to_vec());
2838 store.pools[0]
2839 .put_object(
2840 &bucket,
2841 marker_object,
2842 &mut history_reader,
2843 &ObjectOptions {
2844 versioned: true,
2845 version_id: Some(history_version.to_string()),
2846 mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
2847 ..Default::default()
2848 },
2849 )
2850 .await
2851 .expect("versioned marker history should be written");
2852 store.pools[0]
2853 .delete_object(
2854 &bucket,
2855 marker_object,
2856 ObjectOptions {
2857 versioned: true,
2858 mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
2859 ..Default::default()
2860 },
2861 )
2862 .await
2863 .expect("delete marker should be written");
2864 let mut quorum_reader = PutObjReader::from_vec(b"quorum boundary".to_vec());
2865 store.pools[0]
2866 .put_object(&bucket, quorum_object, &mut quorum_reader, &ObjectOptions::default())
2867 .await
2868 .expect("quorum boundary object should be written");
2869 {
2870 let mut pool_meta = store.pool_meta.write().await;
2871 let mut next = PoolMeta::new(&store.pools, &pool_meta);
2872 next.pools[1].decommission = Some(PoolDecommissionInfo {
2873 start_time: Some(OffsetDateTime::UNIX_EPOCH),
2874 ..Default::default()
2875 });
2876 *pool_meta = next;
2877 }
2878
2879 let (_, duplicate_owner) = store
2880 .get_latest_object_info_with_idx(&bucket, duplicate_object, &ObjectOptions::default())
2881 .await
2882 .expect("duplicate owner should resolve");
2883 assert_eq!(duplicate_owner, 1, "latest duplicate must win when all pools are eligible");
2884 let (_, active_duplicate_owner) = store
2885 .get_latest_object_info_with_idx(
2886 &bucket,
2887 duplicate_object,
2888 &ObjectOptions {
2889 skip_decommissioned: true,
2890 ..Default::default()
2891 },
2892 )
2893 .await
2894 .expect("active duplicate owner should resolve");
2895 assert_eq!(
2896 active_duplicate_owner, 0,
2897 "suspended duplicate must be excluded from active owner selection"
2898 );
2899 let (duplicate_result, duplicate_err) = store
2900 .handle_heal_object(&bucket, duplicate_object, "", &HealOpts::default())
2901 .await
2902 .expect("duplicate owner heal should complete through the production path");
2903 assert_eq!(duplicate_result.object, duplicate_object);
2904 assert!(duplicate_err.is_none(), "active duplicate should be repaired: {duplicate_err:?}");
2905 assert!(
2906 duplicate_missing_disk.read_xl(&bucket, duplicate_object, false).await.is_ok(),
2907 "production heal must repair the active duplicate owner rather than the suspended owner"
2908 );
2909 let (marker_info, marker_owner) = store
2910 .get_latest_object_info_with_idx(
2911 &bucket,
2912 marker_object,
2913 &ObjectOptions {
2914 skip_decommissioned: true,
2915 versioned: true,
2916 ..Default::default()
2917 },
2918 )
2919 .await
2920 .expect("latest delete marker should resolve");
2921 assert_eq!(marker_owner, 0);
2922 assert!(marker_info.delete_marker, "latest version must preserve delete-marker semantics");
2923
2924 let (active_result, active_err) = store
2925 .handle_heal_object(&bucket, active_object, "", &HealOpts::default())
2926 .await
2927 .expect("unscoped active-owner heal should complete");
2928 assert_eq!(active_result.object, active_object);
2929 assert!(active_err.is_none(), "active owner must be selected even with a suspended pool");
2930 assert!(
2931 missing_active_disk.read_xl(&bucket, active_object, false).await.is_ok(),
2932 "active owner heal must write the missing disk metadata: result={active_result:?}, err={active_err:?}"
2933 );
2934 assert!(
2935 store.pools[1]
2936 .get_object_info(&bucket, active_object, &ObjectOptions::default())
2937 .await
2938 .is_err(),
2939 "the suspended pool must not be written for an active-owner object"
2940 );
2941
2942 let (suspended_result, suspended_err) = store
2943 .handle_heal_object(&bucket, suspended_only_object, "", &HealOpts::default())
2944 .await
2945 .expect("unscoped suspended-only heal should return a terminal result");
2946 assert!(suspended_result.object.is_empty());
2947 assert!(matches!(suspended_err, Some(Error::FileNotFound)));
2948 assert!(
2949 store.pools[1]
2950 .get_object_info(&bucket, suspended_only_object, &ObjectOptions::default())
2951 .await
2952 .is_ok(),
2953 "suspended-only data must remain untouched when unscoped heal reports absent"
2954 );
2955
2956 let (_, explicit_err) = store
2957 .handle_heal_object(
2958 &bucket,
2959 suspended_only_object,
2960 "",
2961 &HealOpts {
2962 pool: Some(1),
2963 ..Default::default()
2964 },
2965 )
2966 .await
2967 .expect("explicit suspended-owner heal should return a mapped error");
2968 assert!(matches!(explicit_err, Some(Error::SlowDown)));
2969
2970 let original_quorum_disks = store.pools[0].disk_set[0].disks.read().await.clone();
2971 let surviving_quorum_disk = original_quorum_disks[3].clone();
2972 *store.pools[0].disk_set[0].disks.write().await = vec![None, None, None, surviving_quorum_disk];
2973 let (_, quorum_err) = store
2974 .handle_heal_object(&bucket, quorum_object, "", &HealOpts::default())
2975 .await
2976 .expect("quorum boundary heal should return a mapped result");
2977 *store.pools[0].disk_set[0].disks.write().await = original_quorum_disks;
2978 let quorum_err = quorum_err
2979 .as_ref()
2980 .expect("heal must fail closed when capacity admission cannot verify pool metadata");
2981 let quorum_failure = quorum_err
2982 .pool_metadata_failure()
2983 .expect("capacity admission failure should preserve typed pool metadata context");
2984 assert_eq!(
2985 quorum_failure.kind,
2986 crate::error::PoolMetadataFailure::ReadUnavailable,
2987 "read-only capacity admission failure must remain retryable"
2988 );
2989 assert_eq!(quorum_failure.operation, "target capacity admission failed");
2990 assert_eq!(quorum_failure.phase, "pool_read");
2991 assert!(
2992 store.pool_meta_writes_ready().await,
2993 "read-only capacity admission failure must not latch the pool metadata writer"
2994 );
2995 shutdown.cancel();
2996 }
2997
2998 #[tokio::test]
2999 async fn handle_heal_format_continues_after_a_pool_error() {
3000 let canonical_format = FormatV3::new(1, 3);
3001 let mut foreign_format = canonical_format.clone();
3002 foreign_format.id = Uuid::new_v4();
3003 let mut temp_dirs = Vec::new();
3004 let mut endpoints = Vec::new();
3005 let mut disks = Vec::new();
3006
3007 for disk_index in 0..3 {
3008 let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
3009 let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
3010 .expect("temporary endpoint should parse");
3011 endpoint.set_pool_index(0);
3012 endpoint.set_set_index(0);
3013 endpoint.set_disk_index(disk_index);
3014 let disk = new_disk(
3015 &endpoint,
3016 &DiskOption {
3017 cleanup: false,
3018 health_check: false,
3019 },
3020 )
3021 .await
3022 .expect("temporary disk should open");
3023 let mut disk_format = foreign_format.clone();
3024 disk_format.erasure.this = foreign_format.erasure.sets[0][disk_index];
3025 save_format_file(&Some(disk.clone()), &Some(disk_format))
3026 .await
3027 .expect("foreign format should be written");
3028 temp_dirs.push(temp_dir);
3029 endpoints.push(endpoint);
3030 disks.push(Some(disk));
3031 }
3032
3033 let pool_endpoints = PoolEndpoints {
3034 legacy: false,
3035 set_count: 1,
3036 drives_per_set: 3,
3037 endpoints: Endpoints::from(endpoints),
3038 cmd_line: "foreign-format-majority-test".to_string(),
3039 platform: "test".to_string(),
3040 };
3041 let pool = Sets::new(disks, &pool_endpoints, &canonical_format, 0, 1)
3042 .await
3043 .expect("test pool should build around the cached canonical format");
3044
3045 let mut recoverable_format = FormatV3::new(1, 3);
3046 recoverable_format.id = canonical_format.id;
3047 let mut recoverable_temp_dirs = Vec::new();
3048 let mut recoverable_endpoints = Vec::new();
3049 let mut recoverable_disks = Vec::new();
3050 let mut unformatted_disk = None;
3051 for disk_index in 0..3 {
3052 let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
3053 let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("temporary path should be UTF-8"))
3054 .expect("temporary endpoint should parse");
3055 endpoint.set_pool_index(1);
3056 endpoint.set_set_index(0);
3057 endpoint.set_disk_index(disk_index);
3058 let disk = new_disk(
3059 &endpoint,
3060 &DiskOption {
3061 cleanup: false,
3062 health_check: false,
3063 },
3064 )
3065 .await
3066 .expect("temporary disk should open");
3067 if disk_index < 2 {
3068 let mut disk_format = recoverable_format.clone();
3069 disk_format.erasure.this = recoverable_format.erasure.sets[0][disk_index];
3070 save_format_file(&Some(disk.clone()), &Some(disk_format))
3071 .await
3072 .expect("recoverable format should be written");
3073 } else {
3074 unformatted_disk = Some(disk.clone());
3075 }
3076 recoverable_temp_dirs.push(temp_dir);
3077 recoverable_endpoints.push(endpoint);
3078 recoverable_disks.push(Some(disk));
3079 }
3080 let recoverable_pool_endpoints = PoolEndpoints {
3081 legacy: false,
3082 set_count: 1,
3083 drives_per_set: 3,
3084 endpoints: Endpoints::from(recoverable_endpoints),
3085 cmd_line: "recoverable-format-test".to_string(),
3086 platform: "test".to_string(),
3087 };
3088 let recoverable_pool = Sets::new(recoverable_disks, &recoverable_pool_endpoints, &recoverable_format, 1, 1)
3089 .await
3090 .expect("recoverable test pool should build");
3091
3092 let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints.clone(), recoverable_pool_endpoints.clone()]);
3093 let store = ECStore {
3094 id: canonical_format.id,
3095 disk_map: HashMap::new(),
3096 pools: vec![pool, recoverable_pool],
3097 peer_sys: S3PeerSys::new(&endpoint_pools),
3098 pool_meta: RwLock::new(PoolMeta::default()),
3099 rebalance_meta: RwLock::new(None),
3100 decommission_cancelers: RwLock::new(Vec::new()),
3101 start_gate: Mutex::new(()),
3102 pool_meta_save_gate: Mutex::default(),
3103 ctx: crate::runtime::instance::bootstrap_ctx(),
3104 bucket_fence_registry: std::sync::Arc::default(),
3105 };
3106
3107 let err = store
3108 .handle_heal_format(false)
3109 .await
3110 .expect_err("missing pool metadata must fail closed before format writes");
3111 assert!(err.to_string().contains("no durable bootstrap identity or pool.bin replica"));
3112 store
3113 .ensure_pool_meta_side_effects_safe("missing format-heal metadata side effect")
3114 .await
3115 .expect_err("missing metadata must latch the format-heal write gate");
3116
3117 let pool_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
3118 pool_meta
3119 .save_for_startup(store.pools.clone())
3120 .await
3121 .expect("pool metadata should be persisted before format heal");
3122 *store.pool_meta_save_gate.lock().await = PoolMetaWriteState::default();
3123
3124 let (result, err) = store
3125 .handle_heal_format(false)
3126 .await
3127 .expect("format heal should return the typed pool error");
3128 assert!(
3129 matches!(err, Some(StorageError::CorruptedFormat)),
3130 "foreign format majority must not be downgraded to a successful heal: {err:?}"
3131 );
3132 assert_eq!(result.disk_count, 3, "the recoverable pool should still be inspected");
3133 let healed = load_format_erasure(&unformatted_disk.expect("the unformatted disk handle should be retained"), true)
3134 .await
3135 .expect("the later pool should be healed despite the first pool error");
3136 assert_eq!(healed.erasure.this, recoverable_format.erasure.sets[0][2]);
3137
3138 let mut completed_meta = PoolMeta::new(&store.pools, &PoolMeta::default());
3139 for status in &mut completed_meta.pools {
3140 status.decommission = Some(PoolDecommissionInfo {
3141 complete: true,
3142 ..Default::default()
3143 });
3144 }
3145 completed_meta
3146 .save(store.pools.clone())
3147 .await
3148 .expect("completed pool metadata should be persisted");
3149 let (_, err) = store
3150 .handle_heal_format(false)
3151 .await
3152 .expect("completed pools should be reported as a no-op");
3153 assert!(matches!(err, Some(StorageError::NoHealRequired)));
3154 }
3155}