1use super::*;
16use crate::config::storageclass;
17use crate::core::pools::merge_pool_status_refresh;
18use crate::layout::pool_space::{ServerPoolsAvailableSpace, build_server_pools_available_space};
19use crate::runtime::sources as runtime_sources;
20use crate::storage_api_contracts::{admin::StorageAdminApi, namespace::NamespaceLocking as _, object::ObjectOperations as _};
21use futures::stream::{FuturesUnordered, StreamExt};
22pub(in crate::store) mod support;
23
24const LOG_COMPONENT_ECSTORE: &str = "ecstore";
25const LOG_SUBSYSTEM_POOLS: &str = "pools";
26const EVENT_POOL_META_RELOAD: &str = "pool_meta_reload";
27
28#[cfg(test)]
29struct PreparedPoolReadFallbackBarrierState {
30 object: String,
31 pause_before_refetch: bool,
32 fanout_arrived: tokio::sync::Notify,
33 fanout_release: tokio::sync::Notify,
34 refetch_arrived: tokio::sync::Notify,
35 refetch_release: tokio::sync::Notify,
36}
37
38#[cfg(test)]
39pub(in crate::store) struct PreparedPoolReadFallbackBarrier {
40 state: Arc<PreparedPoolReadFallbackBarrierState>,
41}
42
43#[cfg(test)]
44static PREPARED_POOL_READ_FALLBACK_BARRIER: std::sync::OnceLock<
45 std::sync::Mutex<Option<Arc<PreparedPoolReadFallbackBarrierState>>>,
46> = std::sync::OnceLock::new();
47
48#[cfg(test)]
49impl PreparedPoolReadFallbackBarrier {
50 pub(in crate::store) fn install(object: &str, pause_before_refetch: bool) -> Self {
51 let state = Arc::new(PreparedPoolReadFallbackBarrierState {
52 object: object.to_string(),
53 pause_before_refetch,
54 fanout_arrived: tokio::sync::Notify::new(),
55 fanout_release: tokio::sync::Notify::new(),
56 refetch_arrived: tokio::sync::Notify::new(),
57 refetch_release: tokio::sync::Notify::new(),
58 });
59 *PREPARED_POOL_READ_FALLBACK_BARRIER
60 .get_or_init(|| std::sync::Mutex::new(None))
61 .lock()
62 .expect("prepared pool read fallback barrier must not be poisoned") = Some(Arc::clone(&state));
63 Self { state }
64 }
65
66 pub(in crate::store) async fn wait_after_fanout(&self) {
67 self.state.fanout_arrived.notified().await;
68 }
69
70 pub(in crate::store) fn release_after_fanout(&self) {
71 self.state.fanout_release.notify_one();
72 }
73
74 pub(in crate::store) async fn wait_before_refetch(&self) {
75 self.state.refetch_arrived.notified().await;
76 }
77
78 pub(in crate::store) fn release_before_refetch(&self) {
79 self.state.refetch_release.notify_one();
80 }
81}
82
83#[cfg(test)]
84impl Drop for PreparedPoolReadFallbackBarrier {
85 fn drop(&mut self) {
86 self.state.fanout_release.notify_waiters();
87 self.state.refetch_release.notify_waiters();
88 if let Some(barrier) = PREPARED_POOL_READ_FALLBACK_BARRIER.get() {
89 *barrier
90 .lock()
91 .expect("prepared pool read fallback barrier must not be poisoned") = None;
92 }
93 }
94}
95
96#[cfg(test)]
97async fn pause_prepared_pool_read_after_fanout(object: &str) {
98 let state = PREPARED_POOL_READ_FALLBACK_BARRIER
99 .get_or_init(|| std::sync::Mutex::new(None))
100 .lock()
101 .expect("prepared pool read fallback barrier must not be poisoned")
102 .as_ref()
103 .filter(|state| state.object == object)
104 .cloned();
105 if let Some(state) = state {
106 state.fanout_arrived.notify_one();
107 state.fanout_release.notified().await;
108 }
109}
110
111#[cfg(test)]
112async fn pause_prepared_pool_read_before_refetch(object: &str) {
113 let state = PREPARED_POOL_READ_FALLBACK_BARRIER
114 .get_or_init(|| std::sync::Mutex::new(None))
115 .lock()
116 .expect("prepared pool read fallback barrier must not be poisoned")
117 .as_ref()
118 .filter(|state| state.object == object && state.pause_before_refetch)
119 .cloned();
120 if let Some(state) = state {
121 state.refetch_arrived.notify_one();
122 state.refetch_release.notified().await;
123 }
124}
125#[cfg(test)]
126use support::resolve_latest_object_info_candidates;
127use support::{
128 LatestObjectInfoCandidate, PoolErr, PoolObjInfo, RebalanceDeletePoolResult, pool_lookup_not_found_error,
129 rebalance_disk_set_lookup_error, resolve_latest_object_info_candidates_with_pool_state,
130 resolve_rebalance_delete_from_all_pools_result, resolve_rebalance_delete_from_all_pools_results,
131 resolve_store_rebalance_pool_meta_reload_result, validate_prepared_pool_refetch_identity,
132};
133
134#[derive(Debug, Default, Eq, PartialEq)]
135struct BackendStorageClassInfo {
136 standard_sc_data: Vec<usize>,
137 standard_sc_parities: Vec<usize>,
138 standard_sc_parity: Option<usize>,
139 rr_sc_data: Vec<usize>,
140 rr_sc_parities: Vec<usize>,
141 rr_sc_parity: Option<usize>,
142}
143
144fn resolve_pool_layout(
145 drives_per_set: &[usize],
146 mut parity_for_pool: impl FnMut(usize, usize) -> Option<usize>,
147) -> Option<(Vec<usize>, Vec<usize>)> {
148 let mut parities = Vec::with_capacity(drives_per_set.len());
149 let mut data = Vec::with_capacity(drives_per_set.len());
150
151 for (pool_index, &drives) in drives_per_set.iter().enumerate() {
152 if drives == 0 {
153 return None;
154 }
155
156 let parity = parity_for_pool(pool_index, drives)?;
157 let data_drives = drives.checked_sub(parity)?;
158 if data_drives == 0 || parity > data_drives {
159 return None;
160 }
161
162 data.push(data_drives);
163 parities.push(parity);
164 }
165
166 Some((parities, data))
167}
168
169fn homogeneous_parity(parities: &[usize]) -> Option<usize> {
170 let first = *parities.first()?;
171 parities.iter().all(|&parity| parity == first).then_some(first)
172}
173
174fn resolve_complete_pool_layouts(
175 drives_per_set: &[usize],
176 standard_parity_for_pool: impl FnMut(usize, usize) -> Option<usize>,
177 rr_parity_for_pool: impl FnMut(usize, usize) -> Option<usize>,
178) -> Option<BackendStorageClassInfo> {
179 let (standard_sc_parities, standard_sc_data) = resolve_pool_layout(drives_per_set, standard_parity_for_pool)?;
180 let (rr_sc_parities, rr_sc_data) = resolve_pool_layout(drives_per_set, rr_parity_for_pool)?;
181
182 for ((&standard_parity, &rr_parity), &drives) in standard_sc_parities.iter().zip(&rr_sc_parities).zip(drives_per_set) {
183 storageclass::validate_parity_inner(standard_parity, rr_parity, drives).ok()?;
184 }
185
186 Some(BackendStorageClassInfo {
187 standard_sc_parity: homogeneous_parity(&standard_sc_parities),
188 standard_sc_data,
189 standard_sc_parities,
190 rr_sc_parity: homogeneous_parity(&rr_sc_parities),
191 rr_sc_data,
192 rr_sc_parities,
193 })
194}
195
196fn resolve_backend_storage_class_info(
197 config: &storageclass::Config,
198 drives_per_set: &[usize],
199 default_standard_parities: &[usize],
200) -> BackendStorageClassInfo {
201 if drives_per_set.len() != default_standard_parities.len() {
202 return BackendStorageClassInfo::default();
203 }
204
205 if !config.is_initialized() {
206 let Some((standard_sc_parities, standard_sc_data)) =
207 resolve_pool_layout(drives_per_set, |pool_index, _| default_standard_parities.get(pool_index).copied())
208 else {
209 return BackendStorageClassInfo::default();
210 };
211
212 return BackendStorageClassInfo {
213 standard_sc_parity: homogeneous_parity(&standard_sc_parities),
214 standard_sc_data,
215 standard_sc_parities,
216 ..Default::default()
217 };
218 }
219
220 match (config.parities_for_sc(storageclass::STANDARD), config.parities_for_sc(storageclass::RRS)) {
221 (Some(standard), Some(rr)) if standard.len() == drives_per_set.len() && rr.len() == drives_per_set.len() => {
222 resolve_complete_pool_layouts(
223 drives_per_set,
224 |pool_index, drives| config.parity_for_pool(storageclass::STANDARD, pool_index, drives),
225 |pool_index, drives| config.parity_for_pool(storageclass::RRS, pool_index, drives),
226 )
227 .unwrap_or_default()
228 }
229 (None, None) => {
230 let Some(standard) = config.get_parity_for_sc(storageclass::STANDARD) else {
231 return BackendStorageClassInfo::default();
232 };
233 let Some(rr) = config.get_parity_for_sc(storageclass::RRS) else {
234 return BackendStorageClassInfo::default();
235 };
236
237 resolve_complete_pool_layouts(drives_per_set, |_, _| Some(standard), |_, _| Some(rr)).unwrap_or_default()
238 }
239 _ => BackendStorageClassInfo::default(),
240 }
241}
242
243fn build_backend_info(
244 config: &storageclass::Config,
245 drives_per_set: &[usize],
246 default_standard_parities: &[usize],
247 total_sets: &[usize],
248) -> rustfs_madmin::BackendInfo {
249 let storage_class_info = if total_sets.len() == drives_per_set.len() {
250 resolve_backend_storage_class_info(config, drives_per_set, default_standard_parities)
251 } else {
252 BackendStorageClassInfo::default()
253 };
254
255 rustfs_madmin::BackendInfo {
256 backend_type: rustfs_madmin::BackendByte::Erasure,
257 online_disks: rustfs_madmin::BackendDisks::new(),
258 offline_disks: rustfs_madmin::BackendDisks::new(),
259 standard_sc_data: storage_class_info.standard_sc_data,
260 standard_sc_parities: storage_class_info.standard_sc_parities,
261 standard_sc_parity: storage_class_info.standard_sc_parity,
262 rr_sc_data: storage_class_info.rr_sc_data,
263 rr_sc_parities: storage_class_info.rr_sc_parities,
264 rr_sc_parity: storage_class_info.rr_sc_parity,
265 total_sets: total_sets.to_vec(),
266 drives_per_set: drives_per_set.to_vec(),
267 }
268}
269
270impl ECStore {
271 #[instrument(level = "debug", skip(self))]
272 pub(super) async fn delete_all(&self, bucket: &str, prefix: &str) -> Result<()> {
273 let mut futures = Vec::new();
274 for sets in self.pools.iter() {
275 for set in sets.disk_set.iter() {
276 futures.push(set.delete_all(bucket, prefix));
277 }
294 }
295 let results = join_all(futures).await;
296
297 let mut errs = Vec::new();
298
299 for res in results {
300 match res {
301 Ok(_) => errs.push(None),
302 Err(e) => errs.push(Some(e)),
303 }
304 }
305
306 debug!("store delete_all errs {:?}", errs);
307
308 Ok(())
309 }
310
311 pub(super) async fn delete_prefix(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
312 let dispatch_scope = if opts.tier_delete_journal_api.is_some() {
313 let incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
314 let authorization = opts
315 .tier_delete_dispatch_authorization
316 .as_ref()
317 .ok_or_else(|| Error::other("prefix mutation is missing its dispatched-journal authorization"))?;
318 authorization.ensure_current(bucket, incarnation, object)?;
319 Some((authorization, incarnation))
320 } else {
321 None
322 };
323 if opts.lifecycle_delete_all.is_some() {
324 let mut preflight_opts = opts.clone();
325 preflight_opts
326 .lifecycle_delete_all
327 .as_mut()
328 .ok_or(StorageError::PreconditionFailed)?
329 .phase = crate::object_api::LifecycleDeleteAllPhase::Preflight;
330 for pool in &self.pools {
331 if let Some((authorization, incarnation)) = dispatch_scope {
332 authorization.ensure_current(bucket, incarnation, object)?;
333 }
334 #[cfg(test)]
335 lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Preflight, pool.pool_idx)?;
336 pool.delete_object(bucket, object, preflight_opts.clone()).await?;
337 }
338
339 opts.lifecycle_delete_all_journal()
340 .ok_or(StorageError::PreconditionFailed)?
341 .lock()
342 .mark_mutation_started();
343 if let Some((authorization, incarnation)) = dispatch_scope {
344 authorization.mark_mutation_started(bucket, incarnation, object)?;
345 }
346 let mut non_trigger_opts = opts.clone();
347 non_trigger_opts
348 .lifecycle_delete_all
349 .as_mut()
350 .ok_or(StorageError::PreconditionFailed)?
351 .phase = crate::object_api::LifecycleDeleteAllPhase::History;
352 for pool in &self.pools {
353 if let Some((authorization, incarnation)) = dispatch_scope {
354 authorization.ensure_current(bucket, incarnation, object)?;
355 }
356 #[cfg(test)]
357 lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::History, pool.pool_idx)?;
358 let mut pool_opts = non_trigger_opts.clone();
359 pool_opts.delete_prefix = true;
360 pool.delete_object(bucket, object, pool_opts).await?;
361 }
362
363 let mut final_preflight_opts = opts.clone();
364 final_preflight_opts
365 .lifecycle_delete_all
366 .as_mut()
367 .ok_or(StorageError::PreconditionFailed)?
368 .phase = crate::object_api::LifecycleDeleteAllPhase::FinalPreflight;
369 let mut trigger_pools = Vec::new();
370 for (pool_index, pool) in self.pools.iter().enumerate() {
371 if let Some((authorization, incarnation)) = dispatch_scope {
372 authorization.ensure_current(bucket, incarnation, object)?;
373 }
374 #[cfg(test)]
375 lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::FinalPreflight, pool.pool_idx)?;
376 let result = pool.delete_object(bucket, object, final_preflight_opts.clone()).await?;
377 if !result.name.is_empty() {
378 trigger_pools.push(pool_index);
379 }
380 }
381 if trigger_pools.is_empty() {
382 return Err(StorageError::PreconditionFailed);
383 }
384
385 let mut trigger_opts = opts.clone();
386 trigger_opts
387 .lifecycle_delete_all
388 .as_mut()
389 .ok_or(StorageError::PreconditionFailed)?
390 .phase = crate::object_api::LifecycleDeleteAllPhase::Trigger;
391 for pool_index in trigger_pools {
392 if let Some((authorization, incarnation)) = dispatch_scope {
393 authorization.ensure_current(bucket, incarnation, object)?;
394 }
395 #[cfg(test)]
396 lifecycle_delete_all_test_failure(crate::object_api::LifecycleDeleteAllPhase::Trigger, pool_index)?;
397 let mut pool_opts = trigger_opts.clone();
398 pool_opts.delete_prefix = true;
399 self.pools[pool_index].delete_object(bucket, object, pool_opts).await?;
400 }
401 return Ok(());
402 }
403
404 let mut first_error = None;
405 let mut first_volume_error = None;
406 let mut has_success = false;
407 if let Some((authorization, incarnation)) = dispatch_scope {
408 authorization.mark_mutation_started(bucket, incarnation, object)?;
409 }
410 for pool in &self.pools {
411 if let Some((authorization, incarnation)) = dispatch_scope {
412 authorization.ensure_current(bucket, incarnation, object)?;
413 }
414 let mut opts = opts.clone();
415 opts.delete_prefix = true;
416 match pool.delete_object(bucket, object, opts).await {
417 Ok(_) => has_success = true,
418 Err(err) if is_err_strict_volume_not_found(&err) => {
419 if first_volume_error.is_none() {
420 first_volume_error = Some(err);
421 }
422 }
423 Err(err) => {
424 if first_error.is_none() {
425 first_error = Some(err);
426 }
427 }
428 }
429 }
430
431 match first_error {
432 Some(err) => Err(err),
433 None if has_success => Ok(()),
434 None => match first_volume_error {
435 Some(err) => Err(err),
436 None => Ok(()),
437 },
438 }
439 }
440
441 pub(super) async fn get_available_pool_idx(&self, bucket: &str, object: &str, size: i64) -> Option<usize> {
442 let mut server_pools = self.get_server_pools_available_space(bucket, object, size).await;
445 server_pools.filter_max_used(100 - (100_f64 * DISK_RESERVE_FRACTION) as u64);
446 let total = server_pools.total_available();
447
448 if total == 0 {
449 return None;
450 }
451
452 let mut rng = rand::rng();
453 let random_u64: u64 = rng.random_range(0..total);
454
455 let choose = random_u64 % total;
456 let mut at_total = 0;
457
458 for pool in server_pools.iter() {
459 at_total += pool.available;
460 if at_total > choose && pool.available > 0 {
461 return Some(pool.index);
462 }
463 }
464
465 None
466 }
467
468 pub(super) async fn get_available_pool_idx_excluding(
469 &self,
470 bucket: &str,
471 object: &str,
472 size: i64,
473 excluded_pool_idx: usize,
474 ) -> Option<usize> {
475 let mut server_pools = self.get_server_pools_available_space(bucket, object, size).await;
476 server_pools.filter_max_used(100 - (100_f64 * DISK_RESERVE_FRACTION) as u64);
477
478 if let Some(pool) = server_pools.0.get_mut(excluded_pool_idx) {
479 pool.available = 0;
480 }
481
482 let total = server_pools.total_available();
483 if total == 0 {
484 return None;
485 }
486
487 let mut rng = rand::rng();
488 let random_u64: u64 = rng.random_range(0..total);
489
490 let choose = random_u64 % total;
491 let mut at_total = 0;
492
493 for pool in server_pools.iter() {
494 at_total += pool.available;
495 if at_total > choose && pool.available > 0 {
496 return Some(pool.index);
497 }
498 }
499
500 None
501 }
502
503 async fn get_server_pools_available_space(&self, bucket: &str, object: &str, size: i64) -> ServerPoolsAvailableSpace {
504 let mut n_sets = vec![0; self.pools.len()];
505 let mut infos = vec![Vec::new(); self.pools.len()];
506 let pool_inputs = join_all(self.pools.iter().enumerate().map(|(idx, pool)| async move {
507 if self.is_suspended(idx).await || self.is_pool_rebalancing(idx).await {
508 return (idx, 0, Vec::new());
509 }
510
511 let disks = pool.get_disks_by_key(object).disk_inventory().await;
512 let disk_infos = get_disk_infos(&disks).await;
513
514 (idx, pool.set_count, disk_infos)
515 }))
516 .await;
517
518 for (idx, set_count, disk_infos) in pool_inputs {
519 n_sets[idx] = set_count;
520 infos[idx] = disk_infos;
521 }
522
523 build_server_pools_available_space(bucket, size, &n_sets, &infos).await
524 }
525
526 pub(super) async fn is_suspended(&self, idx: usize) -> bool {
527 let pool_meta = self.pool_meta.read().await;
530
531 pool_meta.is_suspended(idx)
532 }
533
534 pub(super) async fn get_pool_idx(&self, bucket: &str, object: &str, size: i64) -> Result<usize> {
535 let idx = match self
536 .get_pool_idx_existing_with_opts(
537 bucket,
538 object,
539 &ObjectOptions {
540 skip_decommissioned: true,
541 skip_rebalancing: true,
542 ..Default::default()
543 },
544 )
545 .await
546 {
547 Ok(res) => res,
548 Err(err) => {
549 if !is_err_object_not_found(&err) {
550 return Err(err);
551 }
552
553 if let Some(hit_idx) = self.get_available_pool_idx(bucket, object, size).await {
554 hit_idx
555 } else {
556 return Err(Error::DiskFull);
557 }
558 }
559 };
560
561 Ok(idx)
562 }
563
564 pub(super) async fn get_pool_idx_no_lock(&self, bucket: &str, object: &str, size: i64) -> Result<usize> {
565 let idx = match self.get_pool_idx_existing_no_lock(bucket, object).await {
566 Ok(res) => res,
567 Err(err) => {
568 if !is_err_object_not_found(&err) {
569 return Err(err);
570 }
571
572 if let Some(idx) = self.get_available_pool_idx(bucket, object, size).await {
573 idx
574 } else {
575 warn!("get_pool_idx_no_lock: disk full {}/{}", bucket, object);
576 return Err(Error::DiskFull);
577 }
578 }
579 };
580
581 Ok(idx)
582 }
583
584 async fn get_pool_idx_existing_no_lock(&self, bucket: &str, object: &str) -> Result<usize> {
585 self.get_pool_idx_existing_with_opts(
586 bucket,
587 object,
588 &ObjectOptions {
589 no_lock: true,
590 skip_decommissioned: true,
591 skip_rebalancing: true,
592 ..Default::default()
593 },
594 )
595 .await
596 }
597
598 pub(super) async fn get_pool_idx_existing_with_opts(
599 &self,
600 bucket: &str,
601 object: &str,
602 opts: &ObjectOptions,
603 ) -> Result<usize> {
604 let (pinfo, _) = self.get_pool_info_existing_with_opts(bucket, object, opts).await?;
605 Ok(pinfo.index)
606 }
607
608 pub(super) async fn get_pool_info_existing_with_opts(
609 &self,
610 bucket: &str,
611 object: &str,
612 opts: &ObjectOptions,
613 ) -> Result<(PoolObjInfo, Vec<PoolErr>)> {
614 self.internal_get_pool_info_existing_with_opts(bucket, object, opts, false)
615 .await
616 }
617
618 pub(super) async fn get_pool_info_for_delete_marker(
619 &self,
620 bucket: &str,
621 object: &str,
622 opts: &ObjectOptions,
623 ) -> Result<(PoolObjInfo, Vec<PoolErr>)> {
624 self.internal_get_pool_info_existing_with_opts(bucket, object, opts, true)
625 .await
626 }
627
628 async fn internal_get_pool_info_existing_with_opts(
629 &self,
630 bucket: &str,
631 object: &str,
632 opts: &ObjectOptions,
633 require_all_pool_reads: bool,
634 ) -> Result<(PoolObjInfo, Vec<PoolErr>)> {
635 let mut futures = Vec::new();
636 for pool in self.pools.iter() {
637 let mut pool_opts = opts.clone();
638 if !pool_opts.metadata_chg {
639 pool_opts.version_id = None;
640 }
641
642 futures.push(async move { pool.get_object_info(bucket, object, &pool_opts).await });
643 }
644
645 let results = join_all(futures).await;
646
647 let mut ress = Vec::new();
648
649 for (i, res) in results.into_iter().enumerate() {
651 let index = i;
652
653 match res {
654 Ok(r) => {
655 ress.push(PoolObjInfo {
656 index,
657 object_info: r,
658 err: None,
659 });
660 }
661 Err(e) => {
662 if require_all_pool_reads && !is_err_object_not_found(&e) && !is_err_version_not_found(&e) {
666 return Err(match e {
667 Error::ErasureReadQuorum | Error::InsufficientReadQuorum(_, _) => Error::ErasureWriteQuorum,
668 err => err,
669 });
670 }
671 ress.push(PoolObjInfo {
672 index,
673 err: Some(e),
674 ..Default::default()
675 });
676 }
677 }
678 }
679
680 if require_all_pool_reads {
681 let suspended_pools = {
682 let pool_meta = self.pool_meta.read().await;
683 (0..self.pools.len())
684 .map(|idx| pool_meta.is_suspended(idx))
685 .collect::<Vec<_>>()
686 };
687 let candidates = ress
688 .iter()
689 .map(|pinfo| LatestObjectInfoCandidate {
690 info: pinfo.err.is_none().then(|| pinfo.object_info.clone()),
691 idx: pinfo.index,
692 err: pinfo.err.clone(),
693 })
694 .collect();
695 let (object_info, index) =
696 resolve_latest_object_info_candidates_with_pool_state(candidates, &suspended_pools, bucket, object, opts)?;
697 let pools_with_object = self.pools_with_object(&ress, opts).await;
698 return Ok((
699 PoolObjInfo {
700 index,
701 object_info,
702 err: None,
703 },
704 pools_with_object,
705 ));
706 }
707
708 ress.sort_by(|a, b| {
709 let at = a.object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH);
710 let bt = b.object_info.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH);
711
712 bt.cmp(&at)
713 });
714
715 let mut def_pool = PoolObjInfo::default();
716 let mut has_def_pool = false;
717
718 for pinfo in ress.iter() {
719 if opts.skip_decommissioned && self.is_suspended(pinfo.index).await {
720 continue;
721 }
722
723 if opts.skip_rebalancing && self.is_pool_rebalancing(pinfo.index).await {
724 continue;
725 }
726
727 if pinfo.err.is_none() {
728 return Ok((pinfo.clone(), self.pools_with_object(&ress, opts).await));
729 }
730
731 let err = pinfo.err.as_ref().expect("operation should succeed");
732
733 if err == &Error::ErasureReadQuorum && !opts.metadata_chg {
734 return Ok((pinfo.clone(), self.pools_with_object(&ress, opts).await));
735 }
736
737 def_pool = pinfo.clone();
738 has_def_pool = true;
739 if is_err_object_not_found(err)
741 && let Err(err) = opts.precondition_check(&pinfo.object_info)
742 {
743 return Err(err);
744 }
745
746 if !is_err_object_not_found(err) && !is_err_version_not_found(err) {
747 return Err(err.clone());
748 }
749
750 if pinfo.object_info.delete_marker && !pinfo.object_info.name.is_empty() {
751 return Ok((pinfo.clone(), Vec::new()));
752 }
753 }
754
755 if opts.replication_request && opts.delete_marker && has_def_pool {
756 return Ok((def_pool, Vec::new()));
757 }
758
759 Err(pool_lookup_not_found_error(bucket, object, opts))
760 }
761
762 async fn pools_with_object(&self, pools: &[PoolObjInfo], opts: &ObjectOptions) -> Vec<PoolErr> {
763 let mut errs = Vec::new();
764
765 for pool in pools.iter() {
766 if opts.skip_decommissioned && self.is_suspended(pool.index).await {
767 continue;
768 }
769
770 if opts.skip_rebalancing && self.is_pool_rebalancing(pool.index).await {
771 continue;
772 }
773
774 if let Some(err) = &pool.err {
775 if err == &Error::ErasureReadQuorum {
776 errs.push(PoolErr {
777 index: Some(pool.index),
778 err: Some(Error::ErasureReadQuorum),
779 });
780 }
781 } else {
782 errs.push(PoolErr {
783 index: Some(pool.index),
784 err: None,
785 });
786 }
787 }
788 errs
789 }
790
791 pub(super) async fn get_latest_object_info_with_idx(
792 &self,
793 bucket: &str,
794 object: &str,
795 opts: &ObjectOptions,
796 ) -> Result<(ObjectInfo, usize)> {
797 let suspended_pools = if opts.skip_decommissioned {
798 let pool_meta = self.pool_meta.read().await;
799 Some(
800 (0..self.pools.len())
801 .map(|idx| pool_meta.is_suspended(idx))
802 .collect::<Vec<_>>(),
803 )
804 } else {
805 None
806 };
807 let mut futures = Vec::with_capacity(self.pools.len());
808 for (idx, pool) in self.pools.iter().enumerate() {
809 if suspended_pools.as_ref().is_some_and(|pools| pools[idx]) {
810 continue;
811 }
812
813 if opts.skip_rebalancing && self.is_pool_rebalancing(idx).await {
814 continue;
815 }
816
817 futures.push(async move { (idx, pool.get_object_info(bucket, object, opts).await) });
818 }
819
820 let results = join_all(futures).await;
821 let mut candidates = Vec::with_capacity(self.pools.len());
822
823 for (idx, result) in results {
824 match result {
825 Ok(res) => {
826 candidates.push(LatestObjectInfoCandidate {
827 info: Some(res),
828 idx,
829 err: None,
830 });
831 }
832 Err(e) => {
833 candidates.push(LatestObjectInfoCandidate {
834 info: None,
835 idx,
836 err: Some(e),
837 });
838 }
839 }
840 }
841
842 let suspended_pools = match suspended_pools {
843 Some(pools) => pools,
844 None => {
845 let pool_meta = self.pool_meta.read().await;
846 (0..self.pools.len())
847 .map(|idx| pool_meta.is_suspended(idx))
848 .collect::<Vec<_>>()
849 }
850 };
851
852 resolve_latest_object_info_candidates_with_pool_state(candidates, &suspended_pools, bucket, object, opts)
856 }
857
858 pub(super) async fn prepare_latest_object_metadata_with_idx(
859 &self,
860 bucket: &str,
861 object: &str,
862 opts: &ObjectOptions,
863 ) -> Result<(crate::set_disk::PreparedGetObjectMetadata, usize)> {
864 let suspended_pools = if opts.skip_decommissioned {
865 let pool_meta = self.pool_meta.read().await;
866 Some(
867 (0..self.pools.len())
868 .map(|idx| pool_meta.is_suspended(idx))
869 .collect::<Vec<_>>(),
870 )
871 } else {
872 None
873 };
874 let mut futures = FuturesUnordered::new();
875 for (idx, pool) in self.pools.iter().enumerate() {
876 if suspended_pools.as_ref().is_some_and(|pools| pools[idx]) {
877 continue;
878 }
879
880 if opts.skip_rebalancing && self.is_pool_rebalancing(idx).await {
881 continue;
882 }
883
884 futures.push(async move {
885 let result = pool
886 .prepare_get_object_reader_metadata(bucket, object, opts)
887 .await
888 .map_err(|err| to_object_err(err, vec![bucket, object]));
889 (idx, result)
890 });
891 }
892
893 let mut candidates = (0..self.pools.len()).map(|_| None).collect::<Vec<_>>();
894 let mut latest_prepared = None;
898 let mut latest_mod_time = None;
899 let mut provisional_dynamic_pool_state = None;
900 while let Some((idx, result)) = futures.next().await {
901 match result {
902 Ok(metadata) => {
903 let mod_time = metadata.object_info().mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH);
904 let info = metadata.object_info().clone();
905 let retain = match (latest_mod_time, latest_prepared.as_ref()) {
906 (None, _) => true,
907 (Some(current), _) if mod_time > current => true,
908 (Some(current), _) if mod_time < current => false,
909 (Some(_), Some((current_idx, _))) => {
910 if suspended_pools.is_none() && provisional_dynamic_pool_state.is_none() {
911 let pool_meta = self.pool_meta.read().await;
912 provisional_dynamic_pool_state = Some(
913 (0..self.pools.len())
914 .map(|pool_idx| pool_meta.is_suspended(pool_idx))
915 .collect::<Vec<_>>(),
916 );
917 }
918 let provisional_pool_state = suspended_pools
919 .as_ref()
920 .or(provisional_dynamic_pool_state.as_ref())
921 .ok_or_else(|| Error::other("GET pool state snapshot is unavailable"))?;
922 let new_key = (provisional_pool_state.get(idx).copied().unwrap_or(false), std::cmp::Reverse(idx));
923 let current_key = (
924 provisional_pool_state.get(*current_idx).copied().unwrap_or(false),
925 std::cmp::Reverse(*current_idx),
926 );
927 new_key < current_key
928 }
929 (Some(_), None) => true,
930 };
931 if retain {
932 if latest_mod_time.is_none_or(|current| mod_time > current) {
933 latest_mod_time = Some(mod_time);
934 }
935 latest_prepared = Some((idx, metadata));
936 }
937 candidates[idx] = Some(LatestObjectInfoCandidate {
938 info: Some(info),
939 idx,
940 err: None,
941 });
942 }
943 Err(err) => {
944 candidates[idx] = Some(LatestObjectInfoCandidate {
945 info: None,
946 idx,
947 err: Some(err),
948 });
949 }
950 }
951 }
952
953 #[cfg(test)]
954 pause_prepared_pool_read_after_fanout(object).await;
955
956 let suspended_pools = match suspended_pools {
957 Some(pools) => pools,
958 None => {
959 let pool_meta = self.pool_meta.read().await;
960 (0..self.pools.len())
961 .map(|idx| pool_meta.is_suspended(idx))
962 .collect::<Vec<_>>()
963 }
964 };
965
966 let candidates = candidates.into_iter().flatten().collect();
967 let (winner_info, winner_idx) =
968 resolve_latest_object_info_candidates_with_pool_state(candidates, &suspended_pools, bucket, object, opts)?;
969 if let Some((prepared_idx, metadata)) = latest_prepared
970 && prepared_idx == winner_idx
971 {
972 return Ok((metadata, winner_idx));
973 }
974
975 let pool = self.pools.get(winner_idx).ok_or(Error::ErasureReadQuorum)?;
976 #[cfg(test)]
977 pause_prepared_pool_read_before_refetch(object).await;
978 let metadata = pool
979 .prepare_get_object_reader_metadata(bucket, object, opts)
980 .await
981 .map_err(|err| to_object_err(err, vec![bucket, object]))?;
982 validate_prepared_pool_refetch_identity(&winner_info, metadata.object_info())?;
983 Ok((metadata, winner_idx))
984 }
985
986 pub(super) async fn delete_object_from_all_pools(
987 &self,
988 bucket: &str,
989 object: &str,
990 opts: &ObjectOptions,
991 exact: &ObjectInfo,
992 errs: Vec<PoolErr>,
993 ) -> Result<ObjectInfo> {
994 self.reconcile_decommission_capacity_before_exact_delete(bucket, object, opts, exact)
995 .await?;
996
997 let mut results = Vec::with_capacity(errs.len());
998
999 for pe in errs.iter() {
1000 if let Some(err) = &pe.err
1001 && err == &StorageError::ErasureWriteQuorum
1002 {
1003 if let Some(idx) = pe.index {
1004 results.push(RebalanceDeletePoolResult {
1005 pool_idx: idx,
1006 result: Err(StorageError::ErasureWriteQuorum),
1007 });
1008 }
1009 continue;
1010 }
1011
1012 if let Some(idx) = pe.index {
1013 let pool = self.pools[idx].clone();
1014 results.push(RebalanceDeletePoolResult {
1015 pool_idx: idx,
1016 result: self
1017 .run_external_decommission_capacity_object_delete(
1018 idx,
1019 bucket,
1020 object,
1021 object,
1022 opts.clone(),
1023 |opts| async move { pool.delete_object(bucket, object, opts).await },
1024 )
1025 .await,
1026 });
1027 }
1028 }
1029
1030 resolve_rebalance_delete_from_all_pools_result(
1031 resolve_rebalance_delete_from_all_pools_results(results, bucket, object),
1032 bucket,
1033 object,
1034 )
1035 }
1036
1037 pub async fn reload_pool_meta(&self) -> Result<bool> {
1042 let movement_gate = self.ctx.data_movement_operation_gate();
1046 let _movement_guard = movement_gate.write().await;
1047 let reloaded = resolve_store_rebalance_pool_meta_reload_result(
1048 self.load_runtime_pool_meta("store rebalance pool meta reload failed").await,
1049 "reload_pool_meta",
1050 )?;
1051
1052 let active_workers = {
1055 let cancelers = self.decommission_cancelers.read().await;
1056 cancelers
1057 .iter()
1058 .map(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
1059 .collect::<Vec<_>>()
1060 };
1061
1062 let incoming_has_pools = !reloaded.pools.is_empty();
1063 let mut pool_meta = self.pool_meta.write().await;
1064 let movement_before = pool_meta.clone();
1065 let merged_newer = merge_pool_status_refresh(&mut pool_meta, reloaded, &active_workers);
1066 if crate::core::pools::pool_meta_movement_snapshot_changed(&movement_before, &pool_meta) {
1067 self.ctx.advance_data_movement_operation_epoch();
1068 }
1069
1070 if !merged_newer && !incoming_has_pools {
1071 warn!(
1072 event = EVENT_POOL_META_RELOAD,
1073 component = LOG_COMPONENT_ECSTORE,
1074 subsystem = LOG_SUBSYSTEM_POOLS,
1075 result = "ignored",
1076 reason = "missing_metadata",
1077 "Peer pool meta reload ignored because persisted metadata is missing"
1078 );
1079 } else if !merged_newer {
1080 debug!(
1081 event = EVENT_POOL_META_RELOAD,
1082 component = LOG_COMPONENT_ECSTORE,
1083 subsystem = LOG_SUBSYSTEM_POOLS,
1084 result = "ignored",
1085 reason = "stale_snapshot",
1086 "Peer pool meta reload ignored as a stale snapshot"
1087 );
1088 }
1089
1090 Ok(merged_newer)
1091 }
1092
1093 pub(crate) fn deduplicate_disks(disks: Vec<rustfs_madmin::Disk>) -> Vec<rustfs_madmin::Disk> {
1102 use std::collections::HashMap;
1103 use std::collections::hash_map::Entry;
1104
1105 let mut unique_disks: HashMap<String, rustfs_madmin::Disk> = HashMap::new();
1106 let mut duplicate_count = 0;
1107
1108 for disk in disks {
1109 let key = format!(
1110 "{}|{}|p{}s{}d{}",
1111 disk.endpoint, disk.drive_path, disk.pool_index, disk.set_index, disk.disk_index
1112 );
1113
1114 match unique_disks.entry(key) {
1115 Entry::Vacant(entry) => {
1116 entry.insert(disk);
1117 }
1118 Entry::Occupied(_) => {
1119 duplicate_count += 1;
1120 }
1121 }
1122 }
1123
1124 if duplicate_count > 0 {
1125 debug!("Deduplicated {} duplicate disk entries", duplicate_count);
1126 }
1127
1128 unique_disks.into_values().collect()
1129 }
1130
1131 #[instrument(level = "trace", skip(self))]
1132 pub(super) async fn handle_new_ns_lock(&self, bucket: &str, object: &str) -> Result<NamespaceLockWrapper> {
1133 self.pools[0].new_ns_lock(bucket, object).await
1134 }
1135
1136 #[instrument(skip(self))]
1137 pub(super) async fn handle_backend_info(&self) -> rustfs_madmin::BackendInfo {
1138 let drives_per_set = StorageAdminApi::set_drive_counts(self);
1139 let default_standard_parities = self.pools.iter().map(|pool| pool.default_parity_count).collect::<Vec<_>>();
1140 let storage_class = runtime_sources::storage_class_config_snapshot();
1141 let total_sets = self.pools.iter().map(|pool| pool.set_count).collect::<Vec<_>>();
1142
1143 build_backend_info(&storage_class, &drives_per_set, &default_standard_parities, &total_sets)
1144 }
1145
1146 #[instrument(skip(self))]
1147 pub(super) async fn handle_storage_info(&self) -> rustfs_madmin::StorageInfo {
1148 let Some(notification_sy) = runtime_sources::notification_sys() else {
1149 return rustfs_madmin::StorageInfo::default();
1150 };
1151
1152 let mut info = notification_sy.storage_info(self).await;
1153
1154 let original_count = info.disks.len();
1156 info.disks = Self::deduplicate_disks(info.disks);
1157 let final_count = info.disks.len();
1158
1159 if original_count != final_count {
1160 warn!(
1161 "Storage info deduplication: removed {} duplicate disk entries ({} -> {})",
1162 original_count - final_count,
1163 original_count,
1164 final_count
1165 );
1166 }
1167
1168 info
1169 }
1170
1171 #[instrument(skip(self))]
1172 pub(super) async fn handle_local_storage_info(&self) -> rustfs_madmin::StorageInfo {
1173 let mut futures = Vec::with_capacity(self.pools.len());
1174
1175 for pool in self.pools.iter() {
1176 futures.push(pool.local_storage_info_snapshot())
1177 }
1178
1179 let results = join_all(futures).await;
1180
1181 let mut disks = Vec::new();
1182
1183 for res in results.into_iter() {
1184 disks.extend_from_slice(&res.disks);
1185 }
1186
1187 let original_count = disks.len();
1191 disks = Self::deduplicate_disks(disks);
1192
1193 if original_count != disks.len() {
1194 warn!("Local storage info deduplication: {} -> {}", original_count, disks.len());
1195 }
1196
1197 let backend = StorageAdminApi::backend_info(self).await;
1198 rustfs_madmin::StorageInfo {
1199 backend,
1200 disks,
1201 ..Default::default()
1202 }
1203 }
1204
1205 #[instrument(skip(self))]
1206 pub(super) async fn handle_get_disks(&self, pool_idx: usize, set_idx: usize) -> Result<Vec<Option<DiskStore>>> {
1207 if pool_idx < self.pools.len() && set_idx < self.pools[pool_idx].disk_set.len() {
1208 Ok(self.pools[pool_idx].disk_set[set_idx].disk_inventory().await)
1209 } else {
1210 Err(rebalance_disk_set_lookup_error(pool_idx, set_idx, self.pools.len()))
1211 }
1212 }
1213
1214 #[instrument(skip(self))]
1215 pub(super) fn handle_set_drive_counts(&self) -> Vec<usize> {
1216 let mut counts = vec![0; self.pools.len()];
1217
1218 for (i, pool) in self.pools.iter().enumerate() {
1219 counts[i] = pool.set_drive_count();
1220 }
1221 counts
1222 }
1223
1224 #[instrument(skip(self))]
1225 pub(super) async fn handle_get_pool_and_set(&self, id: &str) -> Result<(Option<usize>, Option<usize>, Option<usize>)> {
1226 for (pool_idx, pool) in self.pools.iter().enumerate() {
1227 for (set_idx, set) in pool.format.erasure.sets.iter().enumerate() {
1228 for (disk_idx, disk_id) in set.iter().enumerate() {
1229 if disk_id.to_string() == id {
1230 return Ok((Some(pool_idx), Some(set_idx), Some(disk_idx)));
1231 }
1232 }
1233 }
1234 }
1235
1236 Err(Error::DiskNotFound)
1237 }
1238}
1239
1240#[cfg(test)]
1241static LIFECYCLE_DELETE_ALL_TEST_FAILURE: std::sync::Mutex<Option<(crate::object_api::LifecycleDeleteAllPhase, usize)>> =
1242 std::sync::Mutex::new(None);
1243
1244#[cfg(test)]
1245fn lifecycle_delete_all_test_failure(phase: crate::object_api::LifecycleDeleteAllPhase, pool_index: usize) -> Result<()> {
1246 if LIFECYCLE_DELETE_ALL_TEST_FAILURE
1247 .lock()
1248 .expect("lifecycle delete-all failure hook should not poison")
1249 .is_some_and(|failure| failure == (phase, pool_index))
1250 {
1251 return Err(StorageError::PreconditionFailed);
1252 }
1253 Ok(())
1254}
1255
1256#[cfg(test)]
1257mod tests {
1258 use super::*;
1259 use crate::bucket::replication::{ReplicationStatusType, VersionPurgeStatusType};
1260 use crate::config::com::{read_config_no_lock_preserve_empty_with_metadata, save_config};
1261 use crate::config::storageclass::{CLASS_RRS, CLASS_STANDARD, lookup_config_for_pools_without_env};
1262 use crate::core::pools::{
1263 POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_VERSION, PoolDecommissionInfo, PoolStatus,
1264 initialized_pool_meta_identity_for_test, pool_meta_v3_commit_state_for_test,
1265 };
1266 use crate::core::sets::Sets;
1267 use crate::disk::error::DiskError;
1268 use crate::layout::endpoint::Endpoint;
1269 use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
1270 use crate::object_api::ObjectLockConfigSnapshot;
1271 use crate::set_disk::get_lock_acquire_timeout;
1272 use crate::storage_api_contracts::bucket::MakeBucketOptions;
1273 use crate::storage_api_contracts::object::ObjectIO as _;
1274 use arc_swap::ArcSwap;
1275 use rustfs_config::server_config::KVS;
1276 use rustfs_filemeta::FileInfo;
1277 use std::sync::Arc;
1278 use time::{Duration as TimeDuration, OffsetDateTime};
1279 use tokio_util::sync::CancellationToken;
1280
1281 async fn setup_multi_pool_test_store(
1282 name: &str,
1283 drives_per_pool: &[usize],
1284 ) -> (tempfile::TempDir, Arc<ECStore>, CancellationToken) {
1285 let temp_dir = tempfile::tempdir().expect("multi-pool test directory should be created");
1286 let mut pools = Vec::with_capacity(drives_per_pool.len());
1287 for (pool_index, drives_per_set) in drives_per_pool.iter().copied().enumerate() {
1288 let mut endpoints = Vec::with_capacity(drives_per_set);
1289 for disk_index in 0..drives_per_set {
1290 let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
1291 tokio::fs::create_dir_all(&disk_path)
1292 .await
1293 .expect("multi-pool test disk should be created");
1294 let mut endpoint =
1295 Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
1296 endpoint.set_pool_index(pool_index);
1297 endpoint.set_set_index(0);
1298 endpoint.set_disk_index(disk_index);
1299 endpoints.push(endpoint);
1300 }
1301 pools.push(PoolEndpoints {
1302 legacy: false,
1303 set_count: 1,
1304 drives_per_set,
1305 endpoints: Endpoints::from(endpoints),
1306 cmd_line: format!("{name}-pool-{pool_index}"),
1307 platform: "test".to_string(),
1308 });
1309 }
1310
1311 let endpoint_pools = EndpointServerPools(pools);
1312 let instance_ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
1313 crate::store::init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
1314 .await
1315 .expect("multi-pool local disks should initialize");
1316 let shutdown = CancellationToken::new();
1317 let store = ECStore::new_with_instance_ctx(
1318 "127.0.0.1:0".parse().expect("test address should parse"),
1319 endpoint_pools,
1320 shutdown.clone(),
1321 instance_ctx,
1322 )
1323 .await
1324 .expect("multi-pool store should initialize");
1325 crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
1326 (temp_dir, store, shutdown)
1327 }
1328
1329 struct LifecycleDeleteAllFailureGuard;
1330
1331 impl Drop for LifecycleDeleteAllFailureGuard {
1332 fn drop(&mut self) {
1333 *LIFECYCLE_DELETE_ALL_TEST_FAILURE
1334 .lock()
1335 .expect("lifecycle delete-all failure hook should not poison") = None;
1336 }
1337 }
1338
1339 async fn seed_multi_pool_delete_all(store: &Arc<ECStore>, bucket: &str, object: &str) -> ObjectOptions {
1340 let trigger_id = Uuid::new_v4();
1341 for (pool_index, pool) in store.pools.iter().enumerate() {
1342 let mut history_reader = PutObjReader::from_vec(format!("{object}-history-{pool_index}").into_bytes());
1343 pool.put_object(
1344 bucket,
1345 object,
1346 &mut history_reader,
1347 &ObjectOptions {
1348 versioned: true,
1349 version_id: Some(Uuid::new_v4().to_string()),
1350 mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(1)),
1351 ..Default::default()
1352 },
1353 )
1354 .await
1355 .expect("history should be stored");
1356 let mut trigger_reader = PutObjReader::from_vec(format!("{object}-trigger-{pool_index}").into_bytes());
1357 pool.put_object(
1358 bucket,
1359 object,
1360 &mut trigger_reader,
1361 &ObjectOptions {
1362 versioned: true,
1363 version_id: Some(trigger_id.to_string()),
1364 mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::seconds(2)),
1365 ..Default::default()
1366 },
1367 )
1368 .await
1369 .expect("shared trigger should be stored");
1370 }
1371 let mut opts = ObjectOptions {
1372 delete_prefix: true,
1373 delete_prefix_object: true,
1374 versioned: true,
1375 lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
1376 version_id: Some(trigger_id),
1377 delete_marker: false,
1378 action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
1379 rule_id: "rule".to_string(),
1380 phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
1381 }),
1382 object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
1383 crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
1384 ))),
1385 delete_replication_config_snapshot: Some(Arc::new(
1386 crate::bucket::replication::DeleteReplicationConfigSnapshot::default(),
1387 )),
1388 ..Default::default()
1389 };
1390 opts.ensure_lifecycle_delete_all_journal();
1391 opts
1392 }
1393
1394 async fn ordinary_version_count(store: &ECStore, pool_index: usize, bucket: &str, object: &str) -> usize {
1395 store.pools[pool_index].disk_set[0]
1396 .load_file_info_versions_exact(bucket, object)
1397 .await
1398 .expect("pool metadata should load")
1399 .map(|versions| {
1400 versions
1401 .versions
1402 .iter()
1403 .filter(|version| !version.tier_free_version())
1404 .count()
1405 })
1406 .unwrap_or_default()
1407 }
1408
1409 #[tokio::test]
1410 async fn delete_prefix_attempts_later_pools_after_an_earlier_pool_error() {
1411 let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("delete-prefix", &[2, 4]).await;
1412 let bucket = format!("delete-prefix-{}", Uuid::new_v4().simple());
1413 store
1414 .make_bucket(&bucket, &MakeBucketOptions::default())
1415 .await
1416 .expect("bucket should be created in both pools");
1417
1418 let first_pool_disks = store.pools[0].disk_set[0].disks.read().await.clone();
1419 for disk in first_pool_disks.iter().flatten() {
1420 disk.write_all(&bucket, "blocked", bytes::Bytes::from_static(b"not-a-directory"))
1421 .await
1422 .expect("first pool should contain a blocking parent file");
1423 }
1424 let later_pool_disks = store.pools[1].disk_set[0].disks.read().await.clone();
1425 let later_data_disk = later_pool_disks[0].clone().expect("later pool should have its first disk");
1426 later_data_disk
1427 .write_all(&bucket, "blocked/prefix/object", bytes::Bytes::from_static(b"data"))
1428 .await
1429 .expect("later pool should contain the prefix on its available disk");
1430 *store.pools[1].disk_set[0].disks.write().await = vec![Some(later_data_disk.clone()), None, None, None];
1431
1432 let err = store
1433 .delete_prefix(&bucket, "blocked/prefix", &ObjectOptions::default())
1434 .await
1435 .expect_err("the first pool's hard error must be returned");
1436
1437 assert!(
1438 matches!(err, StorageError::PrefixAccessDenied(ref error_bucket, ref error_prefix)
1439 if error_bucket == &bucket && error_prefix == "blocked/prefix"),
1440 "unexpected multi-pool delete error: {err:?}"
1441 );
1442 assert!(matches!(
1443 later_data_disk.read_all(&bucket, "blocked/prefix/object").await,
1444 Err(DiskError::FileNotFound)
1445 ));
1446
1447 *store.pools[1].disk_set[0].disks.write().await = later_pool_disks.clone();
1448 for disk in first_pool_disks.iter().flatten() {
1449 disk.write_all(&bucket, "second-blocked", bytes::Bytes::from_static(b"not-a-directory"))
1450 .await
1451 .expect("first pool should contain a second blocking parent file");
1452 }
1453 for disk in later_pool_disks.iter().flatten() {
1454 disk.write_all(&bucket, "second-blocked/prefix/object", bytes::Bytes::from_static(b"data"))
1455 .await
1456 .expect("later pool should contain the second prefix");
1457 }
1458 let err = store
1459 .delete_prefix(&bucket, "second-blocked/prefix", &ObjectOptions::default())
1460 .await
1461 .expect_err("a successful later pool must not override the first pool's hard error");
1462 assert!(
1463 matches!(err, StorageError::PrefixAccessDenied(ref error_bucket, ref error_prefix)
1464 if error_bucket == &bucket && error_prefix == "second-blocked/prefix"),
1465 "unexpected hard-error plus success result: {err:?}"
1466 );
1467 for disk in later_pool_disks.iter().flatten() {
1468 assert!(matches!(
1469 disk.read_all(&bucket, "second-blocked/prefix/object").await,
1470 Err(DiskError::FileNotFound)
1471 ));
1472 }
1473
1474 for disk in later_pool_disks.iter().flatten() {
1475 disk.delete_volume(&bucket, true)
1476 .await
1477 .expect("the bucket should be absent from the later pool");
1478 }
1479 let healthy_object = "healthy/prefix/object";
1480 for disk in first_pool_disks.iter().flatten() {
1481 disk.write_all(&bucket, healthy_object, bytes::Bytes::from_static(b"data"))
1482 .await
1483 .expect("the first pool should contain the healthy prefix");
1484 }
1485 store
1486 .delete_prefix(&bucket, "healthy/prefix", &ObjectOptions::default())
1487 .await
1488 .expect("one successful pool should make a partially missing bucket idempotent");
1489 for disk in first_pool_disks.iter().flatten() {
1490 assert!(matches!(disk.read_all(&bucket, healthy_object).await, Err(DiskError::FileNotFound)));
1491 }
1492
1493 let missing_bucket = format!("delete-prefix-missing-{}", Uuid::new_v4().simple());
1494 let err = store
1495 .delete_prefix(&missing_bucket, "missing/prefix", &ObjectOptions::default())
1496 .await
1497 .expect_err("a bucket missing from every pool must remain an error");
1498 assert_eq!(err, StorageError::BucketNotFound(missing_bucket));
1499
1500 shutdown.cancel();
1501 }
1502
1503 #[tokio::test]
1504 #[serial_test::serial]
1505 async fn lifecycle_delete_all_history_failure_preserves_trigger_and_retry_converges() {
1506 let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("lifecycle-delete-all", &[4, 4]).await;
1507 let bucket = format!("lifecycle-delete-all-{}", Uuid::new_v4().simple());
1508 let object = "object";
1509 store
1510 .make_bucket(&bucket, &MakeBucketOptions::default())
1511 .await
1512 .expect("bucket should be created in both pools");
1513
1514 for pool_index in 0..2 {
1515 let mut reader = PutObjReader::from_vec(format!("pool-{pool_index}-history").into_bytes());
1516 store.pools[pool_index]
1517 .put_object(
1518 &bucket,
1519 object,
1520 &mut reader,
1521 &ObjectOptions {
1522 versioned: true,
1523 ..Default::default()
1524 },
1525 )
1526 .await
1527 .expect("historical version should be stored");
1528 }
1529 let marker = store.pools[0]
1530 .delete_object(
1531 &bucket,
1532 object,
1533 ObjectOptions {
1534 versioned: true,
1535 ..Default::default()
1536 },
1537 )
1538 .await
1539 .expect("trigger marker should be stored in the first pool");
1540 let marker_id = marker.version_id.expect("trigger marker should have a version id");
1541 let mut opts = ObjectOptions {
1542 delete_prefix: true,
1543 delete_prefix_object: true,
1544 versioned: true,
1545 lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
1546 version_id: Some(marker_id),
1547 delete_marker: true,
1548 action: rustfs_scanner_metrics::metrics::IlmAction::DelMarkerDeleteAllVersionsAction,
1549 rule_id: "rule".to_string(),
1550 phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
1551 }),
1552 object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
1553 crate::bucket::metadata_sys::ObjectLockConfigState::ConfirmedAbsent,
1554 ))),
1555 delete_replication_config_snapshot: Some(Arc::new(
1556 crate::bucket::replication::DeleteReplicationConfigSnapshot::default(),
1557 )),
1558 ..Default::default()
1559 };
1560 opts.ensure_lifecycle_delete_all_journal();
1561
1562 let _failure_guard = LifecycleDeleteAllFailureGuard;
1563 *LIFECYCLE_DELETE_ALL_TEST_FAILURE
1564 .lock()
1565 .expect("lifecycle delete-all failure hook should not poison") =
1566 Some((crate::object_api::LifecycleDeleteAllPhase::History, 1));
1567 let err = store
1568 .delete_prefix(&bucket, object, &opts)
1569 .await
1570 .expect_err("a later pool history failure must stop before trigger deletion");
1571 assert_eq!(err, StorageError::PreconditionFailed);
1572 assert!(
1573 opts.lifecycle_delete_all_journal()
1574 .expect("delete-all journal should be initialized")
1575 .lock()
1576 .mutation_started()
1577 );
1578
1579 let first_pool = store.pools[0].disk_set[0]
1580 .load_file_info_versions_exact(&bucket, object)
1581 .await
1582 .expect("first pool metadata should load")
1583 .expect("the trigger should remain");
1584 let first_pool_ordinary: Vec<&FileInfo> = first_pool
1585 .versions
1586 .iter()
1587 .filter(|version| !version.tier_free_version())
1588 .collect();
1589 assert_eq!(first_pool_ordinary.len(), 1);
1590 assert_eq!(first_pool_ordinary[0].version_id, Some(marker_id));
1591 assert!(first_pool_ordinary[0].deleted);
1592
1593 *LIFECYCLE_DELETE_ALL_TEST_FAILURE
1594 .lock()
1595 .expect("lifecycle delete-all failure hook should not poison") = None;
1596 store
1597 .delete_prefix(&bucket, object, &opts)
1598 .await
1599 .expect("retry should delete remaining history and its trigger owner");
1600 for pool in &store.pools {
1601 assert!(
1602 pool.disk_set[0]
1603 .load_file_info_versions_exact(&bucket, object)
1604 .await
1605 .expect("pool metadata should load after retry")
1606 .is_none(),
1607 "all ordinary versions should be removed after retry"
1608 );
1609 }
1610
1611 shutdown.cancel();
1612 }
1613
1614 #[tokio::test]
1615 #[serial_test::serial]
1616 async fn lifecycle_delete_all_phase_failures_preserve_barriers_and_retry() {
1617 let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("lifecycle-delete-all-phases", &[4, 4]).await;
1618 let bucket = format!("lifecycle-delete-all-phases-{}", Uuid::new_v4().simple());
1619 store
1620 .make_bucket(&bucket, &MakeBucketOptions::default())
1621 .await
1622 .expect("bucket should be created in both pools");
1623 let _failure_guard = LifecycleDeleteAllFailureGuard;
1624
1625 for (object, phase, expected_counts, mutation_started) in [
1626 ("preflight-failure", crate::object_api::LifecycleDeleteAllPhase::Preflight, [2, 2], false),
1627 (
1628 "final-preflight-failure",
1629 crate::object_api::LifecycleDeleteAllPhase::FinalPreflight,
1630 [1, 1],
1631 true,
1632 ),
1633 ("trigger-failure", crate::object_api::LifecycleDeleteAllPhase::Trigger, [0, 1], true),
1634 ] {
1635 let opts = seed_multi_pool_delete_all(&store, &bucket, object).await;
1636 *LIFECYCLE_DELETE_ALL_TEST_FAILURE
1637 .lock()
1638 .expect("lifecycle delete-all failure hook should not poison") = Some((phase, 1));
1639 let err = store
1640 .delete_prefix(&bucket, object, &opts)
1641 .await
1642 .expect_err("injected phase failure should stop the transaction");
1643 assert_eq!(err, StorageError::PreconditionFailed);
1644 assert_eq!(
1645 opts.lifecycle_delete_all_journal()
1646 .expect("delete-all journal should be initialized")
1647 .lock()
1648 .mutation_started(),
1649 mutation_started
1650 );
1651 assert_eq!(ordinary_version_count(&store, 0, &bucket, object).await, expected_counts[0]);
1652 assert_eq!(ordinary_version_count(&store, 1, &bucket, object).await, expected_counts[1]);
1653
1654 *LIFECYCLE_DELETE_ALL_TEST_FAILURE
1655 .lock()
1656 .expect("lifecycle delete-all failure hook should not poison") = None;
1657 store
1658 .delete_prefix(&bucket, object, &opts)
1659 .await
1660 .expect("retry should converge after the injected failure is removed");
1661 assert_eq!(ordinary_version_count(&store, 0, &bucket, object).await, 0);
1662 assert_eq!(ordinary_version_count(&store, 1, &bucket, object).await, 0);
1663 }
1664
1665 shutdown.cancel();
1666 }
1667
1668 fn assert_backend_layout_empty(info: &rustfs_madmin::BackendInfo) {
1669 assert!(info.standard_sc_parities.is_empty());
1670 assert!(info.standard_sc_data.is_empty());
1671 assert_eq!(info.standard_sc_parity, None);
1672 assert!(info.rr_sc_parities.is_empty());
1673 assert!(info.rr_sc_data.is_empty());
1674 assert_eq!(info.rr_sc_parity, None);
1675 }
1676
1677 #[test]
1678 fn build_backend_info_reports_heterogeneous_automatic_config_in_pool_order() {
1679 let config = lookup_config_for_pools_without_env(&KVS::new(), &[4, 2]).expect("automatic storage class should resolve");
1680
1681 let info = build_backend_info(&config, &[4, 2], &[2, 1], &[7, 3]);
1682
1683 assert!(matches!(info.backend_type, rustfs_madmin::BackendByte::Erasure));
1684 assert_eq!(info.standard_sc_parities, vec![2, 1]);
1685 assert_eq!(info.standard_sc_data, vec![2, 1]);
1686 assert_eq!(info.standard_sc_parity, None);
1687 assert_eq!(info.rr_sc_parities, vec![1, 1]);
1688 assert_eq!(info.rr_sc_data, vec![3, 1]);
1689 assert_eq!(info.rr_sc_parity, Some(1));
1690 assert_eq!(info.drives_per_set, vec![4, 2]);
1691 assert_eq!(info.total_sets, vec![7, 3]);
1692 }
1693
1694 #[test]
1695 fn build_backend_info_keeps_truthful_homogeneous_scalar() {
1696 let mut kvs = KVS::new();
1697 kvs.insert(CLASS_STANDARD.to_string(), "EC:2".to_string());
1698 let config =
1699 lookup_config_for_pools_without_env(&kvs, &[4, 6]).expect("explicit storage class should resolve for every pool");
1700
1701 let info = build_backend_info(&config, &[4, 6], &[2, 3], &[1, 1]);
1702
1703 assert_eq!(info.standard_sc_parities, vec![2, 2]);
1704 assert_eq!(info.standard_sc_data, vec![2, 4]);
1705 assert_eq!(info.standard_sc_parity, Some(2));
1706 assert_eq!(info.rr_sc_parities, vec![1, 1]);
1707 assert_eq!(info.rr_sc_data, vec![3, 5]);
1708 assert_eq!(info.rr_sc_parity, Some(1));
1709 }
1710
1711 #[test]
1712 fn build_backend_info_reports_single_disk_pool_without_inventing_parity() {
1713 let config = lookup_config_for_pools_without_env(&KVS::new(), &[4, 1]).expect("single disk pool should resolve");
1714
1715 let info = build_backend_info(&config, &[4, 1], &[2, 0], &[1, 1]);
1716
1717 assert_eq!(info.standard_sc_parities, vec![2, 0]);
1718 assert_eq!(info.standard_sc_data, vec![2, 1]);
1719 assert_eq!(info.standard_sc_parity, None);
1720 assert_eq!(info.rr_sc_parities, vec![1, 0]);
1721 assert_eq!(info.rr_sc_data, vec![3, 1]);
1722 assert_eq!(info.rr_sc_parity, None);
1723 }
1724
1725 #[test]
1726 fn build_backend_info_uses_pool_defaults_only_when_truly_uninitialized() {
1727 let info = build_backend_info(&Default::default(), &[4, 2], &[1, 0], &[1, 1]);
1728
1729 assert_eq!(info.standard_sc_parities, vec![1, 0]);
1730 assert_eq!(info.standard_sc_data, vec![3, 2]);
1731 assert_eq!(info.standard_sc_parity, None);
1732 assert!(info.rr_sc_parities.is_empty());
1733 assert!(info.rr_sc_data.is_empty());
1734 assert_eq!(info.rr_sc_parity, None);
1735 }
1736
1737 #[test]
1738 fn build_backend_info_fails_closed_on_initialized_snapshot_mismatch() {
1739 let config = lookup_config_for_pools_without_env(&KVS::new(), &[4, 2]).expect("automatic storage class should resolve");
1740
1741 let info = build_backend_info(&config, &[4, 6], &[1, 2], &[1, 1]);
1742
1743 assert_backend_layout_empty(&info);
1744 assert_eq!(info.drives_per_set, vec![4, 6]);
1745 assert_eq!(info.total_sets, vec![1, 1]);
1746 }
1747
1748 #[test]
1749 fn build_backend_info_expands_valid_legacy_scalar_snapshot() {
1750 let mut kvs = KVS::new();
1751 kvs.insert(CLASS_STANDARD.to_string(), "EC:2".to_string());
1752 kvs.insert(CLASS_RRS.to_string(), "EC:1".to_string());
1753 let current =
1754 lookup_config_for_pools_without_env(&kvs, &[4, 6]).expect("distinct legacy scalars should resolve for every pool");
1755 let encoded = serde_json::to_string(¤t).expect("config should serialize");
1756 let legacy: storageclass::Config = serde_json::from_str(&encoded).expect("legacy scalar config should deserialize");
1757
1758 let info = build_backend_info(&legacy, &[4, 6], &[2, 3], &[1, 1]);
1759
1760 assert_eq!(info.standard_sc_parities, vec![2, 2]);
1761 assert_eq!(info.standard_sc_data, vec![2, 4]);
1762 assert_eq!(info.standard_sc_parity, Some(2));
1763 assert_eq!(info.rr_sc_parities, vec![1, 1]);
1764 assert_eq!(info.rr_sc_data, vec![3, 5]);
1765 assert_eq!(info.rr_sc_parity, Some(1));
1766 }
1767
1768 #[test]
1769 fn build_backend_info_rejects_legacy_scalar_invalid_for_later_pool() {
1770 let current = lookup_config_for_pools_without_env(&KVS::new(), &[4, 2]).expect("automatic config should resolve");
1771 let encoded = serde_json::to_string(¤t).expect("config should serialize");
1772 let legacy: storageclass::Config = serde_json::from_str(&encoded).expect("legacy scalar config should deserialize");
1773
1774 let info = build_backend_info(&legacy, &[4, 2], &[2, 1], &[1, 1]);
1775
1776 assert_backend_layout_empty(&info);
1777 }
1778
1779 #[test]
1780 fn build_backend_info_never_reports_invalid_geometry() {
1781 let config = storageclass::Config::default();
1782
1783 assert_backend_layout_empty(&build_backend_info(&config, &[4, 2], &[5, 1], &[1, 1]));
1784 assert_backend_layout_empty(&build_backend_info(&config, &[0], &[0], &[1]));
1785 assert_backend_layout_empty(&build_backend_info(&config, &[4], &[3], &[1]));
1786 }
1787
1788 #[test]
1789 fn build_backend_info_rejects_mismatched_topology_lengths() {
1790 let config = lookup_config_for_pools_without_env(&KVS::new(), &[4, 2]).expect("automatic storage class should resolve");
1791
1792 assert_backend_layout_empty(&build_backend_info(&config, &[4, 2], &[2], &[1, 1]));
1793 assert_backend_layout_empty(&build_backend_info(&config, &[4, 2], &[2, 1], &[1]));
1794
1795 let empty = build_backend_info(&Default::default(), &[], &[], &[]);
1796 assert_backend_layout_empty(&empty);
1797 assert!(empty.drives_per_set.is_empty());
1798 assert!(empty.total_sets.is_empty());
1799 }
1800
1801 #[test]
1802 fn build_backend_info_uses_one_complete_arc_swap_snapshot() {
1803 let old = lookup_config_for_pools_without_env(&KVS::new(), &[4, 2]).expect("old config should resolve");
1804 let mut new_kvs = KVS::new();
1805 new_kvs.insert(CLASS_STANDARD.to_string(), "EC:1".to_string());
1806 let new = lookup_config_for_pools_without_env(&new_kvs, &[4, 2]).expect("new config should resolve");
1807 let snapshots = ArcSwap::from_pointee(old);
1808 let held_old = snapshots.load_full();
1809 snapshots.store(Arc::new(new));
1810
1811 let old_info = build_backend_info(&held_old, &[4, 2], &[2, 1], &[1, 1]);
1812 let new_info = build_backend_info(&snapshots.load_full(), &[4, 2], &[2, 1], &[1, 1]);
1813
1814 assert_eq!(old_info.standard_sc_parities, vec![2, 1]);
1815 assert_eq!(old_info.standard_sc_data, vec![2, 1]);
1816 assert_eq!(old_info.standard_sc_parity, None);
1817 assert_eq!(old_info.rr_sc_parities, vec![1, 1]);
1818 assert_eq!(new_info.standard_sc_parities, vec![1, 1]);
1819 assert_eq!(new_info.standard_sc_data, vec![3, 1]);
1820 assert_eq!(new_info.standard_sc_parity, Some(1));
1821 assert_eq!(new_info.rr_sc_parities, vec![1, 1]);
1822 }
1823
1824 fn object_info_with_mod_time(unix_ts: i64, delete_marker: bool) -> ObjectInfo {
1825 ObjectInfo {
1826 mod_time: Some(OffsetDateTime::from_unix_timestamp(unix_ts).expect("operation should succeed")),
1827 delete_marker,
1828 ..Default::default()
1829 }
1830 }
1831
1832 fn object_info_with_identity(unix_ts: i64, delete_marker: bool, version_id: Uuid, etag: Option<String>) -> ObjectInfo {
1833 ObjectInfo {
1834 version_id: Some(version_id),
1835 etag,
1836 ..object_info_with_mod_time(unix_ts, delete_marker)
1837 }
1838 }
1839
1840 #[test]
1841 fn resolve_latest_object_info_candidates_returns_latest_delete_marker() {
1842 let candidates = vec![
1843 LatestObjectInfoCandidate {
1844 info: Some(object_info_with_mod_time(10, false)),
1845 idx: 0,
1846 err: None,
1847 },
1848 LatestObjectInfoCandidate {
1849 info: Some(object_info_with_mod_time(20, true)),
1850 idx: 1,
1851 err: None,
1852 },
1853 ];
1854
1855 let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
1856 .expect("operation should succeed");
1857
1858 assert_eq!(idx, 1);
1859 assert!(info.delete_marker);
1860 }
1861
1862 #[test]
1863 fn resolve_latest_object_info_candidates_prefers_higher_pool_idx_on_equal_mod_time_for_equivalent_candidates() {
1864 let candidates = vec![
1865 LatestObjectInfoCandidate {
1866 info: Some(object_info_with_mod_time(10, false)),
1867 idx: 0,
1868 err: None,
1869 },
1870 LatestObjectInfoCandidate {
1871 info: Some(object_info_with_mod_time(10, false)),
1872 idx: 1,
1873 err: None,
1874 },
1875 ];
1876
1877 let (_, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
1878 .expect("operation should succeed");
1879
1880 assert_eq!(idx, 1);
1881 }
1882
1883 #[test]
1884 fn resolve_latest_object_info_candidates_prefers_active_target_over_higher_suspended_source() {
1885 let source = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
1886 let target = source.clone();
1887
1888 let (info, idx) = resolve_latest_object_info_candidates_with_pool_state(
1889 vec![
1890 LatestObjectInfoCandidate {
1891 info: Some(target),
1892 idx: 0,
1893 err: None,
1894 },
1895 LatestObjectInfoCandidate {
1896 info: Some(source),
1897 idx: 1,
1898 err: None,
1899 },
1900 ],
1901 &[false, true],
1902 "bucket",
1903 "object",
1904 &ObjectOptions::default(),
1905 )
1906 .expect("an active committed target should fence an equivalent suspended source");
1907
1908 assert_eq!(idx, 0);
1909 assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
1910 }
1911
1912 #[test]
1913 fn resolve_latest_object_info_candidates_keeps_lone_suspended_source_readable() {
1914 let source = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
1915
1916 let (_, idx) = resolve_latest_object_info_candidates_with_pool_state(
1917 vec![LatestObjectInfoCandidate {
1918 info: Some(source),
1919 idx: 1,
1920 err: None,
1921 }],
1922 &[false, true],
1923 "bucket",
1924 "object",
1925 &ObjectOptions::default(),
1926 )
1927 .expect("a source-only object must remain readable before migration commits");
1928
1929 assert_eq!(idx, 1);
1930 }
1931
1932 #[test]
1933 fn resolve_latest_object_info_candidates_rejects_suspended_source_identity_conflict() {
1934 let target = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-new".to_string()));
1935 let source = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-old".to_string()));
1936
1937 let err = resolve_latest_object_info_candidates_with_pool_state(
1938 vec![
1939 LatestObjectInfoCandidate {
1940 info: Some(target),
1941 idx: 0,
1942 err: None,
1943 },
1944 LatestObjectInfoCandidate {
1945 info: Some(source),
1946 idx: 1,
1947 err: None,
1948 },
1949 ],
1950 &[false, true],
1951 "bucket",
1952 "object",
1953 &ObjectOptions::default(),
1954 )
1955 .expect_err("pool state must not mask an equal-time identity conflict");
1956
1957 assert_eq!(err, Error::ErasureReadQuorum);
1958 }
1959
1960 #[test]
1961 fn resolve_latest_object_info_candidates_keeps_index_fallback_for_fully_equivalent_identities() {
1962 let candidates = vec![
1963 LatestObjectInfoCandidate {
1964 info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
1965 idx: 2,
1966 err: None,
1967 },
1968 LatestObjectInfoCandidate {
1969 info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
1970 idx: 7,
1971 err: None,
1972 },
1973 ];
1974
1975 let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
1976 .expect("equivalent replicas must resolve deterministically");
1977
1978 assert_eq!(idx, 7);
1979 assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
1980 }
1981
1982 #[test]
1983 fn resolve_latest_object_info_candidates_rejects_equal_time_version_id_conflict() {
1984 let candidates = vec![
1985 LatestObjectInfoCandidate {
1986 info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
1987 idx: 0,
1988 err: None,
1989 },
1990 LatestObjectInfoCandidate {
1991 info: Some(object_info_with_identity(10, false, Uuid::from_u128(2), Some("etag-a".to_string()))),
1992 idx: 1,
1993 err: None,
1994 },
1995 ];
1996
1997 let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
1998 .expect_err("divergent version ids must not silently resolve to the higher pool index");
1999
2000 assert_eq!(err, Error::ErasureReadQuorum);
2001 }
2002
2003 #[test]
2004 fn resolve_latest_object_info_candidates_rejects_equal_time_etag_conflict() {
2005 let candidates = vec![
2006 LatestObjectInfoCandidate {
2007 info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-old".to_string()))),
2008 idx: 0,
2009 err: None,
2010 },
2011 LatestObjectInfoCandidate {
2012 info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-new".to_string()))),
2013 idx: 1,
2014 err: None,
2015 },
2016 ];
2017
2018 let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
2019 .expect_err("divergent etags must not silently resolve to the higher pool index");
2020
2021 assert_eq!(err, Error::ErasureReadQuorum);
2022 }
2023
2024 #[test]
2025 fn resolve_latest_object_info_candidates_rejects_equal_time_delete_marker_conflict() {
2026 let candidates = vec![
2027 LatestObjectInfoCandidate {
2028 info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), None)),
2029 idx: 0,
2030 err: None,
2031 },
2032 LatestObjectInfoCandidate {
2033 info: Some(object_info_with_identity(10, true, Uuid::from_u128(1), Some("etag-a".to_string()))),
2034 idx: 1,
2035 err: None,
2036 },
2037 ];
2038
2039 let err = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
2040 .expect_err("a delete marker tied with a live version must not be masked by the pool index");
2041
2042 assert_eq!(err, Error::ErasureReadQuorum);
2043 }
2044
2045 fn assert_equal_time_identity_conflict(left: ObjectInfo, right: ObjectInfo) {
2046 let err = resolve_latest_object_info_candidates(
2047 vec![
2048 LatestObjectInfoCandidate {
2049 info: Some(left),
2050 idx: 0,
2051 err: None,
2052 },
2053 LatestObjectInfoCandidate {
2054 info: Some(right),
2055 idx: 1,
2056 err: None,
2057 },
2058 ],
2059 "bucket",
2060 "object",
2061 &ObjectOptions::default(),
2062 )
2063 .expect_err("equal-time identity divergence must fail closed");
2064
2065 assert_eq!(err, Error::ErasureReadQuorum);
2066 }
2067
2068 #[test]
2069 fn resolve_latest_object_info_candidates_rejects_equal_time_payload_identity_conflicts() {
2070 let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2071
2072 let mut size = base.clone();
2073 size.size = 1;
2074 assert_equal_time_identity_conflict(base.clone(), size);
2075
2076 let mut actual_size = base.clone();
2077 actual_size.actual_size = 1;
2078 assert_equal_time_identity_conflict(base.clone(), actual_size);
2079
2080 let mut checksum = base.clone();
2081 checksum.checksum = Some(bytes::Bytes::from_static(b"checksum"));
2082 assert_equal_time_identity_conflict(base.clone(), checksum);
2083
2084 let mut parts = base.clone();
2085 parts.parts = std::sync::Arc::new(vec![rustfs_filemeta::ObjectPartInfo {
2086 etag: "part-etag".to_string(),
2087 number: 1,
2088 size: 1,
2089 ..Default::default()
2090 }]);
2091 assert_equal_time_identity_conflict(base.clone(), parts);
2092
2093 let mut transition = base;
2094 transition.transitioned_object.tier = "tier-a".to_string();
2095 assert_equal_time_identity_conflict(
2096 object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
2097 transition,
2098 );
2099 }
2100
2101 #[test]
2102 fn resolve_latest_object_info_candidates_accepts_data_movement_rewrites() {
2103 let mut source = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2104 source.data_dir = Some(Uuid::from_u128(1));
2105
2106 let mut target = source.clone();
2107 target.data_dir = Some(Uuid::from_u128(2));
2108 rustfs_utils::http::insert_str(
2109 Arc::make_mut(&mut target.user_defined),
2110 rustfs_utils::http::SUFFIX_DATA_MOVED,
2111 "true".to_string(),
2112 );
2113 rustfs_utils::http::insert_str(
2114 Arc::make_mut(&mut target.user_defined),
2115 rustfs_utils::http::SUFFIX_ACTUAL_SIZE,
2116 "0".to_string(),
2117 );
2118
2119 let (info, idx) = resolve_latest_object_info_candidates(
2120 vec![
2121 LatestObjectInfoCandidate {
2122 info: Some(source),
2123 idx: 0,
2124 err: None,
2125 },
2126 LatestObjectInfoCandidate {
2127 info: Some(target.clone()),
2128 idx: 1,
2129 err: None,
2130 },
2131 ],
2132 "bucket",
2133 "object",
2134 &ObjectOptions::default(),
2135 )
2136 .expect("a committed data-movement target must remain readable before source cleanup");
2137
2138 assert_eq!(idx, 1);
2139 assert_eq!(info.data_dir, target.data_dir);
2140 }
2141
2142 #[test]
2143 fn resolve_latest_object_info_candidates_rejects_unmarked_data_dir_conflict() {
2144 let mut left = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2145 left.data_dir = Some(Uuid::from_u128(1));
2146
2147 let mut right = left.clone();
2148 right.data_dir = Some(Uuid::from_u128(2));
2149
2150 assert_equal_time_identity_conflict(left, right);
2151 }
2152
2153 #[test]
2154 fn resolve_latest_object_info_candidates_accepts_internal_metadata_aliases() {
2155 let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2156 let mut rustfs_alias = base.clone();
2157 rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
2158 "x-rustfs-internal-compression".to_string(),
2159 "zstd".to_string(),
2160 )]));
2161 let mut minio_alias = base.clone();
2162 minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
2163 "X-MINIO-INTERNAL-COMPRESSION".to_string(),
2164 "zstd".to_string(),
2165 )]));
2166
2167 let (_, idx) = resolve_latest_object_info_candidates(
2168 vec![
2169 LatestObjectInfoCandidate {
2170 info: Some(rustfs_alias),
2171 idx: 0,
2172 err: None,
2173 },
2174 LatestObjectInfoCandidate {
2175 info: Some(minio_alias),
2176 idx: 1,
2177 err: None,
2178 },
2179 ],
2180 "bucket",
2181 "object",
2182 &ObjectOptions::default(),
2183 )
2184 .expect("same-value internal aliases should resolve");
2185 assert_eq!(idx, 1);
2186
2187 let mut dual_alias = base.clone();
2188 dual_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
2189 ("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
2190 ("x-minio-internal-compression".to_string(), "zstd".to_string()),
2191 ]));
2192 let mut single_alias = base;
2193 single_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
2194 "x-rustfs-internal-compression".to_string(),
2195 "zstd".to_string(),
2196 )]));
2197
2198 let (_, idx) = resolve_latest_object_info_candidates(
2199 vec![
2200 LatestObjectInfoCandidate {
2201 info: Some(dual_alias),
2202 idx: 0,
2203 err: None,
2204 },
2205 LatestObjectInfoCandidate {
2206 info: Some(single_alias),
2207 idx: 1,
2208 err: None,
2209 },
2210 ],
2211 "bucket",
2212 "object",
2213 &ObjectOptions::default(),
2214 )
2215 .expect("dual-key and single-key internal metadata should resolve");
2216 assert_eq!(idx, 1);
2217 }
2218
2219 #[test]
2220 fn resolve_latest_object_info_candidates_rejects_different_internal_metadata_alias_values() {
2221 let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2222 let mut rustfs_alias = base.clone();
2223 rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
2224 "x-rustfs-internal-compression".to_string(),
2225 "zstd".to_string(),
2226 )]));
2227 let mut minio_alias = base;
2228 minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
2229 "x-minio-internal-compression".to_string(),
2230 "snappy".to_string(),
2231 )]));
2232
2233 assert_equal_time_identity_conflict(rustfs_alias, minio_alias);
2234 }
2235
2236 #[test]
2237 fn resolve_latest_object_info_candidates_preserves_dynamic_internal_metadata_identity_case() {
2238 for suffix_prefix in ["replication-reset-", "replication-delete-marker-version-"] {
2239 let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2240 let mut rustfs_alias = base.clone();
2241 rustfs_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
2242 format!(
2243 "X-RUSTFS-INTERNAL-{}{suffix}",
2244 suffix_prefix.to_uppercase(),
2245 suffix = "arn:aws:s3:::Bucket"
2246 ),
2247 "value".to_string(),
2248 )]));
2249 let mut minio_alias = base.clone();
2250 minio_alias.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
2251 format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::Bucket"),
2252 "value".to_string(),
2253 )]));
2254
2255 let (_, idx) = resolve_latest_object_info_candidates(
2256 vec![
2257 LatestObjectInfoCandidate {
2258 info: Some(rustfs_alias.clone()),
2259 idx: 0,
2260 err: None,
2261 },
2262 LatestObjectInfoCandidate {
2263 info: Some(minio_alias),
2264 idx: 1,
2265 err: None,
2266 },
2267 ],
2268 "bucket",
2269 "object",
2270 &ObjectOptions::default(),
2271 )
2272 .expect("dynamic internal aliases with the same target should resolve");
2273 assert_eq!(idx, 1);
2274
2275 let mut different_target_case = base;
2276 different_target_case.user_defined = std::sync::Arc::new(std::collections::HashMap::from([(
2277 format!("x-minio-internal-{suffix_prefix}arn:aws:s3:::bucket"),
2278 "value".to_string(),
2279 )]));
2280
2281 assert_equal_time_identity_conflict(rustfs_alias, different_target_case);
2282 }
2283 }
2284
2285 #[test]
2286 fn resolve_latest_object_info_candidates_rejects_conflicting_internal_metadata_aliases_in_one_candidate() {
2287 let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2288 let mut first = base.clone();
2289 first.user_defined = std::sync::Arc::new(std::collections::HashMap::from([
2290 ("x-rustfs-internal-compression".to_string(), "zstd".to_string()),
2291 ("x-minio-internal-compression".to_string(), "snappy".to_string()),
2292 ]));
2293 let mut second = base;
2294 second.user_defined = first.user_defined.clone();
2295
2296 assert_equal_time_identity_conflict(first, second);
2297 }
2298
2299 #[test]
2300 fn resolve_latest_object_info_candidates_rejects_replication_identity_conflict() {
2301 let base = object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2302
2303 let mut replication = base.clone();
2304 replication.replication_status_internal = Some("PENDING".to_string());
2305 replication.replication_status = ReplicationStatusType::Pending;
2306 assert_equal_time_identity_conflict(base.clone(), replication);
2307
2308 let mut purge = base.clone();
2309 purge.version_purge_status_internal = Some("PENDING".to_string());
2310 purge.version_purge_status = VersionPurgeStatusType::Pending;
2311 assert_equal_time_identity_conflict(base.clone(), purge);
2312
2313 let mut decision = base;
2314 decision.replication_decision = "replicate".to_string();
2315 assert_equal_time_identity_conflict(
2316 object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string())),
2317 decision,
2318 );
2319 }
2320
2321 #[test]
2322 fn resolve_latest_object_info_candidates_rejects_none_vs_unix_epoch_mod_time() {
2323 let mut without_mod_time = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2324 without_mod_time.mod_time = None;
2325 let with_unix_epoch = object_info_with_identity(0, false, Uuid::from_u128(1), Some("etag-a".to_string()));
2326
2327 assert_equal_time_identity_conflict(without_mod_time, with_unix_epoch);
2328 }
2329
2330 #[test]
2331 fn resolve_latest_object_info_candidates_ignores_older_identity_conflicts() {
2332 let latest = object_info_with_identity(20, false, Uuid::from_u128(1), Some("etag-latest".to_string()));
2333 let mut older = object_info_with_identity(10, true, Uuid::from_u128(2), Some("etag-old".to_string()));
2334 older.data_dir = Some(Uuid::from_u128(2));
2335
2336 let (info, idx) = resolve_latest_object_info_candidates(
2337 vec![
2338 LatestObjectInfoCandidate {
2339 info: Some(latest),
2340 idx: 0,
2341 err: None,
2342 },
2343 LatestObjectInfoCandidate {
2344 info: Some(older),
2345 idx: 9,
2346 err: None,
2347 },
2348 ],
2349 "bucket",
2350 "object",
2351 &ObjectOptions::default(),
2352 )
2353 .expect("older identity divergence must not affect the latest candidate");
2354
2355 assert_eq!(idx, 0);
2356 assert_eq!(
2357 info.mod_time,
2358 Some(OffsetDateTime::from_unix_timestamp(20).expect("operation should succeed"))
2359 );
2360 }
2361
2362 #[test]
2363 fn resolve_latest_object_info_candidates_ignores_not_found_pools_when_resolving() {
2364 let candidates = vec![
2365 LatestObjectInfoCandidate {
2366 info: Some(object_info_with_identity(10, false, Uuid::from_u128(1), Some("etag-a".to_string()))),
2367 idx: 0,
2368 err: None,
2369 },
2370 LatestObjectInfoCandidate {
2371 info: None,
2372 idx: 1,
2373 err: Some(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
2374 },
2375 ];
2376
2377 let (info, idx) = resolve_latest_object_info_candidates(candidates, "bucket", "object", &ObjectOptions::default())
2378 .expect("not-found pools must not block resolution of found candidates");
2379
2380 assert_eq!(idx, 0);
2381 assert_eq!(info.version_id, Some(Uuid::from_u128(1)));
2382 }
2383
2384 #[test]
2385 fn resolve_latest_object_info_candidates_returns_non_not_found_error() {
2386 let err = resolve_latest_object_info_candidates(
2387 vec![LatestObjectInfoCandidate {
2388 info: None,
2389 idx: 0,
2390 err: Some(Error::ErasureReadQuorum),
2391 }],
2392 "bucket",
2393 "object",
2394 &ObjectOptions::default(),
2395 )
2396 .unwrap_err();
2397
2398 assert_eq!(err, Error::ErasureReadQuorum);
2399 }
2400
2401 #[test]
2402 fn resolve_latest_object_info_candidates_returns_version_not_found_for_versioned_lookups() {
2403 let err = resolve_latest_object_info_candidates(
2404 vec![LatestObjectInfoCandidate {
2405 info: None,
2406 idx: 0,
2407 err: Some(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
2408 }],
2409 "bucket",
2410 "object",
2411 &ObjectOptions {
2412 version_id: Some("vid-1".to_string()),
2413 ..Default::default()
2414 },
2415 )
2416 .unwrap_err();
2417
2418 assert_eq!(
2419 err,
2420 Error::VersionNotFound("bucket".to_string(), "object".to_string(), "vid-1".to_string())
2421 );
2422 }
2423
2424 #[test]
2425 fn pool_lookup_not_found_error_returns_object_not_found_for_latest_lookup() {
2426 let err = pool_lookup_not_found_error("bucket", "object", &ObjectOptions::default());
2427
2428 assert_eq!(err, Error::ObjectNotFound("bucket".to_string(), "object".to_string()));
2429 }
2430
2431 #[test]
2432 fn pool_lookup_not_found_error_returns_version_not_found_for_versioned_lookup() {
2433 let err = pool_lookup_not_found_error(
2434 "bucket",
2435 "object",
2436 &ObjectOptions {
2437 version_id: Some("vid-1".to_string()),
2438 ..Default::default()
2439 },
2440 );
2441
2442 assert_eq!(
2443 err,
2444 Error::VersionNotFound("bucket".to_string(), "object".to_string(), "vid-1".to_string())
2445 );
2446 }
2447
2448 #[test]
2449 fn resolve_store_rebalance_pool_meta_reload_result_passthrough_ok() {
2450 resolve_store_rebalance_pool_meta_reload_result(Ok(()), "reload_pool_meta")
2451 .expect("successful pool meta reload should pass through");
2452 }
2453
2454 #[test]
2455 fn resolve_store_rebalance_pool_meta_reload_result_wraps_error_context() {
2456 let err = resolve_store_rebalance_pool_meta_reload_result::<()>(Err(Error::SlowDown), "reload_pool_meta")
2457 .expect_err("failed pool meta reload should be wrapped");
2458 let err_message = err.to_string();
2459 assert!(err_message.contains("store rebalance pool meta reload failed during reload_pool_meta"));
2460 assert!(err_message.contains(&Error::SlowDown.to_string()));
2461 }
2462
2463 #[test]
2464 fn resolve_rebalance_delete_from_all_pools_result_passthrough_ok() {
2465 let info = ObjectInfo {
2466 bucket: "bucket".to_string(),
2467 name: "object".to_string(),
2468 ..Default::default()
2469 };
2470
2471 let resolved = resolve_rebalance_delete_from_all_pools_result(Ok(info.clone()), "bucket", "object")
2472 .expect("successful rebalance delete should pass through");
2473
2474 assert_eq!(resolved.bucket, info.bucket);
2475 assert_eq!(resolved.name, info.name);
2476 }
2477
2478 #[test]
2479 fn resolve_rebalance_delete_from_all_pools_result_wraps_object_context() {
2480 let err = resolve_rebalance_delete_from_all_pools_result(Err(Error::SlowDown), "bucket", "object")
2481 .expect_err("failed rebalance delete should be wrapped");
2482 let rendered = err.to_string();
2483
2484 assert!(rendered.contains("failed to delete rebalance source object bucket/object"), "{rendered}");
2485 assert!(rendered.contains(&Error::SlowDown.to_string()), "{rendered}");
2486 }
2487
2488 #[test]
2489 fn resolve_rebalance_delete_from_all_pools_results_fails_on_later_pool_error() {
2490 let err = resolve_rebalance_delete_from_all_pools_results(
2491 vec![
2492 RebalanceDeletePoolResult {
2493 pool_idx: 0,
2494 result: Ok(ObjectInfo {
2495 bucket: "bucket".to_string(),
2496 name: "object".to_string(),
2497 ..Default::default()
2498 }),
2499 },
2500 RebalanceDeletePoolResult {
2501 pool_idx: 1,
2502 result: Err(Error::SlowDown),
2503 },
2504 ],
2505 "bucket",
2506 "object",
2507 )
2508 .expect_err("non-ignorable errors from later pools must not be hidden");
2509 let rendered = err.to_string();
2510
2511 assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
2512 assert!(rendered.contains(&Error::SlowDown.to_string()), "{rendered}");
2513 }
2514
2515 #[test]
2516 fn resolve_rebalance_delete_from_all_pools_results_ignores_later_not_found_after_success() {
2517 let info = ObjectInfo {
2518 bucket: "bucket".to_string(),
2519 name: "object".to_string(),
2520 ..Default::default()
2521 };
2522
2523 let resolved = resolve_rebalance_delete_from_all_pools_results(
2524 vec![
2525 RebalanceDeletePoolResult {
2526 pool_idx: 0,
2527 result: Ok(info.clone()),
2528 },
2529 RebalanceDeletePoolResult {
2530 pool_idx: 1,
2531 result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
2532 },
2533 ],
2534 "bucket",
2535 "object",
2536 )
2537 .expect("not-found errors from other pools should be ignored when a delete succeeds");
2538
2539 assert_eq!(resolved.bucket, info.bucket);
2540 assert_eq!(resolved.name, info.name);
2541 }
2542
2543 #[test]
2544 fn resolve_rebalance_delete_from_all_pools_results_accepts_success_after_not_found() {
2545 let info = ObjectInfo {
2546 bucket: "bucket".to_string(),
2547 name: "object".to_string(),
2548 ..Default::default()
2549 };
2550
2551 let resolved = resolve_rebalance_delete_from_all_pools_results(
2552 vec![
2553 RebalanceDeletePoolResult {
2554 pool_idx: 0,
2555 result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
2556 },
2557 RebalanceDeletePoolResult {
2558 pool_idx: 1,
2559 result: Ok(info.clone()),
2560 },
2561 ],
2562 "bucket",
2563 "object",
2564 )
2565 .expect("a successful delete should pass even when an earlier pool reports not-found");
2566
2567 assert_eq!(resolved.bucket, info.bucket);
2568 assert_eq!(resolved.name, info.name);
2569 }
2570
2571 #[test]
2572 fn resolve_rebalance_delete_from_all_pools_results_fails_when_all_results_are_ignored_errors() {
2573 let err = resolve_rebalance_delete_from_all_pools_results(
2574 vec![
2575 RebalanceDeletePoolResult {
2576 pool_idx: 0,
2577 result: Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())),
2578 },
2579 RebalanceDeletePoolResult {
2580 pool_idx: 1,
2581 result: Err(Error::VersionNotFound("bucket".to_string(), "object".to_string(), "vid-1".to_string())),
2582 },
2583 ],
2584 "bucket",
2585 "object",
2586 )
2587 .expect_err("all ignored errors without any successful delete should still fail");
2588 let rendered = err.to_string();
2589
2590 assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
2591 assert!(rendered.contains("Version not found"), "{rendered}");
2592 }
2593
2594 #[test]
2595 fn resolve_rebalance_delete_from_all_pools_results_fails_on_write_quorum_even_with_success() {
2596 let err = resolve_rebalance_delete_from_all_pools_results(
2597 vec![
2598 RebalanceDeletePoolResult {
2599 pool_idx: 0,
2600 result: Ok(ObjectInfo {
2601 bucket: "bucket".to_string(),
2602 name: "object".to_string(),
2603 ..Default::default()
2604 }),
2605 },
2606 RebalanceDeletePoolResult {
2607 pool_idx: 1,
2608 result: Err(Error::ErasureWriteQuorum),
2609 },
2610 ],
2611 "bucket",
2612 "object",
2613 )
2614 .expect_err("write quorum failures must fail the aggregate delete");
2615 let rendered = err.to_string();
2616
2617 assert!(rendered.contains("pool 1 delete failed for bucket/object"), "{rendered}");
2618 assert!(rendered.contains(&Error::ErasureWriteQuorum.to_string()), "{rendered}");
2619 }
2620
2621 #[test]
2622 fn rebalance_disk_set_lookup_error_formats_pool_and_set_context() {
2623 let err = rebalance_disk_set_lookup_error(2, 7, 3);
2624
2625 assert!(
2626 err.to_string()
2627 .contains("failed to resolve rebalance disk set: pool index 2, set index 7, pool count 3")
2628 );
2629 }
2630
2631 fn reload_test_pool_status(decommission: Option<PoolDecommissionInfo>, last_update: time::OffsetDateTime) -> PoolStatus {
2632 PoolStatus {
2633 id: 0,
2634 cmd_line: "pool-0".to_string(),
2635 last_update,
2636 decommission,
2637 }
2638 }
2639
2640 fn reload_test_pool_meta(pool: PoolStatus) -> PoolMeta {
2641 PoolMeta {
2642 version: POOL_META_VERSION,
2643 pools: vec![pool],
2644 dont_save: false,
2645 }
2646 }
2647
2648 async fn persist_reload_snapshot(store: &ECStore, snapshot: &PoolMeta) {
2649 snapshot
2650 .save_for_startup(store.pools.clone())
2651 .await
2652 .expect("pool meta snapshot should persist to every pool");
2653 }
2654
2655 #[derive(Debug)]
2656 struct PoolMetaCommitPauseStorage {
2657 inner: Arc<Sets>,
2658 pause_committed_pool_meta: bool,
2659 paused: Arc<tokio::sync::Notify>,
2660 release: Arc<tokio::sync::Notify>,
2661 }
2662
2663 #[async_trait::async_trait]
2664 impl crate::storage_api_contracts::object::ObjectIO for PoolMetaCommitPauseStorage {
2665 type Error = Error;
2666 type RangeSpec = crate::storage_api_contracts::range::HTTPRangeSpec;
2667 type HeaderMap = http::HeaderMap;
2668 type ObjectOptions = crate::object_api::ObjectOptions;
2669 type ObjectInfo = crate::object_api::ObjectInfo;
2670 type GetObjectReader = crate::object_api::GetObjectReader;
2671 type PutObjectReader = crate::object_api::PutObjReader;
2672
2673 async fn get_object_reader(
2674 &self,
2675 bucket: &str,
2676 object: &str,
2677 range: Option<Self::RangeSpec>,
2678 h: Self::HeaderMap,
2679 opts: &Self::ObjectOptions,
2680 ) -> std::result::Result<Self::GetObjectReader, Error> {
2681 self.inner.get_object_reader(bucket, object, range, h, opts).await
2682 }
2683
2684 async fn put_object(
2685 &self,
2686 bucket: &str,
2687 object: &str,
2688 data: &mut Self::PutObjectReader,
2689 opts: &Self::ObjectOptions,
2690 ) -> std::result::Result<Self::ObjectInfo, Error> {
2691 let result = self.inner.put_object(bucket, object, data, opts).await;
2692 if result.is_ok() && object == POOL_META_NAME && self.pause_committed_pool_meta {
2693 let committed = read_config_no_lock_preserve_empty_with_metadata(self.inner.clone(), POOL_META_NAME)
2694 .await
2695 .ok()
2696 .and_then(|(payload, _)| pool_meta_v3_commit_state_for_test(payload).ok())
2697 .is_some_and(|(_, committed)| committed);
2698 if committed {
2699 self.paused.notify_one();
2700 self.release.notified().await;
2701 }
2702 }
2703 result
2704 }
2705 }
2706
2707 #[tokio::test]
2708 #[serial_test::serial]
2709 async fn pool_meta_runtime_load_waits_for_multi_pool_commit_fence() {
2710 let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-load-fence", &[2, 2]).await;
2711 let old = PoolMeta::new(&store.pools, &PoolMeta::default());
2712 old.save_for_startup(store.pools.clone())
2713 .await
2714 .expect("old V3 pool metadata should persist");
2715 let (old_payload, _) = read_config_no_lock_preserve_empty_with_metadata(store.pools[0].clone(), POOL_META_NAME)
2716 .await
2717 .expect("old V3 pool metadata should be readable");
2718 let (old_generation, old_committed) =
2719 pool_meta_v3_commit_state_for_test(old_payload).expect("old replica should be a V3 envelope");
2720 assert!(old_committed, "old generation should be committed before the mixed-state save");
2721 let next_generation = old_generation.checked_add(1).expect("test generation should not exhaust");
2722 let mut newer = old.clone();
2723 newer.pools[0].last_update += TimeDuration::seconds(1);
2724
2725 let pool_meta_lock = store.pools[0]
2726 .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
2727 .await
2728 .expect("pool metadata lock should be created");
2729 let pool_meta_guard = pool_meta_lock
2730 .get_write_lock(get_lock_acquire_timeout())
2731 .await
2732 .expect("pool metadata write fence should be acquired");
2733 let started = Arc::new(tokio::sync::Notify::new());
2734 let mut load_task = tokio::spawn({
2735 let store = store.clone();
2736 let started = started.clone();
2737 async move {
2738 started.notify_one();
2739 store.load_runtime_pool_meta("test runtime pool metadata load").await
2740 }
2741 });
2742 started.notified().await;
2743 let save_paused = Arc::new(tokio::sync::Notify::new());
2744 let save_release = Arc::new(tokio::sync::Notify::new());
2745 let save_task = tokio::spawn({
2746 let newer = newer.clone();
2747 let pools = vec![
2748 Arc::new(PoolMetaCommitPauseStorage {
2749 inner: store.pools[0].clone(),
2750 pause_committed_pool_meta: true,
2751 paused: save_paused.clone(),
2752 release: save_release.clone(),
2753 }),
2754 Arc::new(PoolMetaCommitPauseStorage {
2755 inner: store.pools[1].clone(),
2756 pause_committed_pool_meta: false,
2757 paused: save_paused.clone(),
2758 release: save_release.clone(),
2759 }),
2760 ];
2761 async move { newer.save_for_startup(pools).await }
2762 });
2763 tokio::time::timeout(std::time::Duration::from_secs(5), save_paused.notified())
2764 .await
2765 .expect("the newer V3 generation should pause after the first replica commit");
2766 assert!(
2767 tokio::time::timeout(std::time::Duration::from_millis(100), &mut load_task)
2768 .await
2769 .is_err(),
2770 "runtime load must not observe a partially committed V3 generation while fenced"
2771 );
2772 let (committed_payload, _) = read_config_no_lock_preserve_empty_with_metadata(store.pools[0].clone(), POOL_META_NAME)
2773 .await
2774 .expect("first replica should be readable while the runtime read is fenced");
2775 let (pending_payload, _) = read_config_no_lock_preserve_empty_with_metadata(store.pools[1].clone(), POOL_META_NAME)
2776 .await
2777 .expect("second replica should be readable while the runtime read is fenced");
2778 assert_eq!(
2779 pool_meta_v3_commit_state_for_test(committed_payload).expect("first replica should be a V3 envelope"),
2780 (next_generation, true),
2781 "first replica should hold the committed new generation before the fence releases"
2782 );
2783 assert_eq!(
2784 pool_meta_v3_commit_state_for_test(pending_payload).expect("second replica should be a V3 envelope"),
2785 (next_generation, false),
2786 "second replica should still hold the pending new generation before the fence releases"
2787 );
2788
2789 save_release.notify_one();
2790 save_task
2791 .await
2792 .expect("newer V3 save task should not panic")
2793 .expect("the newer generation should finish while the runtime read is fenced");
2794 drop(pool_meta_guard);
2795
2796 let loaded = tokio::time::timeout(std::time::Duration::from_secs(5), load_task)
2797 .await
2798 .expect("runtime load should finish after commit publication")
2799 .expect("runtime load task should not panic")
2800 .expect("runtime load should select the completed snapshot");
2801 assert_eq!(loaded.pools[0].last_update, newer.pools[0].last_update);
2802 shutdown.cancel();
2803 }
2804
2805 #[tokio::test]
2806 #[serial_test::serial]
2807 async fn runtime_save_current_pool_meta_reaches_v3_cas_and_disarms_transaction() {
2808 let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-runtime-cas", &[2]).await;
2809 let bootstrapped = store
2810 .load_runtime_pool_meta("verify fresh bootstrap V3 CAS save")
2811 .await
2812 .expect("fresh startup should durably publish pool metadata");
2813 assert_eq!(bootstrapped.version, 3);
2814 let (bootstrap_payload, _) = read_config_no_lock_preserve_empty_with_metadata(store.pools[0].clone(), POOL_META_NAME)
2815 .await
2816 .expect("fresh bootstrap V3 replica should be readable");
2817 assert_eq!(
2818 pool_meta_v3_commit_state_for_test(bootstrap_payload).expect("fresh bootstrap V3 envelope should decode"),
2819 (1, true)
2820 );
2821
2822 let saved_at = OffsetDateTime::now_utc() + TimeDuration::seconds(30);
2823 {
2824 let mut pool_meta = store.pool_meta.write().await;
2825 pool_meta.pools[0].last_update = saved_at;
2826 }
2827
2828 store
2829 .save_current_pool_meta_for_test(&[0])
2830 .await
2831 .expect("runtime pool metadata save should reach V3 conditional writes");
2832 store
2833 .ensure_pool_meta_side_effects_safe("runtime save publication")
2834 .await
2835 .expect("successful runtime publication must disarm the transaction guard");
2836 let persisted = store
2837 .load_runtime_pool_meta("verify runtime V3 CAS save")
2838 .await
2839 .expect("runtime load should observe the committed save");
2840 assert_eq!(persisted.version, 3);
2841 assert_eq!(persisted.pools[0].last_update, saved_at);
2842 let (runtime_payload, _) = read_config_no_lock_preserve_empty_with_metadata(store.pools[0].clone(), POOL_META_NAME)
2843 .await
2844 .expect("runtime V3 replica should be readable");
2845 assert_eq!(
2846 pool_meta_v3_commit_state_for_test(runtime_payload).expect("runtime V3 envelope should decode"),
2847 (2, true)
2848 );
2849
2850 shutdown.cancel();
2851 }
2852
2853 #[tokio::test]
2854 #[serial_test::serial]
2855 async fn stale_clear_decommission_cannot_erase_active_or_queued_replacement() {
2856 let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-stale-clear", &[2, 2]).await;
2857 let replacement_time = OffsetDateTime::UNIX_EPOCH + TimeDuration::seconds(100);
2858 let cases = [
2859 (
2860 "failed-active",
2861 PoolDecommissionInfo {
2862 failed: true,
2863 ..Default::default()
2864 },
2865 PoolDecommissionInfo {
2866 start_time: Some(replacement_time),
2867 ..Default::default()
2868 },
2869 ),
2870 (
2871 "canceled-queued",
2872 PoolDecommissionInfo {
2873 canceled: true,
2874 ..Default::default()
2875 },
2876 PoolDecommissionInfo {
2877 queued: true,
2878 ..Default::default()
2879 },
2880 ),
2881 ];
2882
2883 for (case, stale_terminal, replacement_info) in cases {
2884 let mut replacement = store
2885 .load_runtime_pool_meta("prepare stale clear replacement")
2886 .await
2887 .expect("the current durable pool metadata should load");
2888 replacement.pools[0].last_update = replacement_time;
2889 replacement.pools[0].decommission = Some(replacement_info.clone());
2890 persist_reload_snapshot(&store, &replacement).await;
2891
2892 let mut stale_local = replacement.clone();
2893 stale_local.pools[0].last_update = OffsetDateTime::UNIX_EPOCH;
2894 stale_local.pools[0].decommission = Some(stale_terminal.clone());
2895 *store.pool_meta.write().await = stale_local;
2896
2897 let err = store
2898 .clear_decommission(0)
2899 .await
2900 .expect_err("a stale terminal clear must not erase a durable replacement");
2901 assert!(
2902 err.to_string()
2903 .contains("persisted active or queued decommission cannot be cleared"),
2904 "unexpected {case} rejection: {err}"
2905 );
2906
2907 let durable = store
2908 .load_runtime_pool_meta("verify stale clear replacement")
2909 .await
2910 .expect("the durable replacement should remain readable");
2911 let durable_info = durable.pools[0]
2912 .decommission
2913 .as_ref()
2914 .expect("the durable replacement must not be cleared");
2915 assert_eq!(durable.pools[0].last_update, replacement_time, "{case} replacement revision changed");
2916 assert_eq!(
2917 durable_info.start_time, replacement_info.start_time,
2918 "{case} replacement generation changed"
2919 );
2920 assert_eq!(durable_info.queued, replacement_info.queued, "{case} replacement queue state changed");
2921
2922 let local = store.pool_meta.read().await;
2923 let local_info = local.pools[0]
2924 .decommission
2925 .as_ref()
2926 .expect("the failed clear must roll back the stale local terminal state");
2927 assert_eq!(local_info.failed, stale_terminal.failed, "{case} failed state was not rolled back");
2928 assert_eq!(local_info.canceled, stale_terminal.canceled, "{case} canceled state was not rolled back");
2929 }
2930
2931 shutdown.cancel();
2932 }
2933
2934 #[tokio::test]
2935 #[serial_test::serial]
2936 async fn peer_pool_meta_reload_does_not_rollback_newer_local_states() {
2937 let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-stale", &[2]).await;
2938
2939 let stale_time = OffsetDateTime::now_utc();
2940 let newer_time = stale_time + TimeDuration::seconds(30);
2941 let progressed_states: [(&str, PoolDecommissionInfo); 4] = [
2942 (
2943 "queued",
2944 PoolDecommissionInfo {
2945 queued: true,
2946 start_time: Some(stale_time),
2947 ..Default::default()
2948 },
2949 ),
2950 (
2951 "canceled",
2952 PoolDecommissionInfo {
2953 canceled: true,
2954 start_time: Some(stale_time),
2955 ..Default::default()
2956 },
2957 ),
2958 (
2959 "failed",
2960 PoolDecommissionInfo {
2961 failed: true,
2962 start_time: Some(stale_time),
2963 ..Default::default()
2964 },
2965 ),
2966 (
2967 "complete",
2968 PoolDecommissionInfo {
2969 complete: true,
2970 start_time: Some(stale_time),
2971 ..Default::default()
2972 },
2973 ),
2974 ];
2975
2976 for (state_label, local_state) in progressed_states {
2977 {
2978 let mut pool_meta = store.pool_meta.write().await;
2979 *pool_meta = reload_test_pool_meta(reload_test_pool_status(Some(local_state.clone()), newer_time));
2980 }
2981
2982 let stale_snapshot = reload_test_pool_meta(reload_test_pool_status(
2984 Some(PoolDecommissionInfo {
2985 start_time: Some(stale_time),
2986 ..Default::default()
2987 }),
2988 stale_time,
2989 ));
2990 persist_reload_snapshot(&store, &stale_snapshot).await;
2991
2992 let merged_newer = store.reload_pool_meta().await.expect("stale reload should succeed");
2993 assert!(
2994 !merged_newer,
2995 "a delayed reload must not report merged newer state for the {state_label} progression"
2996 );
2997
2998 let pool_meta = store.pool_meta.read().await;
2999 let info = pool_meta.pools[0]
3000 .decommission
3001 .as_ref()
3002 .expect("local decommission state should survive a stale reload");
3003 assert_eq!(info.queued, local_state.queued, "{state_label} queued flag must not roll back");
3004 assert_eq!(info.canceled, local_state.canceled, "{state_label} canceled flag must not roll back");
3005 assert_eq!(info.failed, local_state.failed, "{state_label} failed flag must not roll back");
3006 assert_eq!(info.complete, local_state.complete, "{state_label} complete flag must not roll back");
3007 assert_eq!(
3008 pool_meta.pools[0].last_update, newer_time,
3009 "{state_label} progress timestamp must be kept"
3010 );
3011 }
3012
3013 shutdown.cancel();
3014 }
3015
3016 #[tokio::test]
3017 #[serial_test::serial]
3018 async fn peer_pool_meta_reload_merges_newer_state_and_is_idempotent_on_duplicate_delivery() {
3019 let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-duplicate", &[2]).await;
3020
3021 let older_time = OffsetDateTime::now_utc();
3022 let newer_time = older_time + TimeDuration::seconds(30);
3023
3024 {
3025 let mut pool_meta = store.pool_meta.write().await;
3026 *pool_meta = reload_test_pool_meta(reload_test_pool_status(
3027 Some(PoolDecommissionInfo {
3028 items_decommissioned: 1,
3029 ..Default::default()
3030 }),
3031 older_time,
3032 ));
3033 }
3034 let newer_snapshot = reload_test_pool_meta(reload_test_pool_status(
3035 Some(PoolDecommissionInfo {
3036 complete: true,
3037 items_decommissioned: 10,
3038 ..Default::default()
3039 }),
3040 newer_time,
3041 ));
3042 persist_reload_snapshot(&store, &newer_snapshot).await;
3043
3044 let merged_newer = store.reload_pool_meta().await.expect("first reload should succeed");
3045 assert!(merged_newer, "a strictly newer persisted snapshot must merge");
3046
3047 {
3048 let pool_meta = store.pool_meta.read().await;
3049 let info = pool_meta.pools[0].decommission.as_ref().expect("merged decommission state");
3050 assert!(info.complete);
3051 assert_eq!(info.items_decommissioned, 10);
3052 assert_eq!(pool_meta.pools[0].last_update, newer_time);
3053 }
3054
3055 let duplicate_merged = store.reload_pool_meta().await.expect("duplicate reload should succeed");
3057 assert!(!duplicate_merged, "a duplicate delivery must not re-apply merged state");
3058
3059 {
3060 let pool_meta = store.pool_meta.read().await;
3061 let info = pool_meta.pools[0].decommission.as_ref().expect("merged decommission state");
3062 assert!(info.complete);
3063 assert_eq!(info.items_decommissioned, 10);
3064 assert_eq!(pool_meta.pools[0].last_update, newer_time);
3065 }
3066
3067 shutdown.cancel();
3068 }
3069
3070 #[tokio::test]
3071 #[serial_test::serial]
3072 async fn peer_pool_meta_reload_keeps_active_worker_progress_over_newer_snapshot() {
3073 let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-worker", &[2]).await;
3074 *store.decommission_cancelers.write().await = vec![Some(crate::core::pools::DecommissionCanceler::new_for_test(
3075 CancellationToken::new(),
3076 ))];
3077
3078 let worker_time = OffsetDateTime::now_utc();
3079 let newer_time = worker_time + TimeDuration::seconds(30);
3080
3081 {
3082 let mut pool_meta = store.pool_meta.write().await;
3083 *pool_meta = reload_test_pool_meta(reload_test_pool_status(
3084 Some(PoolDecommissionInfo {
3085 start_time: Some(worker_time),
3086 items_decommissioned: 10,
3087 bytes_done: 1_024,
3088 ..Default::default()
3089 }),
3090 worker_time,
3091 ));
3092 }
3093 let newer_terminal_snapshot = reload_test_pool_meta(reload_test_pool_status(
3095 Some(PoolDecommissionInfo {
3096 complete: true,
3097 ..Default::default()
3098 }),
3099 newer_time,
3100 ));
3101 persist_reload_snapshot(&store, &newer_terminal_snapshot).await;
3102
3103 let merged_newer = store
3104 .reload_pool_meta()
3105 .await
3106 .expect("reload under an active worker should succeed");
3107 assert!(!merged_newer, "an active local worker must block snapshot replacement");
3108
3109 let pool_meta = store.pool_meta.read().await;
3110 let info = pool_meta.pools[0]
3111 .decommission
3112 .as_ref()
3113 .expect("worker progress should remain");
3114 assert!(!info.complete);
3115 assert_eq!(info.items_decommissioned, 10);
3116 assert_eq!(info.bytes_done, 1_024);
3117 assert_eq!(pool_meta.pools[0].last_update, worker_time);
3118
3119 shutdown.cancel();
3120 }
3121
3122 #[tokio::test]
3123 #[serial_test::serial]
3124 async fn peer_pool_meta_reload_latches_recovery_when_initialized_metadata_is_missing() {
3125 let (temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-missing", &[2]).await;
3126
3127 let kept_time = OffsetDateTime::now_utc();
3128 {
3129 let mut pool_meta = store.pool_meta.write().await;
3130 *pool_meta = reload_test_pool_meta(reload_test_pool_status(
3131 Some(PoolDecommissionInfo {
3132 complete: true,
3133 ..Default::default()
3134 }),
3135 kept_time,
3136 ));
3137 }
3138 persist_reload_snapshot(
3140 &store,
3141 &reload_test_pool_meta(reload_test_pool_status(
3142 Some(PoolDecommissionInfo {
3143 complete: true,
3144 ..Default::default()
3145 }),
3146 kept_time,
3147 )),
3148 )
3149 .await;
3150
3151 let mut deleted_any = false;
3152 for disk_index in 0..2 {
3153 let pool_bin_dir = temp_dir
3154 .path()
3155 .join(format!("pool0-disk{disk_index}"))
3156 .join(crate::disk::RUSTFS_META_BUCKET)
3157 .join(crate::core::pools::POOL_META_NAME);
3158 if pool_bin_dir.exists() {
3159 tokio::fs::remove_dir_all(&pool_bin_dir)
3160 .await
3161 .expect("persisted pool metadata object dir should be removable");
3162 deleted_any = true;
3163 }
3164 }
3165 if !deleted_any {
3168 panic!("no pool.bin found under {:?}", temp_dir.path());
3169 }
3170
3171 let err = store
3172 .reload_pool_meta()
3173 .await
3174 .expect_err("an initialized cluster with every pool.bin missing must require recovery");
3175 assert!(err.to_string().contains("initialized cluster identity exists"));
3176 store
3177 .ensure_pool_meta_side_effects_safe("test reload side effect")
3178 .await
3179 .expect_err("the recovery-required reload must latch the runtime side-effect gate");
3180
3181 let pool_meta = store.pool_meta.read().await;
3182 let info = pool_meta.pools[0]
3183 .decommission
3184 .as_ref()
3185 .expect("missing persisted metadata must not default local state away");
3186 assert!(info.complete);
3187 assert_eq!(pool_meta.pools[0].last_update, kept_time);
3188
3189 shutdown.cancel();
3190 }
3191
3192 #[tokio::test]
3193 #[serial_test::serial]
3194 async fn peer_pool_meta_reload_latches_recovery_after_identity_epoch_conflict() {
3195 let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-identity-conflict", &[2, 2]).await;
3196 let conflicting_identity =
3197 initialized_pool_meta_identity_for_test(store.id, 2).expect("conflicting initialized identity should encode");
3198 save_config(store.pools[1].clone(), POOL_META_IDENTITY_NAME, conflicting_identity)
3199 .await
3200 .expect("second pool identity should be replaced for the conflict scenario");
3201
3202 let err = store
3203 .reload_pool_meta()
3204 .await
3205 .expect_err("divergent identity epochs must reject runtime reload");
3206 assert!(
3207 err.to_string()
3208 .contains("identity replicas disagree on cluster identity or epoch")
3209 );
3210 store
3211 .ensure_pool_meta_side_effects_safe("identity epoch conflict side effect")
3212 .await
3213 .expect_err("identity epoch conflict must latch the runtime side-effect gate");
3214
3215 shutdown.cancel();
3216 }
3217
3218 #[tokio::test]
3219 #[serial_test::serial]
3220 async fn peer_pool_meta_reload_latches_recovery_after_future_identity_version() {
3221 let (_temp_dir, store, shutdown) = setup_multi_pool_test_store("pool-meta-reload-future-identity", &[2, 2]).await;
3222 let (mut future_identity, _) =
3223 read_config_no_lock_preserve_empty_with_metadata(store.pools[1].clone(), POOL_META_IDENTITY_NAME)
3224 .await
3225 .expect("current identity should be readable");
3226 let current_version = u16::from_le_bytes([future_identity[2], future_identity[3]]);
3227 let future_version = current_version
3228 .checked_add(1)
3229 .expect("identity version should have a future value");
3230 future_identity[2..4].copy_from_slice(&future_version.to_le_bytes());
3231 save_config(store.pools[1].clone(), POOL_META_IDENTITY_NAME, future_identity)
3232 .await
3233 .expect("second pool identity should be replaced with a future version");
3234
3235 let err = store
3236 .reload_pool_meta()
3237 .await
3238 .expect_err("a future identity version must reject runtime reload");
3239 assert!(err.to_string().contains("pool metadata incompatible"));
3240 store
3241 .ensure_pool_meta_side_effects_safe("future identity version side effect")
3242 .await
3243 .expect_err("future identity version must latch the runtime side-effect gate");
3244
3245 shutdown.cancel();
3246 }
3247}