1use super::meta::{
2 RebalanceMetaMergeOutcome, clone_first_arc, clone_rebalance_pool_stats, defer_bucket_in_rebalance_queue,
3 ensure_valid_rebalance_pool_index, invalid_rebalance_pool_index_error, is_rebalance_actively_running,
4 is_rebalance_conflicting_with_decommission, mark_rebalance_bucket_done, merge_rebalance_meta, percent_free_ratio,
5 rebalance_metadata_not_initialized_error, record_rebalance_cleanup_warning_in_meta,
6 record_rebalance_stop_propagation_snapshot, resolve_next_rebalance_bucket, rollback_rebalance_start_meta_snapshot_for_id,
7 should_accept_rebalance_stats_update, should_pool_participate, stop_rebalance_meta_snapshot_for_id,
8 validate_init_rebalance_state,
9};
10use super::worker::{
11 rebalance_max_attempts, rebalance_meta_lock_error, resolve_load_rebalance_stats_update_result,
12 resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result, retry_rebalance_metadata_access,
13};
14use super::{
15 DiskStat, EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
16 RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord,
17 encode_rebalance_stop_propagation_record,
18};
19use crate::core::pools::{
20 PoolMeta, PoolRebalanceActivationFence, acquire_pool_activation_fleet_proof, acquire_pool_rebalance_activation_locks,
21 pool_meta_has_active_decommission,
22};
23use crate::error::{Error, Result};
24use crate::object_api::ObjectOptions;
25use crate::set_disk::get_lock_acquire_timeout;
26use crate::storage_api_contracts::{
27 admin::StorageAdminApi,
28 namespace::NamespaceLocking as StorageNamespaceLocking,
29 object::{EcstoreObjectIO, EcstoreObjectOperations},
30};
31use crate::store::ECStore;
32use rustfs_filemeta::FileInfo;
33use std::sync::Arc;
34use time::OffsetDateTime;
35use tracing::{debug, info};
36use uuid::Uuid;
37
38#[cfg(test)]
39static FAIL_NEXT_REBALANCE_ACTIVATION_SAVE: std::sync::Mutex<Option<String>> = std::sync::Mutex::new(None);
40
41#[cfg(test)]
42static REBALANCE_DISK_STATS_OVERRIDES: std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<Uuid, Vec<DiskStat>>>> =
43 std::sync::OnceLock::new();
44
45#[cfg(test)]
46pub(super) fn fail_next_rebalance_activation_save_for_test(rebalance_id: &str) {
47 *FAIL_NEXT_REBALANCE_ACTIVATION_SAVE
48 .lock()
49 .expect("rebalance activation save failure hook should not be poisoned") = Some(rebalance_id.to_string());
50}
51
52#[cfg(test)]
53fn set_rebalance_disk_stats_override_for_test(store_id: Uuid, disk_stats: Vec<DiskStat>) {
54 REBALANCE_DISK_STATS_OVERRIDES
55 .get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
56 .lock()
57 .expect("rebalance disk stats override should not be poisoned")
58 .insert(store_id, disk_stats);
59}
60
61#[cfg(test)]
62fn take_rebalance_disk_stats_override_for_test(store_id: Uuid) -> Option<Vec<DiskStat>> {
63 REBALANCE_DISK_STATS_OVERRIDES
64 .get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
65 .lock()
66 .expect("rebalance disk stats override should not be poisoned")
67 .remove(&store_id)
68}
69
70fn ensure_rebalance_activation_pool_meta_allowed(meta: &PoolMeta) -> Result<()> {
71 if pool_meta_has_active_decommission(meta) {
72 return Err(Error::DecommissionAlreadyRunning);
73 }
74
75 Ok(())
76}
77
78pub(super) enum RebalanceWorkerActivationFence {
79 Ready(Box<PoolRebalanceActivationFence>),
80 NotStartedTerminal,
81}
82
83pub(super) struct RebalanceRunGuard {
84 _local_guard: tokio::sync::OwnedRwLockReadGuard<()>,
85 persisted_guard: rustfs_lock::NamespaceLockGuard,
86}
87
88#[cfg(any(test, feature = "test-util"))]
89struct RebalanceStopWaitProbeState {
90 expected_id: String,
91 attempted: std::sync::atomic::AtomicBool,
92 notify: tokio::sync::Notify,
93}
94
95#[cfg(any(test, feature = "test-util"))]
96static REBALANCE_STOP_WAIT_PROBES: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<RebalanceStopWaitProbeState>>>> =
97 std::sync::OnceLock::new();
98
99#[cfg(any(test, feature = "test-util"))]
100pub(super) struct RebalanceStopWaitProbe {
101 state: Arc<RebalanceStopWaitProbeState>,
102}
103
104#[cfg(any(test, feature = "test-util"))]
105impl RebalanceStopWaitProbe {
106 pub(super) fn install(expected_id: &str) -> Self {
107 let state = Arc::new(RebalanceStopWaitProbeState {
108 expected_id: expected_id.to_string(),
109 attempted: std::sync::atomic::AtomicBool::new(false),
110 notify: tokio::sync::Notify::new(),
111 });
112 REBALANCE_STOP_WAIT_PROBES
113 .get_or_init(|| std::sync::Mutex::new(Vec::new()))
114 .lock()
115 .expect("rebalance stop wait probe should not be poisoned")
116 .push(Arc::clone(&state));
117 Self { state }
118 }
119
120 pub(super) async fn wait_until_attempted(&self) {
121 tokio::time::timeout(std::time::Duration::from_secs(30), async {
122 while !self.state.attempted.load(std::sync::atomic::Ordering::Acquire) {
123 self.state.notify.notified().await;
124 }
125 })
126 .await
127 .expect("rebalance stop should reach the deterministic activation wait probe");
128 }
129}
130
131#[cfg(any(test, feature = "test-util"))]
132impl Drop for RebalanceStopWaitProbe {
133 fn drop(&mut self) {
134 let mut probes = REBALANCE_STOP_WAIT_PROBES
135 .get_or_init(|| std::sync::Mutex::new(Vec::new()))
136 .lock()
137 .expect("rebalance stop wait probe should not be poisoned");
138 probes.retain(|state| !Arc::ptr_eq(state, &self.state));
139 }
140}
141
142#[cfg(any(test, feature = "test-util"))]
143fn observe_rebalance_stop_wait_attempt(expected_id: Option<&str>) {
144 let probes = REBALANCE_STOP_WAIT_PROBES
145 .get_or_init(|| std::sync::Mutex::new(Vec::new()))
146 .lock()
147 .expect("rebalance stop wait probe should not be poisoned")
148 .clone();
149 for state in probes {
150 if expected_id == Some(state.expected_id.as_str()) {
151 state.attempted.store(true, std::sync::atomic::Ordering::Release);
152 state.notify.notify_one();
153 }
154 }
155}
156
157impl RebalanceRunGuard {
158 pub(super) fn ensure_held(&self, stage: &str) -> Result<()> {
159 #[cfg(test)]
160 let forced_lost = self
161 .persisted_guard
162 .lock_lost_signal()
163 .is_some_and(|signal| crate::object_api::namespace_lock_signal_test_fence_is_lost(&signal));
164 #[cfg(not(test))]
165 let forced_lost = false;
166 if self.persisted_guard.is_lock_lost() || forced_lost {
167 return Err(Error::other(format!("rebalance distributed run fence lost during {stage}")));
168 }
169 Ok(())
170 }
171
172 pub(super) fn lock_lost_signal(&self) -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
173 self.persisted_guard.lock_lost_signal()
174 }
175}
176
177async fn acquire_persisted_rebalance_run_guard<S>(
178 pool: Arc<S>,
179 expected_id: &str,
180 stage: &str,
181) -> Result<rustfs_lock::NamespaceLockGuard>
182where
183 S: EcstoreObjectIO
184 + EcstoreObjectOperations
185 + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
186{
187 let ns_lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
188 let guard = ns_lock
189 .get_read_lock(get_lock_acquire_timeout())
190 .await
191 .map_err(|err| rebalance_meta_lock_error(err, "read"))?;
192 let mut opts = ObjectOptions {
193 no_lock: true,
194 ..Default::default()
195 };
196 opts.add_namespace_lock_guard(&guard);
197 let object_info = pool
198 .get_object_info(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME, &opts)
199 .await;
200 let persisted_run_id = match object_info {
201 Ok(object_info) => {
202 let metadata = object_info.user_defined.as_ref();
203 let run_id = rustfs_utils::http::metadata_compat::get_consistent_str(
204 metadata,
205 rustfs_utils::http::metadata_compat::SUFFIX_REBALANCE_RUN_ID,
206 );
207 if run_id.is_none()
208 && rustfs_utils::http::metadata_compat::contains_key_str(
209 metadata,
210 rustfs_utils::http::metadata_compat::SUFFIX_REBALANCE_RUN_ID,
211 )
212 {
213 return Err(Error::other(format!("rebalance run marker is inconsistent during {stage}")));
214 }
215 run_id.map(str::to_string)
216 }
217 Err(err) if crate::error::is_err_object_not_found(&err) => None,
218 Err(err) => return Err(err),
219 };
220 if let Some(persisted_run_id) = persisted_run_id {
221 if persisted_run_id != expected_id {
222 return Err(Error::other(format!(
223 "stale rebalance run rejected during {stage}: expected {expected_id}, found {persisted_run_id}"
224 )));
225 }
226 } else {
227 let mut persisted = RebalanceMeta::new();
230 persisted.load_with_opts(pool, opts).await?;
231 ensure_rebalance_worker_active(Some(&persisted), expected_id, stage)?;
232 }
233 if guard.is_lock_lost() {
234 return Err(Error::other(format!("rebalance distributed run fence lost during {stage}")));
235 }
236 Ok(guard)
237}
238
239pub(super) async fn merge_and_save_rebalance_meta_no_lock<S>(
240 pool: Arc<S>,
241 local_snapshot: &RebalanceMeta,
242 stage: &str,
243 mut opts: ObjectOptions,
244 activation_fence: Option<&PoolRebalanceActivationFence>,
245 expected_id: Option<&str>,
246) -> Result<()>
247where
248 S: EcstoreObjectIO,
249{
250 let mut merged = RebalanceMeta::new();
251 match merged.load_with_opts(pool.clone(), opts.clone()).await {
252 Ok(()) => {
253 if let Some(expected_id) = expected_id {
254 ensure_rebalance_run_id(Some(&merged), expected_id, stage)?;
255 }
256 if merge_rebalance_meta(&mut merged, local_snapshot) == RebalanceMetaMergeOutcome::RejectedActiveConflict {
257 return Err(Error::RebalanceAlreadyRunning);
258 }
259 }
260 Err(Error::ConfigNotFound) => {
261 if expected_id.is_some() {
262 return Err(rebalance_metadata_not_initialized_error(stage));
263 }
264 merged = local_snapshot.clone();
265 }
266 Err(err) => return Err(Error::other(format!("rebalance meta load before save failed during {stage}: {err}"))),
267 }
268
269 if let Some(fence) = activation_fence {
270 fence.add_namespace_lock_fence(&mut opts);
271 fence.ensure_held()?;
272 }
273 #[cfg(test)]
274 let barrier_pool = pool.clone();
275 merged.save_with_opts(pool, opts).await?;
276 #[cfg(test)]
277 if let Some(fence) = activation_fence {
278 crate::core::pools::pause_pool_activation_after_durable_save(&barrier_pool, fence).await;
279 }
280 Ok(())
283}
284
285pub(super) fn ensure_rebalance_run_id(meta: Option<&RebalanceMeta>, expected_id: &str, stage: &str) -> Result<()> {
286 let Some(meta) = meta else {
287 return Err(rebalance_metadata_not_initialized_error(stage));
288 };
289 if meta.id != expected_id {
290 return Err(Error::other(format!(
291 "stale rebalance worker rejected during {stage}: expected {expected_id}, found {}",
292 meta.id
293 )));
294 }
295 Ok(())
296}
297
298pub(super) fn ensure_rebalance_worker_active(meta: Option<&RebalanceMeta>, expected_id: &str, stage: &str) -> Result<()> {
299 ensure_rebalance_run_id(meta, expected_id, stage)?;
300 let Some(meta) = meta else {
301 return Err(rebalance_metadata_not_initialized_error(stage));
302 };
303 if meta.stopped_at.is_some() || meta.stop_requested {
304 return Err(Error::OperationCanceled);
305 }
306 if meta
307 .cancel
308 .as_ref()
309 .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
310 || !is_rebalance_conflicting_with_decommission(meta)
311 {
312 return Err(Error::other(format!("inactive rebalance worker rejected during {stage}: {expected_id}")));
313 }
314 Ok(())
315}
316
317pub(super) fn validate_rebalance_disk_stats_coverage(disk_stats: &[DiskStat]) -> Result<()> {
318 for (idx, disk_stat) in disk_stats.iter().enumerate() {
319 if disk_stat.total_space == 0 {
320 return Err(Error::other(format!(
321 "rebalance storage info is incomplete: pool {idx} has no reported capacity"
322 )));
323 }
324 }
325
326 Ok(())
327}
328
329fn pool_rebalance_status_from_meta(meta: Option<&RebalanceMeta>, pool_index: usize) -> (RebalStatus, bool) {
330 meta.and_then(|meta| meta.pool_stats.get(pool_index))
331 .filter(|pool_stat| pool_stat.participating)
332 .map(|pool_stat| (pool_stat.info.status, pool_stat.info.stopping))
333 .unwrap_or_default()
334}
335
336fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted: RebalanceMeta) -> bool {
337 if persisted.id.is_empty() && persisted.pool_stats.is_empty() {
338 return clear_rebalance_status_refresh(current);
339 }
340
341 let before = current.clone();
342 match current.as_mut() {
343 Some(current_meta) => {
344 if merge_rebalance_meta(current_meta, &persisted) == RebalanceMetaMergeOutcome::RejectedActiveConflict
345 && !is_rebalance_actively_running(current_meta)
346 {
347 *current = Some(persisted);
348 }
349 }
350 None => {
351 *current = Some(persisted);
352 }
353 }
354
355 match (before.as_ref(), current.as_ref()) {
356 (None, None) => false,
357 (None, Some(_)) | (Some(_), None) => true,
358 (Some(before), Some(after)) => rebalance_movement_snapshot_changed(Some(before), after),
359 }
360}
361
362fn clear_rebalance_status_refresh(current: &mut Option<RebalanceMeta>) -> bool {
363 if current.as_ref().is_none_or(|meta| !is_rebalance_actively_running(meta)) {
364 current.take().is_some()
365 } else {
366 false
367 }
368}
369
370fn rebalance_movement_snapshot_changed(current: Option<&RebalanceMeta>, persisted: &RebalanceMeta) -> bool {
371 let Some(current) = current else {
372 return true;
373 };
374
375 current.id != persisted.id
376 || current.stopped_at != persisted.stopped_at
377 || current.pool_stats.len() != persisted.pool_stats.len()
378 || current
379 .pool_stats
380 .iter()
381 .zip(persisted.pool_stats.iter())
382 .any(|(current, persisted)| {
383 current.participating != persisted.participating
384 || current.info.status != persisted.info.status
385 || current.info.stopping != persisted.info.stopping
386 })
387}
388
389impl ECStore {
390 pub(super) async fn rebalance_activation_write_guard(
393 &self,
394 expected_id: Option<&str>,
395 stage: &str,
396 ) -> Result<Option<tokio::sync::OwnedRwLockWriteGuard<()>>> {
397 let activation_gate = {
398 let meta = self.rebalance_meta.read().await;
399 if let Some(expected_id) = expected_id {
400 ensure_rebalance_run_id(meta.as_ref(), expected_id, stage)?;
401 }
402 meta.as_ref().map(|meta| Arc::clone(&meta.activation_gate))
403 };
404 Ok(match activation_gate {
405 Some(gate) => Some(gate.write_owned().await),
406 None => None,
407 })
408 }
409
410 pub(super) async fn rebalance_run_guard(&self, expected_id: &str, stage: &str) -> Result<RebalanceRunGuard> {
411 let activation_gate = {
413 let meta = self.rebalance_meta.read().await;
414 ensure_rebalance_worker_active(meta.as_ref(), expected_id, stage)?;
415 Arc::clone(
416 &meta
417 .as_ref()
418 .ok_or_else(|| rebalance_metadata_not_initialized_error(stage))?
419 .activation_gate,
420 )
421 };
422 let guard = Arc::clone(&activation_gate).read_owned().await;
423 let meta = self.rebalance_meta.read().await;
424 ensure_rebalance_worker_active(meta.as_ref(), expected_id, stage)?;
425 let current_gate = &meta
426 .as_ref()
427 .ok_or_else(|| rebalance_metadata_not_initialized_error(stage))?
428 .activation_gate;
429 if !Arc::ptr_eq(&activation_gate, current_gate) {
430 return Err(Error::other(format!(
431 "stale rebalance activation gate rejected during {stage}: {expected_id}"
432 )));
433 }
434 drop(meta);
435 let pool = clone_first_arc(self.pools.as_slice(), "rebalance run fence: no pools available")?;
436 let persisted_guard = acquire_persisted_rebalance_run_guard(pool, expected_id, stage).await?;
437 Ok(RebalanceRunGuard {
438 _local_guard: guard,
439 persisted_guard,
440 })
441 }
442
443 pub(super) async fn save_rebalance_meta_with_merge<S>(
444 &self,
445 pool: Arc<S>,
446 local_snapshot: &RebalanceMeta,
447 stage: &str,
448 ) -> Result<()>
449 where
450 S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
451 {
452 self.save_rebalance_meta_with_merge_for_id(pool, local_snapshot, stage, None)
453 .await
454 }
455
456 pub(super) async fn save_rebalance_meta_for_id_with_merge<S>(
457 &self,
458 pool: Arc<S>,
459 local_snapshot: &RebalanceMeta,
460 stage: &str,
461 expected_id: &str,
462 ) -> Result<()>
463 where
464 S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
465 {
466 self.save_rebalance_meta_with_merge_for_id(pool, local_snapshot, stage, Some(expected_id))
467 .await
468 }
469
470 pub(super) async fn save_rebalance_meta_under_activation_fence<S>(
471 &self,
472 pool: Arc<S>,
473 local_snapshot: &RebalanceMeta,
474 stage: &str,
475 activation_fence: &PoolRebalanceActivationFence,
476 expected_id: &str,
477 ) -> Result<()>
478 where
479 S: EcstoreObjectIO,
480 {
481 #[cfg(test)]
482 {
483 let mut fail_id = FAIL_NEXT_REBALANCE_ACTIVATION_SAVE
484 .lock()
485 .expect("rebalance activation save failure hook should not be poisoned");
486 if fail_id.as_deref() == Some(local_snapshot.id.as_str()) {
487 fail_id.take();
488 return Err(Error::other("injected rebalance activation save failure"));
489 }
490 }
491 merge_and_save_rebalance_meta_no_lock(
492 pool,
493 local_snapshot,
494 stage,
495 ObjectOptions {
496 no_lock: true,
497 ..Default::default()
498 },
499 Some(activation_fence),
500 Some(expected_id),
501 )
502 .await
503 }
504
505 async fn save_rebalance_meta_with_merge_for_id<S>(
506 &self,
507 pool: Arc<S>,
508 local_snapshot: &RebalanceMeta,
509 stage: &str,
510 expected_id: Option<&str>,
511 ) -> Result<()>
512 where
513 S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
514 {
515 let ns_lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
516 let guard = retry_rebalance_metadata_access(None, rebalance_max_attempts(), || async {
517 ns_lock
518 .get_write_lock(get_lock_acquire_timeout())
519 .await
520 .map_err(|err| rebalance_meta_lock_error(err, "write"))
521 })
522 .await?;
523 let mut opts = ObjectOptions {
524 no_lock: true,
525 ..Default::default()
526 };
527 opts.add_namespace_lock_guard(&guard);
528
529 merge_and_save_rebalance_meta_no_lock(pool, local_snapshot, stage, opts, None, expected_id).await?;
530 if guard.is_lock_lost() {
531 return Err(Error::other("rebalance metadata lock lost during metadata commit"));
532 }
533 Ok(())
534 }
535
536 async fn save_rebalance_activation_meta_with_merge<S>(
537 &self,
538 pool: Arc<S>,
539 local_snapshot: &RebalanceMeta,
540 stage: &str,
541 ) -> Result<()>
542 where
543 S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
544 {
545 #[cfg(test)]
546 crate::core::pools::observe_pool_activation_start_attempt(crate::core::pools::PoolActivationStartKind::Rebalance);
547 let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
548 let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
549 pool_meta_guard.ensure_write_safe(stage)?;
550 let activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), fleet_proof).await?;
551 let pool_meta = self
552 .load_runtime_pool_meta_under_activation_fence(&mut pool_meta_guard, &activation_fence, stage)
553 .await?;
554 ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?;
555
556 merge_and_save_rebalance_meta_no_lock(
557 pool,
558 local_snapshot,
559 stage,
560 ObjectOptions {
561 no_lock: true,
562 ..Default::default()
563 },
564 Some(&activation_fence),
565 None,
566 )
567 .await
568 }
569
570 pub(super) async fn fence_rebalance_worker_activation<S>(
571 &self,
572 pool: Arc<S>,
573 expected_id: &str,
574 ) -> Result<RebalanceWorkerActivationFence>
575 where
576 S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
577 {
578 let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
580 pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
581 let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
585 let pool_meta = self
586 .load_runtime_pool_meta_under_activation_fence(&mut pool_meta_guard, &activation_fence, "rebalance worker activation")
587 .await?;
588 ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?;
589
590 let mut persisted = RebalanceMeta::new();
591 persisted
592 .load_with_opts(
593 pool,
594 ObjectOptions {
595 no_lock: true,
596 ..Default::default()
597 },
598 )
599 .await?;
600 if persisted.id != expected_id {
601 return Err(Error::other(format!(
602 "rebalance metadata changed before worker activation: expected {expected_id}, found {}",
603 persisted.id
604 )));
605 }
606
607 activation_fence.ensure_held()?;
608 if !crate::services::rebalance::rebalance_requires_worker_activation(&persisted) {
609 return Ok(RebalanceWorkerActivationFence::NotStartedTerminal);
610 }
611
612 let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
616 activation_fence.set_fleet_proof(fleet_proof);
617 activation_fence.ensure_held()?;
618
619 Ok(RebalanceWorkerActivationFence::Ready(Box::new(activation_fence)))
620 }
621
622 #[tracing::instrument(skip_all)]
623 pub async fn load_rebalance_meta(&self) -> Result<()> {
624 let _start_guard = self.start_gate.lock().await;
625 self.load_rebalance_meta_under_start_gate().await
626 }
627
628 pub async fn prepare_rebalance_stop(&self) -> Result<Option<String>> {
630 let _start_guard = self.start_gate.lock().await;
631
632 {
633 let mut rebalance_meta = self.rebalance_meta.write().await;
634 if let Some(meta) = rebalance_meta.as_mut()
635 && is_rebalance_conflicting_with_decommission(meta)
636 {
637 meta.stop_requested = true;
638 meta.cancel
639 .get_or_insert_with(tokio_util::sync::CancellationToken::new)
640 .cancel();
641 #[cfg(any(test, feature = "test-util"))]
642 observe_rebalance_stop_wait_attempt(Some(meta.id.as_str()));
643 }
644 }
645
646 self.load_rebalance_meta_under_start_gate().await?;
647
648 let mut rebalance_meta = self.rebalance_meta.write().await;
649 let Some(meta) = rebalance_meta.as_mut() else {
650 return Ok(None);
651 };
652 if meta.stopped_at.is_some() || (!is_rebalance_conflicting_with_decommission(meta) && !meta.stop_requested) {
653 return Ok(None);
654 }
655 if meta.id.is_empty() {
656 return Err(Error::other("active rebalance metadata has no activation id"));
657 }
658 meta.stop_requested = true;
659 meta.cancel
660 .get_or_insert_with(tokio_util::sync::CancellationToken::new)
661 .cancel();
662 Ok(Some(meta.id.clone()))
663 }
664
665 pub(crate) async fn load_rebalance_meta_under_start_gate(&self) -> Result<()> {
666 let mut meta = RebalanceMeta::new();
667 debug!(
668 event = EVENT_REBALANCE_STATE,
669 component = LOG_COMPONENT_ECSTORE,
670 subsystem = LOG_SUBSYSTEM_REBALANCE,
671 state = "metadata_loading",
672 "Loading rebalance metadata"
673 );
674 let pool = clone_first_arc(&self.pools, "rebalanceMeta: no pools available")?;
675 let _activation_guard = self.rebalance_activation_write_guard(None, "load rebalance metadata").await?;
676 let movement_gate = self.ctx.data_movement_operation_gate();
677 let _movement_guard = movement_gate.write().await;
678 let loaded = resolve_rebalance_meta_load_result(meta.load(pool).await)?;
679 if loaded {
680 let movement_changed = rebalance_movement_snapshot_changed(self.rebalance_meta.read().await.as_ref(), &meta);
681 {
682 let mut rebalance_meta = self.rebalance_meta.write().await;
683 if let Some(current) = rebalance_meta.as_ref()
684 && current.id == meta.id
685 {
686 meta.cancel = current.cancel.clone();
687 meta.activation_gate = Arc::clone(¤t.activation_gate);
688 meta.stop_requested = current.stop_requested;
689 }
690 *rebalance_meta = Some(meta);
691
692 drop(rebalance_meta);
693 }
694
695 if movement_changed {
696 self.ctx.advance_data_movement_operation_epoch();
697 }
698 drop(_movement_guard);
699 resolve_load_rebalance_stats_update_result(self.update_rebalance_stats().await)?;
700 debug!(
701 event = EVENT_REBALANCE_STATE,
702 component = LOG_COMPONENT_ECSTORE,
703 subsystem = LOG_SUBSYSTEM_REBALANCE,
704 state = "metadata_loaded",
705 "Loaded rebalance metadata"
706 );
707 } else {
708 let movement_changed = self.rebalance_meta.read().await.is_some();
709 {
710 let mut rebalance_meta = self.rebalance_meta.write().await;
711 *rebalance_meta = None;
712 }
713 if movement_changed {
714 self.ctx.advance_data_movement_operation_epoch();
715 }
716 drop(_movement_guard);
717 debug!(
718 event = EVENT_REBALANCE_STATE,
719 component = LOG_COMPONENT_ECSTORE,
720 subsystem = LOG_SUBSYSTEM_REBALANCE,
721 state = "metadata_missing",
722 reason = "rebalance_not_started",
723 "Rebalance metadata not found"
724 );
725 }
726
727 Ok(())
728 }
729
730 #[tracing::instrument(skip_all)]
731 pub async fn refresh_rebalance_status_meta(&self) -> Result<()> {
732 let _start_guard = self.start_gate.lock().await;
733 let pool = clone_first_arc(&self.pools, "refresh_rebalance_status_meta: no pools available")?;
734 let mut persisted = RebalanceMeta::new();
735 let _activation_guard = self
736 .rebalance_activation_write_guard(None, "refresh rebalance metadata")
737 .await?;
738 let movement_gate = self.ctx.data_movement_operation_gate();
739 let _movement_guard = movement_gate.write().await;
740 let loaded = persisted.load(pool).await;
741 match loaded {
742 Ok(()) => {
743 let mut rebalance_meta = self.rebalance_meta.write().await;
744 if merge_rebalance_status_refresh(&mut rebalance_meta, persisted) {
745 self.ctx.advance_data_movement_operation_epoch();
746 }
747 }
748 Err(Error::ConfigNotFound) => {
749 let mut rebalance_meta = self.rebalance_meta.write().await;
750 if clear_rebalance_status_refresh(&mut rebalance_meta) {
751 self.ctx.advance_data_movement_operation_epoch();
752 }
753 }
754 Err(err) => {
755 return Err(Error::other(format!("rebalance metadata refresh failed during pool status: {err}")));
756 }
757 }
758
759 Ok(())
760 }
761
762 #[tracing::instrument(skip_all)]
763 pub async fn update_rebalance_stats(&self) -> Result<()> {
764 let mut ok = false;
765
766 let pool_stats = {
767 let rebalance_meta = self.rebalance_meta.read().await;
768 clone_rebalance_pool_stats(rebalance_meta.as_ref())?
769 };
770
771 debug!(
772 event = EVENT_REBALANCE_STATE,
773 component = LOG_COMPONENT_ECSTORE,
774 subsystem = LOG_SUBSYSTEM_REBALANCE,
775 pool_count = pool_stats.len(),
776 "Refreshing rebalance stats snapshot"
777 );
778
779 for i in 0..self.pools.len() {
780 if pool_stats.get(i).is_none() {
781 let mut rebalance_meta = self.rebalance_meta.write().await;
782 debug!(
783 event = EVENT_REBALANCE_STATE,
784 component = LOG_COMPONENT_ECSTORE,
785 subsystem = LOG_SUBSYSTEM_REBALANCE,
786 pool_index = i,
787 state = "pool_stat_missing",
788 "Adding missing rebalance pool stats entry"
789 );
790 if let Some(meta) = rebalance_meta.as_mut() {
791 meta.pool_stats.push(RebalanceStats::default());
792 }
793 ok = true;
794 drop(rebalance_meta);
795 }
796 }
797
798 if ok {
799 debug!(
800 event = EVENT_REBALANCE_STATE,
801 component = LOG_COMPONENT_ECSTORE,
802 subsystem = LOG_SUBSYSTEM_REBALANCE,
803 state = "metadata_saving",
804 "Saving rebalance metadata after stats refresh"
805 );
806
807 let rebalance_meta = self.rebalance_meta.read().await;
808 if let Some(meta) = rebalance_meta.as_ref() {
809 let pool = clone_first_arc(&self.pools, "update_rebalance_stats: no pools available")?;
810 resolve_rebalance_meta_save_result(
811 self.save_rebalance_meta_for_id_with_merge(pool, meta, "update_rebalance_stats", meta.id.as_str())
812 .await,
813 "update_rebalance_stats",
814 )?;
815 }
816 }
817
818 Ok(())
819 }
820
821 #[tracing::instrument(skip(self))]
822 pub async fn init_rebalance_meta(&self, bucktes: Vec<String>) -> Result<String> {
823 info!(
824 event = EVENT_REBALANCE_STATE,
825 component = LOG_COMPONENT_ECSTORE,
826 subsystem = LOG_SUBSYSTEM_REBALANCE,
827 state = "initializing",
828 bucket_count = bucktes.len(),
829 "Initializing rebalance metadata"
830 );
831 let si = StorageAdminApi::storage_info(self).await;
832
833 let mut disk_stats = vec![DiskStat::default(); self.pools.len()];
834
835 let mut total_cap = 0;
836 let mut total_free = 0;
837 for disk in si.disks.iter() {
838 if disk.pool_index < 0 || disk_stats.len() <= disk.pool_index as usize {
839 continue;
840 }
841
842 total_cap += disk.total_space;
843 total_free += disk.available_space;
844
845 disk_stats[disk.pool_index as usize].total_space += disk.total_space;
846 disk_stats[disk.pool_index as usize].available_space += disk.available_space;
847 }
848
849 #[cfg(test)]
850 if let Some(overridden) = take_rebalance_disk_stats_override_for_test(self.id) {
851 disk_stats = overridden;
852 total_cap = disk_stats.iter().map(|stat| stat.total_space).sum();
853 total_free = disk_stats.iter().map(|stat| stat.available_space).sum();
854 }
855
856 let percent_free_goal = percent_free_ratio(total_free, total_cap);
857 validate_rebalance_disk_stats_coverage(&disk_stats)?;
858
859 let mut pool_stats = Vec::with_capacity(self.pools.len());
860
861 let now = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
862
863 for disk_stat in disk_stats.iter() {
864 let mut pool_stat = RebalanceStats {
865 init_free_space: disk_stat.available_space,
866 init_capacity: disk_stat.total_space,
867 buckets: bucktes.clone(),
868 rebalanced_buckets: Vec::with_capacity(bucktes.len()),
869 ..Default::default()
870 };
871
872 if should_pool_participate(disk_stat.available_space, disk_stat.total_space, percent_free_goal) {
873 pool_stat.participating = true;
874 pool_stat.info = RebalanceInfo {
875 start_time: Some(now),
876 status: RebalStatus::Started,
877 ..Default::default()
878 };
879 }
880
881 pool_stats.push(pool_stat);
882 }
883
884 let has_participating_pool = pool_stats.iter().any(|pool_stat| pool_stat.participating);
885 let meta = RebalanceMeta {
886 id: Uuid::new_v4().to_string(),
887 stopped_at: (!has_participating_pool).then_some(now),
888 percent_free_goal,
889 pool_stats,
890 ..Default::default()
891 };
892
893 let pool = clone_first_arc(&self.pools, "init_rebalance_meta: no pools available")?;
894 resolve_rebalance_meta_save_result(
895 self.save_rebalance_activation_meta_with_merge(pool, &meta, "init_rebalance_meta")
896 .await,
897 "init_rebalance_meta",
898 )?;
899
900 info!(
901 event = EVENT_REBALANCE_STATE,
902 component = LOG_COMPONENT_ECSTORE,
903 subsystem = LOG_SUBSYSTEM_REBALANCE,
904 state = "metadata_initialized",
905 bucket_count = bucktes.len(),
906 "Rebalance metadata initialized"
907 );
908
909 let id = meta.id.clone();
910
911 {
912 let mut rebalance_meta = self.rebalance_meta.write().await;
913 *rebalance_meta = Some(meta);
914 drop(rebalance_meta);
915 }
916
917 Ok(id)
918 }
919
920 #[tracing::instrument(skip(self, bucktes))]
921 pub async fn init_and_start_rebalance(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
922 let id = self.init_rebalance_start(bucktes).await?;
923 if let Err(start_err) = self.start_rebalance_for_id(&id).await {
924 if let Err(rollback_err) = self
925 .rollback_rebalance_start_without_worker_for_id(Some(&id), start_err.to_string())
926 .await
927 {
928 return Err(Error::other(format!(
929 "failed to start rebalance after metadata initialized for {id}: {start_err}; rollback failed: {rollback_err}"
930 )));
931 }
932
933 return Err(Error::other(format!(
934 "failed to start rebalance after metadata initialized for {id}; local metadata was finalized as failed: {start_err}"
935 )));
936 }
937
938 Ok(id)
939 }
940
941 #[tracing::instrument(skip(self, bucktes))]
942 pub async fn init_rebalance_start(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
943 let _start_guard = self.start_gate.lock().await;
944 {
945 let decommission_running = self.is_decommission_running().await;
946 let rebalance_meta = self.rebalance_meta.read().await;
947 validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
948 }
949 let _activation_guard = self
950 .rebalance_activation_write_guard(None, "initialize replacement rebalance")
951 .await?;
952 let movement_gate = self.ctx.data_movement_operation_gate();
953 let _movement_guard = movement_gate.write().await;
954
955 let id = self.init_rebalance_meta(bucktes).await?;
956 self.ctx.advance_data_movement_operation_epoch();
957 Ok(id)
958 }
959
960 #[tracing::instrument(skip(self))]
961 pub async fn start_rebalance_for_id(self: &Arc<Self>, expected_id: &str) -> Result<()> {
962 let _start_guard = self.start_gate.lock().await;
963 let _activation_guard = self
964 .rebalance_activation_write_guard(Some(expected_id), "start rebalance")
965 .await?;
966 let movement_gate = self.ctx.data_movement_operation_gate();
967 let _movement_guard = movement_gate.write().await;
968
969 {
970 let rebalance_meta = self.rebalance_meta.read().await;
971 let Some(meta) = rebalance_meta.as_ref() else {
972 return Err(Error::ConfigNotFound);
973 };
974 if meta.id != expected_id {
975 return Err(Error::other(format!(
976 "rebalance metadata changed before start: expected {expected_id}, found {}",
977 meta.id
978 )));
979 }
980 if meta.stopped_at.is_some() {
981 if !is_rebalance_conflicting_with_decommission(meta) {
982 debug!(
983 event = EVENT_REBALANCE_STATE,
984 component = LOG_COMPONENT_ECSTORE,
985 subsystem = LOG_SUBSYSTEM_REBALANCE,
986 state = "start_skipped",
987 reason = "not_started_terminal",
988 rebalance_id = %expected_id,
989 "Skipped rebalance start because metadata is already terminal"
990 );
991 return Ok(());
992 }
993 return Err(Error::other(format!("rebalance {expected_id} was stopped before start")));
994 }
995 }
996
997 if self.start_rebalance_under_gate().await? {
998 self.ctx.advance_data_movement_operation_epoch();
999 }
1000 Ok(())
1001 }
1002
1003 pub async fn rollback_rebalance_start_for_id(self: &Arc<Self>, expected_id: Option<&str>, start_error: String) -> Result<()> {
1004 self.rollback_rebalance_start_without_worker_for_id(expected_id, start_error)
1005 .await
1006 }
1007
1008 #[tracing::instrument(skip(self, fi))]
1009 pub async fn update_pool_stats(&self, pool_index: usize, bucket: String, fi: &FileInfo) -> Result<()> {
1010 self.update_pool_stats_batch(pool_index, bucket, &[fi]).await
1011 }
1012
1013 #[tracing::instrument(skip(self, versions))]
1014 pub async fn update_pool_stats_batch(&self, pool_index: usize, bucket: String, versions: &[&FileInfo]) -> Result<()> {
1015 self.update_pool_stats_batch_for_id(pool_index, bucket, versions, None).await
1016 }
1017
1018 pub(super) async fn update_pool_stats_batch_for_rebalance(
1019 &self,
1020 pool_index: usize,
1021 bucket: String,
1022 versions: &[&FileInfo],
1023 expected_id: &str,
1024 ) -> Result<()> {
1025 self.update_pool_stats_batch_for_id(pool_index, bucket, versions, Some(expected_id))
1026 .await
1027 }
1028
1029 async fn update_pool_stats_batch_for_id(
1030 &self,
1031 pool_index: usize,
1032 bucket: String,
1033 versions: &[&FileInfo],
1034 expected_id: Option<&str>,
1035 ) -> Result<()> {
1036 if versions.is_empty() {
1037 return Ok(());
1038 }
1039
1040 let mut rebalance_meta = self.rebalance_meta.write().await;
1041 if let Some(expected_id) = expected_id {
1042 ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "update rebalance pool stats")?;
1043 }
1044 if let Some(meta) = rebalance_meta.as_mut() {
1045 if !should_accept_rebalance_stats_update(meta, pool_index) {
1046 return Ok(());
1047 }
1048
1049 if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
1050 pool_stat.update_batch(bucket, versions);
1051 }
1052 }
1053
1054 Ok(())
1055 }
1056
1057 #[tracing::instrument(skip(self))]
1058 pub async fn next_rebal_bucket(&self, pool_index: usize, expected_id: &str) -> Result<Option<String>> {
1059 let rebalance_meta = self.rebalance_meta.read().await;
1060 ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "next rebalance bucket")?;
1061 debug!(
1062 event = EVENT_REBALANCE_BUCKET,
1063 component = LOG_COMPONENT_ECSTORE,
1064 subsystem = LOG_SUBSYSTEM_REBALANCE,
1065 pool_index,
1066 has_meta = rebalance_meta.is_some(),
1067 state = "next_bucket_lookup",
1068 "Rebalance next bucket lookup"
1069 );
1070 resolve_next_rebalance_bucket(rebalance_meta.as_ref(), pool_index)
1071 }
1072
1073 #[tracing::instrument(skip(self))]
1074 pub async fn bucket_rebalance_done(&self, pool_index: usize, bucket: String, expected_id: &str) -> Result<()> {
1075 let mut rebalance_meta = self.rebalance_meta.write().await;
1076 ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "mark rebalance bucket done")?;
1077 mark_rebalance_bucket_done(rebalance_meta.as_mut(), pool_index, &bucket)
1078 }
1079
1080 pub(super) async fn record_rebalance_cleanup_warning(
1081 &self,
1082 pool_index: usize,
1083 bucket: &str,
1084 object: &str,
1085 message: String,
1086 expected_id: &str,
1087 ) -> Result<()> {
1088 let mut rebalance_meta = self.rebalance_meta.write().await;
1089 ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "record rebalance cleanup warning")?;
1090 record_rebalance_cleanup_warning_in_meta(
1091 rebalance_meta.as_mut(),
1092 pool_index,
1093 bucket,
1094 object,
1095 message,
1096 OffsetDateTime::now_utc(),
1097 )
1098 }
1099
1100 pub(super) async fn defer_rebalance_bucket(
1101 &self,
1102 pool_index: usize,
1103 bucket: String,
1104 last_error: String,
1105 expected_id: &str,
1106 ) -> Result<()> {
1107 let mut rebalance_meta = self.rebalance_meta.write().await;
1108 ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "defer rebalance bucket")?;
1109 let Some(meta) = rebalance_meta.as_mut() else {
1110 return Err(rebalance_metadata_not_initialized_error("defer rebalance bucket"));
1111 };
1112 let pool_count = meta.pool_stats.len();
1113 ensure_valid_rebalance_pool_index(pool_count, pool_index)?;
1114 let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) else {
1115 return Err(invalid_rebalance_pool_index_error(pool_index, pool_count));
1116 };
1117
1118 defer_bucket_in_rebalance_queue(pool_stat, &bucket)?;
1119 pool_stat.info.last_error = Some(last_error);
1120 meta.last_refreshed_at = Some(OffsetDateTime::now_utc());
1121 Ok(())
1122 }
1123
1124 pub async fn is_rebalance_started(&self) -> bool {
1125 let rebalance_meta = self.rebalance_meta.read().await;
1126 if let Some(meta) = rebalance_meta.as_ref() {
1127 meta.pool_stats.iter().enumerate().for_each(|(i, v)| {
1128 debug!(
1129 event = EVENT_REBALANCE_STATE,
1130 component = LOG_COMPONENT_ECSTORE,
1131 subsystem = LOG_SUBSYSTEM_REBALANCE,
1132 pool_index = i,
1133 participating = v.participating,
1134 status = ?v.info.status,
1135 state = "status_inspected",
1136 "Rebalance status inspected"
1137 );
1138 });
1139
1140 let started = is_rebalance_conflicting_with_decommission(meta);
1141 if started {
1142 debug!(
1143 event = EVENT_REBALANCE_STATE,
1144 component = LOG_COMPONENT_ECSTORE,
1145 subsystem = LOG_SUBSYSTEM_REBALANCE,
1146 state = "running",
1147 "Rebalance is running"
1148 );
1149 return true;
1150 }
1151 }
1152
1153 debug!(
1154 event = EVENT_REBALANCE_STATE,
1155 component = LOG_COMPONENT_ECSTORE,
1156 subsystem = LOG_SUBSYSTEM_REBALANCE,
1157 state = "not_running",
1158 "Rebalance is not running"
1159 );
1160 false
1161 }
1162
1163 pub async fn is_rebalance_conflicting_with_decommission(&self) -> bool {
1164 let rebalance_meta = self.rebalance_meta.read().await;
1165 rebalance_meta
1166 .as_ref()
1167 .is_some_and(is_rebalance_conflicting_with_decommission)
1168 }
1169
1170 pub async fn is_pool_rebalancing(&self, pool_index: usize) -> bool {
1171 let rebalance_meta = self.rebalance_meta.read().await;
1172 if let Some(ref meta) = *rebalance_meta {
1173 if meta.stopped_at.is_some() {
1174 return false;
1175 }
1176
1177 if let Some(pool_stat) = meta.pool_stats.get(pool_index) {
1178 return pool_stat.participating && pool_stat.info.status == RebalStatus::Started;
1179 }
1180 }
1181
1182 false
1183 }
1184
1185 pub async fn pool_rebalance_status(&self, pool_index: usize) -> (RebalStatus, bool) {
1186 let rebalance_meta = self.rebalance_meta.read().await;
1187 pool_rebalance_status_from_meta(rebalance_meta.as_ref(), pool_index)
1188 }
1189
1190 pub async fn current_rebalance_id(&self) -> Option<String> {
1191 let rebalance_meta = self.rebalance_meta.read().await;
1192 rebalance_meta
1193 .as_ref()
1194 .and_then(|meta| (!meta.id.is_empty()).then(|| meta.id.clone()))
1195 }
1196
1197 pub async fn cancel_rebalance_admission_for_id(&self, expected_id: &str) -> Result<()> {
1198 let _start_guard = self.start_gate.lock().await;
1199 let mut rebalance_meta = self.rebalance_meta.write().await;
1200 ensure_rebalance_run_id(rebalance_meta.as_ref(), expected_id, "cancel rebalance admission")?;
1201 let meta = rebalance_meta
1202 .as_mut()
1203 .ok_or_else(|| rebalance_metadata_not_initialized_error("cancel rebalance admission"))?;
1204 if meta.stopped_at.is_some() || (!is_rebalance_conflicting_with_decommission(meta) && !meta.stop_requested) {
1205 return Err(Error::other(format!(
1206 "inactive rebalance rejected while cancelling admission: {expected_id}"
1207 )));
1208 }
1209 meta.stop_requested = true;
1210 meta.cancel
1211 .get_or_insert_with(tokio_util::sync::CancellationToken::new)
1212 .cancel();
1213 Ok(())
1214 }
1215
1216 #[tracing::instrument(skip(self))]
1217 pub async fn stop_rebalance(self: &Arc<Self>) -> Result<()> {
1218 self.stop_rebalance_for_id(None).await
1219 }
1220
1221 #[tracing::instrument(skip(self))]
1222 pub async fn stop_rebalance_for_id(self: &Arc<Self>, expected_id: Option<&str>) -> Result<()> {
1223 let _start_guard = self.start_gate.lock().await;
1224 let activation_gate = {
1225 let mut rebalance_meta = self.rebalance_meta.write().await;
1226 if let Some(expected_id) = expected_id {
1227 ensure_rebalance_run_id(rebalance_meta.as_ref(), expected_id, "stop rebalance")?;
1228 }
1229 rebalance_meta.as_mut().map(|meta| {
1230 meta.stop_requested |= is_rebalance_conflicting_with_decommission(meta);
1231 let cancel = meta.cancel.get_or_insert_with(tokio_util::sync::CancellationToken::new);
1232 cancel.cancel();
1233 Arc::clone(&meta.activation_gate)
1234 })
1235 };
1236 let _activation_guard = match activation_gate {
1237 Some(gate) => {
1238 #[cfg(any(test, feature = "test-util"))]
1239 observe_rebalance_stop_wait_attempt(expected_id);
1240 Some(gate.write_owned().await)
1241 }
1242 None => None,
1243 };
1244 let movement_gate = self.ctx.data_movement_operation_gate();
1245 let _movement_guard = movement_gate.write().await;
1246 let stopped_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
1247 let (previous_meta, meta_to_save) = {
1248 let mut rebalance_meta = self.rebalance_meta.write().await;
1249 let previous_meta = rebalance_meta.clone();
1250 let meta_to_save = stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), stopped_at, expected_id)?;
1251 (previous_meta, meta_to_save)
1252 };
1253
1254 if let Some(meta_to_save) = meta_to_save {
1255 let pool = clone_first_arc(self.pools.as_slice(), "stop_rebalance: no pools available")?;
1256 let save_result = resolve_rebalance_meta_save_result(
1257 self.save_rebalance_meta_for_id_with_merge(pool, &meta_to_save, "stop_rebalance", meta_to_save.id.as_str())
1258 .await,
1259 "stop_rebalance",
1260 );
1261 if let Err(err) = save_result {
1262 *self.rebalance_meta.write().await = previous_meta;
1263 return Err(err);
1264 }
1265 self.ctx.advance_data_movement_operation_epoch();
1266 }
1267
1268 Ok(())
1269 }
1270
1271 async fn rollback_rebalance_start_without_worker_for_id(
1272 self: &Arc<Self>,
1273 expected_id: Option<&str>,
1274 start_error: String,
1275 ) -> Result<()> {
1276 let _start_guard = self.start_gate.lock().await;
1277 let _activation_guard = self
1278 .rebalance_activation_write_guard(expected_id, "rollback rebalance start")
1279 .await?;
1280 let movement_gate = self.ctx.data_movement_operation_gate();
1281 let _movement_guard = movement_gate.write().await;
1282 let failed_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
1283 let meta_to_save = {
1284 let mut rebalance_meta = self.rebalance_meta.write().await;
1285 rollback_rebalance_start_meta_snapshot_for_id(rebalance_meta.as_mut(), failed_at, expected_id, start_error)
1286 };
1287
1288 if let Some(meta_to_save) = meta_to_save {
1289 let pool = clone_first_arc(self.pools.as_slice(), "rollback_rebalance_start: no pools available")?;
1290 resolve_rebalance_meta_save_result(
1291 self.save_rebalance_meta_for_id_with_merge(
1292 pool,
1293 &meta_to_save,
1294 "rollback_rebalance_start",
1295 meta_to_save.id.as_str(),
1296 )
1297 .await,
1298 "rollback_rebalance_start",
1299 )?;
1300 self.ctx.advance_data_movement_operation_epoch();
1301 }
1302
1303 Ok(())
1304 }
1305
1306 pub async fn record_rebalance_stop_propagation(
1307 self: &Arc<Self>,
1308 expected_id: &str,
1309 record: RebalanceStopPropagationRecord,
1310 ) -> Result<()> {
1311 if !record.has_failures() {
1312 return Ok(());
1313 }
1314
1315 let _start_guard = self.start_gate.lock().await;
1316 let _activation_guard = self
1317 .rebalance_activation_write_guard(Some(expected_id), "record rebalance stop propagation")
1318 .await?;
1319 let movement_gate = self.ctx.data_movement_operation_gate();
1320 let _movement_guard = movement_gate.write().await;
1321 let encoded_error = encode_rebalance_stop_propagation_record(&record);
1322 let meta_to_save = {
1323 let mut rebalance_meta = self.rebalance_meta.write().await;
1324 ensure_rebalance_run_id(rebalance_meta.as_ref(), expected_id, "record rebalance stop propagation")?;
1325 record_rebalance_stop_propagation_snapshot(rebalance_meta.as_mut(), encoded_error, OffsetDateTime::now_utc())
1326 };
1327
1328 if let Some(meta_to_save) = meta_to_save {
1329 let pool = clone_first_arc(self.pools.as_slice(), "record_rebalance_stop_propagation: no pools available")?;
1330 resolve_rebalance_meta_save_result(
1331 self.save_rebalance_meta_for_id_with_merge(pool, &meta_to_save, "record_rebalance_stop_propagation", expected_id)
1332 .await,
1333 "record_rebalance_stop_propagation",
1334 )?;
1335 self.ctx.advance_data_movement_operation_epoch();
1336 }
1337
1338 Ok(())
1339 }
1340}
1341
1342#[cfg(test)]
1343mod tests {
1344 use super::*;
1345 use crate::config::com::delete_config;
1346 use crate::core::pools::{
1347 DecommissionErasureLayout, DecommissionPoolCapacityInfo, POOL_META_NAME, PoolActivationDurableSaveBarrier,
1348 PoolActivationStartKind, PoolActivationStartProbe, PoolMetaWriteState, persist_pool_meta_identity_for_startup,
1349 set_decommission_capacity_info_overrides_for_test,
1350 };
1351 use crate::object_api::NamespaceLockFence;
1352 use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause, hermetic_set_disks_isolated};
1353
1354 #[tokio::test]
1355 async fn rebalance_status_read_recovers_after_metadata_lock_timeout() {
1356 let id = "rebalance-status-lock-retry";
1357 let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1358 id: id.to_string(),
1359 pool_stats: vec![RebalanceStats::default()],
1360 ..Default::default()
1361 })
1362 .await;
1363 let ns_lock = store.pools[0]
1364 .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME)
1365 .await
1366 .expect("metadata lock should be available");
1367 let writer = ns_lock
1368 .get_write_lock(get_lock_acquire_timeout())
1369 .await
1370 .expect("hold metadata writer");
1371 let retried = Arc::new(tokio::sync::Notify::new());
1372 let refresh = super::super::worker::REBALANCE_METADATA_RETRY_PROBE
1373 .scope(Arc::clone(&retried), store.refresh_rebalance_status_meta());
1374 tokio::pin!(refresh);
1375 tokio::time::timeout(std::time::Duration::from_secs(30), async {
1376 tokio::select! {
1377 _ = retried.notified() => {},
1378 result = &mut refresh => panic!("status read must retry its contended metadata lock: {result:?}"),
1379 }
1380 })
1381 .await
1382 .expect("status read should encounter a real lock timeout");
1383 drop(writer);
1384 tokio::time::timeout(std::time::Duration::from_secs(30), refresh)
1385 .await
1386 .expect("status refresh should finish after the writer releases")
1387 .expect("transient metadata contention must not fail status refresh");
1388 assert_eq!(
1389 store
1390 .rebalance_meta
1391 .read()
1392 .await
1393 .as_ref()
1394 .expect("metadata must remain present")
1395 .id,
1396 id
1397 );
1398 }
1399
1400 #[tokio::test]
1401 async fn rebalance_stats_save_recovers_after_metadata_lock_timeout() {
1402 let id = "rebalance-stats-lock-retry";
1403 let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1404 id: id.to_string(),
1405 pool_stats: vec![RebalanceStats::default()],
1406 ..Default::default()
1407 })
1408 .await;
1409 store
1410 .rebalance_meta
1411 .write()
1412 .await
1413 .as_mut()
1414 .expect("local metadata must exist")
1415 .pool_stats[0]
1416 .num_objects = 7;
1417 let ns_lock = store.pools[0]
1418 .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME)
1419 .await
1420 .expect("metadata lock should be available");
1421 let reader = ns_lock
1422 .get_read_lock(get_lock_acquire_timeout())
1423 .await
1424 .expect("hold a migration read fence");
1425 let retried = Arc::new(tokio::sync::Notify::new());
1426 let save = super::super::worker::REBALANCE_METADATA_RETRY_PROBE.scope(
1427 Arc::clone(&retried),
1428 store.save_rebalance_stats_for_id(0, super::super::RebalSaveOpt::Stats, id),
1429 );
1430 tokio::pin!(save);
1431 tokio::time::timeout(std::time::Duration::from_secs(30), async {
1432 tokio::select! {
1433 _ = retried.notified() => {},
1434 result = &mut save => panic!("stats save must retry its contended metadata lock: {result:?}"),
1435 }
1436 })
1437 .await
1438 .expect("stats save should encounter a real lock timeout");
1439 drop(reader);
1440 tokio::time::timeout(std::time::Duration::from_secs(30), save)
1441 .await
1442 .expect("stats save should finish after the migration read fence releases")
1443 .expect("transient metadata contention must not lose stats");
1444 let mut persisted = RebalanceMeta::new();
1445 persisted
1446 .load(Arc::clone(&store.pools[0]))
1447 .await
1448 .expect("persisted stats must be readable");
1449 assert_eq!(persisted.id, id);
1450 assert_eq!(persisted.pool_stats[0].num_objects, 7);
1451 }
1452
1453 async fn persist_initialized_identity_then_remove_pool_meta(store: &Arc<ECStore>) {
1454 let deployment_id = store
1455 .ctx
1456 .deployment_id()
1457 .expect("test store should have a deployment identity");
1458 let mut write_state = PoolMetaWriteState::for_startup(deployment_id, false);
1459 persist_pool_meta_identity_for_startup(store.pools.clone(), &mut write_state, true)
1460 .await
1461 .expect("initialized pool metadata identity should persist");
1462 *store.pool_meta_save_gate.lock().await = write_state;
1463 for pool in &store.pools {
1464 delete_config(pool.clone(), POOL_META_NAME)
1465 .await
1466 .expect("every pool metadata replica should be removed");
1467 }
1468 }
1469
1470 async fn assert_activation_locks_released(store: &Arc<ECStore>) {
1471 let fleet_proof = acquire_pool_activation_fleet_proof(&store.ctx)
1472 .await
1473 .expect("fleet proof should remain available");
1474 tokio::time::timeout(
1475 std::time::Duration::from_secs(5),
1476 acquire_pool_rebalance_activation_locks(store.pools[0].clone(), fleet_proof),
1477 )
1478 .await
1479 .expect("a rejected activation must release namespace fences promptly")
1480 .expect("a rejected activation must release both namespace fences");
1481 }
1482
1483 #[tokio::test]
1484 async fn rebalance_stop_wait_probe_matches_run_id() {
1485 let probe = RebalanceStopWaitProbe::install("rebalance-stop-current");
1486
1487 observe_rebalance_stop_wait_attempt(Some("rebalance-stop-stale"));
1488 assert!(!probe.state.attempted.load(std::sync::atomic::Ordering::Acquire));
1489
1490 observe_rebalance_stop_wait_attempt(Some("rebalance-stop-current"));
1491 probe.wait_until_attempted().await;
1492 }
1493
1494 #[test]
1495 fn rebalance_stop_classification_checks_identity_and_explicit_intent() {
1496 let mut meta = RebalanceMeta {
1497 id: "current".to_string(),
1498 cancel: Some(tokio_util::sync::CancellationToken::new()),
1499 pool_stats: vec![RebalanceStats {
1500 participating: true,
1501 info: RebalanceInfo {
1502 status: RebalStatus::Started,
1503 ..Default::default()
1504 },
1505 ..Default::default()
1506 }],
1507 ..Default::default()
1508 };
1509 ensure_rebalance_worker_active(Some(&meta), "current", "test").expect("active worker");
1510 meta.cancel.as_ref().unwrap().cancel();
1511 assert!(
1512 !matches!(
1513 ensure_rebalance_worker_active(Some(&meta), "current", "test"),
1514 Err(Error::OperationCanceled)
1515 ),
1516 "a sibling failure is not an operator stop"
1517 );
1518 meta.stop_requested = true;
1519 assert!(matches!(
1520 ensure_rebalance_worker_active(Some(&meta), "current", "test"),
1521 Err(Error::OperationCanceled)
1522 ));
1523 assert!(
1524 !matches!(ensure_rebalance_worker_active(Some(&meta), "old", "test"), Err(Error::OperationCanceled)),
1525 "stale identity remains a failure even during stop"
1526 );
1527 assert!(!matches!(
1528 ensure_rebalance_worker_active(None, "current", "test"),
1529 Err(Error::OperationCanceled)
1530 ));
1531 }
1532
1533 #[tokio::test]
1534 async fn rebalance_stop_intent_does_not_survive_replacement_run_reload() {
1535 let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1536 id: "replacement".to_string(),
1537 pool_stats: vec![RebalanceStats {
1538 participating: true,
1539 info: RebalanceInfo {
1540 status: RebalStatus::Completed,
1541 ..Default::default()
1542 },
1543 ..Default::default()
1544 }],
1545 ..Default::default()
1546 })
1547 .await;
1548 let previous_gate = {
1549 let mut meta = store.rebalance_meta.write().await;
1550 let meta = meta.as_mut().unwrap();
1551 meta.id = "previous".to_string();
1552 meta.stop_requested = true;
1553 let cancel = tokio_util::sync::CancellationToken::new();
1554 cancel.cancel();
1555 meta.cancel = Some(cancel);
1556 Arc::clone(&meta.activation_gate)
1557 };
1558 store.load_rebalance_meta().await.expect("reload replacement run");
1559 let meta = store.rebalance_meta.read().await;
1560 let meta = meta.as_ref().unwrap();
1561 assert_eq!(meta.id, "replacement");
1562 assert!(!meta.stop_requested);
1563 assert!(meta.cancel.is_none());
1564 assert!(!Arc::ptr_eq(&previous_gate, &meta.activation_gate));
1565 }
1566
1567 #[tokio::test]
1568 async fn cancel_rebalance_admission_is_id_checked_and_idempotent() {
1569 let rebalance_id = "rebalance-admission-current";
1570 let cancel = tokio_util::sync::CancellationToken::new();
1571 let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1572 id: rebalance_id.to_string(),
1573 percent_free_goal: 1.0,
1574 cancel: Some(cancel.clone()),
1575 pool_stats: vec![RebalanceStats {
1576 participating: true,
1577 init_capacity: 100,
1578 info: RebalanceInfo {
1579 start_time: Some(OffsetDateTime::now_utc()),
1580 status: RebalStatus::Started,
1581 ..Default::default()
1582 },
1583 ..Default::default()
1584 }],
1585 ..Default::default()
1586 })
1587 .await;
1588
1589 let error = store
1590 .cancel_rebalance_admission_for_id("rebalance-admission-replacement")
1591 .await
1592 .expect_err("a stale admin stop must not cancel the current run");
1593 assert!(error.to_string().contains("expected rebalance-admission-replacement"));
1594 assert!(!cancel.is_cancelled());
1595
1596 store
1597 .cancel_rebalance_admission_for_id(rebalance_id)
1598 .await
1599 .expect("the current run admission should close");
1600 store
1601 .cancel_rebalance_admission_for_id(rebalance_id)
1602 .await
1603 .expect("retrying admission cancellation should be idempotent");
1604 assert!(cancel.is_cancelled());
1605 let err = store
1606 .update_pool_stats_batch_for_rebalance(0, "bucket".to_string(), &[&FileInfo::default()], rebalance_id)
1607 .await
1608 .expect_err("stop racing with a final stats update must cancel that update");
1609 assert!(matches!(err, Error::OperationCanceled), "operator stop lost its cancellation type: {err}");
1610 }
1611
1612 #[tokio::test]
1613 async fn prepare_rebalance_stop_preserves_intent_when_worker_stops_before_reload() {
1614 let id = "stop-worker-before-reload";
1615 let cancel = tokio_util::sync::CancellationToken::new();
1616 let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1617 id: id.to_string(),
1618 cancel: Some(cancel.clone()),
1619 pool_stats: vec![RebalanceStats {
1620 participating: true,
1621 info: RebalanceInfo {
1622 status: RebalStatus::Stopped,
1623 ..Default::default()
1624 },
1625 ..Default::default()
1626 }],
1627 ..Default::default()
1628 })
1629 .await;
1630 let gate = {
1631 let mut meta = store.rebalance_meta.write().await;
1632 let meta = meta.as_mut().expect("local rebalance metadata");
1633 meta.pool_stats[0].info.status = RebalStatus::Started;
1634 Arc::clone(&meta.activation_gate)
1635 };
1636 assert_eq!(
1637 store.prepare_rebalance_stop().await.expect("prepare the same run stop"),
1638 Some(id.to_string())
1639 );
1640 {
1641 let meta = store.rebalance_meta.read().await;
1642 let meta = meta.as_ref().expect("reloaded stop target");
1643 assert!(Arc::ptr_eq(&gate, &meta.activation_gate), "reload must retain the drained run's gate");
1644 assert!(meta.cancel.as_ref().is_some_and(|token| token.is_cancelled()));
1645 }
1646 store
1647 .stop_rebalance_for_id(Some(id))
1648 .await
1649 .expect("finish the stop after the worker's terminal event");
1650 store
1651 .load_rebalance_meta()
1652 .await
1653 .expect("reload the acknowledged durable stop");
1654 let meta = store.rebalance_meta.read().await;
1655 let meta = meta.as_ref().expect("durable stopped metadata");
1656 assert!(meta.stopped_at.is_some(), "a successful stop must retain its durable timestamp");
1657 assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped);
1658 }
1659
1660 #[tokio::test]
1661 #[serial_test::serial]
1662 async fn equal_free_ratio_admin_no_participant_rebalance_succeeds_and_persists_terminal_generation_after_restart() {
1663 let (_temp_dirs, store, restarted) =
1664 crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
1665 let movement_floor = OffsetDateTime::from_unix_timestamp(4_100_000_000).expect("future test timestamp should be valid");
1666 *store.rebalance_meta.write().await = Some(RebalanceMeta {
1667 id: "previous-terminal-rebalance".to_string(),
1668 stopped_at: Some(movement_floor),
1669 ..Default::default()
1670 });
1671 set_rebalance_disk_stats_override_for_test(
1672 store.id,
1673 vec![
1674 DiskStat {
1675 total_space: 100,
1676 available_space: 50,
1677 },
1678 DiskStat {
1679 total_space: 100,
1680 available_space: 50,
1681 },
1682 ],
1683 );
1684
1685 let rebalance_id = store
1686 .init_and_start_rebalance(vec!["equal-ratio-no-op".to_string()])
1687 .await
1688 .expect("equal free ratio admin rebalance should succeed as a terminal no-op");
1689 let stopped_at = {
1690 let local = store.rebalance_meta.read().await;
1691 let local = local.as_ref().expect("no-op rebalance metadata should remain available");
1692 assert_eq!(local.id, rebalance_id);
1693 assert!(local.pool_stats.iter().all(|pool_stat| !pool_stat.participating));
1694 let stopped_at = local.stopped_at.expect("no-op rebalance must persist a terminal timestamp");
1695 assert_eq!(stopped_at, movement_floor + time::Duration::nanoseconds(1));
1696 stopped_at
1697 };
1698
1699 let stopped_generation =
1700 u64::try_from(stopped_at.unix_timestamp_nanos()).expect("terminal timestamp should map to scanner generation");
1701 let live_status = store.scanner_data_movement_pause_status().await;
1702 assert!(!live_status.paused);
1703 assert_eq!(live_status.movement_generation, stopped_generation);
1704
1705 restarted
1706 .load_rebalance_meta()
1707 .await
1708 .expect("restarted store should load the persisted no-op rebalance metadata");
1709 let status = restarted.scanner_data_movement_pause_status().await;
1710
1711 assert!(!status.paused);
1712 assert_eq!(status.movement_generation, stopped_generation);
1713 assert_eq!(restarted.scanner_data_movement_generation(), stopped_generation);
1714 }
1715
1716 #[tokio::test]
1717 #[serial_test::serial]
1718 async fn rebalance_activation_rejects_initialized_cluster_with_all_pool_meta_missing() {
1719 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
1720 persist_initialized_identity_then_remove_pool_meta(&store).await;
1721 set_rebalance_disk_stats_override_for_test(
1722 store.id,
1723 vec![
1724 DiskStat {
1725 total_space: 100,
1726 available_space: 0,
1727 },
1728 DiskStat {
1729 total_space: 100,
1730 available_space: 100,
1731 },
1732 ],
1733 );
1734
1735 let err = store
1736 .init_rebalance_start(vec!["missing-pool-meta".to_string()])
1737 .await
1738 .expect_err("rebalance activation must fail closed when every pool.bin is missing");
1739 assert!(err.to_string().contains("initialized cluster identity exists"));
1740 store
1741 .ensure_pool_meta_side_effects_safe("rebalance activation after missing pool metadata")
1742 .await
1743 .expect_err("the missing metadata observation must latch the shared runtime gate");
1744
1745 let mut persisted = RebalanceMeta::new();
1746 assert!(
1747 matches!(persisted.load(store.pools[0].clone()).await, Err(Error::ConfigNotFound)),
1748 "rejected activation must not create rebalance metadata"
1749 );
1750 assert_activation_locks_released(&store).await;
1751 }
1752
1753 #[tokio::test]
1754 #[serial_test::serial]
1755 async fn rebalance_worker_rejects_initialized_cluster_with_all_pool_meta_missing() {
1756 let rebalance_id = "missing-pool-meta-worker";
1757 let active = RebalanceMeta {
1758 id: rebalance_id.to_string(),
1759 percent_free_goal: 0.5,
1760 pool_stats: vec![
1761 RebalanceStats {
1762 participating: true,
1763 init_capacity: 100,
1764 info: RebalanceInfo {
1765 status: RebalStatus::Started,
1766 ..Default::default()
1767 },
1768 ..Default::default()
1769 },
1770 RebalanceStats {
1771 participating: true,
1772 init_capacity: 100,
1773 info: RebalanceInfo {
1774 status: RebalStatus::Started,
1775 ..Default::default()
1776 },
1777 ..Default::default()
1778 },
1779 ],
1780 ..Default::default()
1781 };
1782 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(active)).await;
1783 persist_initialized_identity_then_remove_pool_meta(&store).await;
1784
1785 let err = match store
1786 .fence_rebalance_worker_activation(store.pools[0].clone(), rebalance_id)
1787 .await
1788 {
1789 Ok(_) => panic!("worker activation must not return a fence when every pool.bin is missing"),
1790 Err(err) => err,
1791 };
1792 assert!(err.to_string().contains("initialized cluster identity exists"));
1793 store
1794 .ensure_pool_meta_side_effects_safe("rebalance worker after missing pool metadata")
1795 .await
1796 .expect_err("worker validation must latch the shared runtime gate");
1797 assert_activation_locks_released(&store).await;
1798 }
1799
1800 #[tokio::test]
1801 #[serial_test::serial]
1802 async fn rebalance_worker_skips_terminal_metadata_without_fleet_proof() {
1803 let rebalance_id = "terminal-metadata-without-proof";
1804 let completed = RebalanceMeta {
1805 id: rebalance_id.to_string(),
1806 pool_stats: vec![RebalanceStats {
1807 participating: true,
1808 info: RebalanceInfo {
1809 status: RebalStatus::Completed,
1810 ..Default::default()
1811 },
1812 ..Default::default()
1813 }],
1814 ..Default::default()
1815 };
1816 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(completed)).await;
1817 let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
1818
1819 let activation = store
1820 .fence_rebalance_worker_activation(store.pools[0].clone(), rebalance_id)
1821 .await
1822 .expect("terminal metadata should not require a fleet proof");
1823 assert!(matches!(activation, RebalanceWorkerActivationFence::NotStartedTerminal));
1824 }
1825
1826 #[tokio::test]
1827 #[serial_test::serial]
1828 async fn rebalance_worker_still_requires_fleet_proof_for_active_metadata() {
1829 let rebalance_id = "active-metadata-without-proof";
1830 let active = RebalanceMeta {
1831 id: rebalance_id.to_string(),
1832 pool_stats: vec![RebalanceStats {
1833 participating: true,
1834 info: RebalanceInfo {
1835 status: RebalStatus::Started,
1836 ..Default::default()
1837 },
1838 ..Default::default()
1839 }],
1840 ..Default::default()
1841 };
1842 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(active)).await;
1843 let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
1844
1845 let err = match store
1846 .fence_rebalance_worker_activation(store.pools[0].clone(), rebalance_id)
1847 .await
1848 {
1849 Ok(_) => panic!("active metadata must not be admitted without a fleet proof"),
1850 Err(err) => err,
1851 };
1852 assert!(
1853 err.to_string()
1854 .contains("pool activation requires a live fleet capability proof")
1855 );
1856 }
1857
1858 #[tokio::test]
1859 #[serial_test::serial]
1860 async fn rebalance_activation_adopts_commit_after_post_save_fence_loss() {
1861 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
1862 set_rebalance_disk_stats_override_for_test(
1863 store.id,
1864 vec![
1865 DiskStat {
1866 total_space: 100,
1867 available_space: 0,
1868 },
1869 DiskStat {
1870 total_space: 100,
1871 available_space: 100,
1872 },
1873 ],
1874 );
1875 let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
1876 let start_store = Arc::clone(&store);
1877 let start_task = tokio::spawn(async move {
1878 start_store
1879 .init_rebalance_start(vec!["post-commit-fence-loss".to_string()])
1880 .await
1881 });
1882
1883 barrier.wait_until_paused().await;
1884 barrier.release_after_fence_loss();
1885 let rebalance_id = tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
1886 .await
1887 .expect("rebalance activation should finish after its durable commit")
1888 .expect("rebalance activation task should not panic")
1889 .expect("post-commit fence loss must not report the committed activation as failed");
1890
1891 let local_meta = store.rebalance_meta.read().await;
1892 let local = local_meta
1893 .as_ref()
1894 .expect("the committed rebalance metadata should be installed locally");
1895 assert_eq!(local.id, rebalance_id);
1896 assert!(is_rebalance_conflicting_with_decommission(local));
1897 drop(local_meta);
1898
1899 let mut persisted = RebalanceMeta::new();
1900 persisted
1901 .load(store.pools[0].clone())
1902 .await
1903 .expect("the committed rebalance metadata should remain readable");
1904 assert_eq!(persisted.id, rebalance_id);
1905 assert!(is_rebalance_conflicting_with_decommission(&persisted));
1906 }
1907
1908 #[tokio::test]
1909 #[serial_test::serial]
1910 async fn rebalance_worker_admission_installs_committed_candidate_after_post_save_fence_loss() {
1911 let rebalance_id = "post-save-worker-admission";
1912 let active = RebalanceMeta {
1913 id: rebalance_id.to_string(),
1914 percent_free_goal: 0.5,
1915 pool_stats: vec![
1916 RebalanceStats {
1917 participating: true,
1918 init_free_space: 50,
1919 init_capacity: 100,
1920 buckets: vec!["completed-pool-bucket".to_string()],
1921 info: RebalanceInfo {
1922 status: RebalStatus::Started,
1923 ..Default::default()
1924 },
1925 ..Default::default()
1926 },
1927 RebalanceStats {
1928 participating: true,
1929 init_capacity: 100,
1930 buckets: vec!["active-pool-bucket".to_string()],
1931 info: RebalanceInfo {
1932 status: RebalStatus::Started,
1933 ..Default::default()
1934 },
1935 ..Default::default()
1936 },
1937 ],
1938 ..Default::default()
1939 };
1940 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(active)).await;
1941 let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
1942 let start_store = Arc::clone(&store);
1943 let start_task = tokio::spawn(async move { start_store.start_rebalance().await });
1944
1945 barrier.wait_until_paused().await;
1946 barrier.release_after_fence_loss();
1947 tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
1948 .await
1949 .expect("rebalance worker admission should finish after its durable commit")
1950 .expect("rebalance worker admission task should not panic")
1951 .expect("post-commit fence loss must not reject the committed worker candidate");
1952
1953 let local_meta = store.rebalance_meta.read().await;
1954 let local = local_meta
1955 .as_ref()
1956 .expect("the committed worker candidate should remain installed locally");
1957 assert_eq!(local.id, rebalance_id);
1958 assert_eq!(local.pool_stats[0].info.status, RebalStatus::Completed);
1959 let admitted_cancel = local
1960 .cancel
1961 .clone()
1962 .expect("the committed worker candidate should install its cancellation token");
1963 assert!(!admitted_cancel.is_cancelled());
1964 drop(local_meta);
1965
1966 admitted_cancel.cancel();
1967 assert!(admitted_cancel.is_cancelled());
1968
1969 let mut persisted = RebalanceMeta::new();
1970 persisted
1971 .load(store.pools[0].clone())
1972 .await
1973 .expect("the committed worker candidate should remain readable");
1974 assert_eq!(persisted.id, rebalance_id);
1975 assert_eq!(persisted.pool_stats[0].info.status, RebalStatus::Completed);
1976 }
1977
1978 async fn assert_real_activation_start_race(paused_kind: PoolActivationStartKind) {
1979 let (_temp_dirs, rebalance_store, decommission_store) =
1980 crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
1981 crate::services::rebalance::promote_test_pool_meta_to_v2(&rebalance_store).await;
1982 crate::services::rebalance::promote_test_pool_meta_to_v2(&decommission_store).await;
1983 let disk_stats = vec![
1984 DiskStat {
1985 total_space: 100,
1986 available_space: 0,
1987 },
1988 DiskStat {
1989 total_space: 100,
1990 available_space: 100,
1991 },
1992 ];
1993 set_rebalance_disk_stats_override_for_test(rebalance_store.id, disk_stats.clone());
1994 set_rebalance_disk_stats_override_for_test(decommission_store.id, disk_stats);
1995 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
1996 let capacity_snapshot = vec![
1997 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
1998 DecommissionPoolCapacityInfo::for_test(1, layout, 200, 200, 0),
1999 ];
2000 set_decommission_capacity_info_overrides_for_test(
2002 decommission_store.id,
2003 vec![capacity_snapshot.clone(), capacity_snapshot],
2004 );
2005 let (first_object, competing_object, competing_kind) = match paused_kind {
2006 PoolActivationStartKind::Rebalance => {
2007 (REBAL_META_NAME, crate::core::pools::POOL_META_NAME, PoolActivationStartKind::Decommission)
2008 }
2009 PoolActivationStartKind::Decommission => {
2010 (crate::core::pools::POOL_META_NAME, REBAL_META_NAME, PoolActivationStartKind::Rebalance)
2011 }
2012 };
2013 let first_barrier = PutObjectCommitBarrier::install(
2014 crate::disk::RUSTFS_META_BUCKET,
2015 first_object,
2016 PutObjectCommitPause::BeforeQuotaRename,
2017 );
2018 let competing_barrier = PutObjectCommitBarrier::install(
2019 crate::disk::RUSTFS_META_BUCKET,
2020 competing_object,
2021 PutObjectCommitPause::BeforeQuotaRename,
2022 );
2023
2024 let mut rebalance_task;
2025 let mut decommission_task;
2026 let mut observed_rebalance_result = None;
2027 let mut observed_decommission_result = None;
2028 let reached_activation = if paused_kind == PoolActivationStartKind::Rebalance {
2029 let store = Arc::clone(&rebalance_store);
2030 rebalance_task =
2031 tokio::spawn(async move { store.init_rebalance_start(vec!["bucket".to_string()]).await.map(|_| ()) });
2032 tokio::select! {
2033 _ = first_barrier.wait_until_paused() => {}
2034 result = &mut rebalance_task => {
2035 panic!("real rebalance start finished before its activation commit: {:?}", result.expect("rebalance activation task should not panic"));
2036 }
2037 }
2038 let probe = PoolActivationStartProbe::install(competing_kind);
2039 let store = Arc::clone(&decommission_store);
2040 decommission_task = tokio::spawn(async move { store.start_decommission(vec![0]).await });
2041 tokio::time::timeout(std::time::Duration::from_secs(15), async {
2042 tokio::select! {
2043 _ = probe.wait_until_attempted() => true,
2044 result = &mut decommission_task => {
2045 observed_decommission_result = Some(result.expect("decommission activation task should not panic"));
2046 false
2047 }
2048 }
2049 })
2050 .await
2051 .unwrap_or(false)
2052 } else {
2053 let store = Arc::clone(&decommission_store);
2054 decommission_task = tokio::spawn(async move { store.start_decommission(vec![0]).await });
2055 tokio::select! {
2056 _ = first_barrier.wait_until_paused() => {}
2057 result = &mut decommission_task => {
2058 panic!("real decommission start finished before its activation commit: {:?}", result.expect("decommission activation task should not panic"));
2059 }
2060 }
2061 let probe = PoolActivationStartProbe::install(competing_kind);
2062 let store = Arc::clone(&rebalance_store);
2063 rebalance_task =
2064 tokio::spawn(async move { store.init_rebalance_start(vec!["bucket".to_string()]).await.map(|_| ()) });
2065 tokio::time::timeout(std::time::Duration::from_secs(15), async {
2066 tokio::select! {
2067 _ = probe.wait_until_attempted() => true,
2068 result = &mut rebalance_task => {
2069 observed_rebalance_result = Some(result.expect("rebalance activation task should not panic"));
2070 false
2071 }
2072 }
2073 })
2074 .await
2075 .unwrap_or(false)
2076 };
2077
2078 let competing_reached_commit = if reached_activation {
2081 tokio::time::timeout(std::time::Duration::from_secs(15), async {
2082 if paused_kind == PoolActivationStartKind::Rebalance {
2083 tokio::select! {
2084 _ = competing_barrier.wait_until_paused() => true,
2085 result = &mut decommission_task => {
2086 observed_decommission_result = Some(result.expect("decommission activation task should not panic"));
2087 false
2088 }
2089 }
2090 } else {
2091 tokio::select! {
2092 _ = competing_barrier.wait_until_paused() => true,
2093 result = &mut rebalance_task => {
2094 observed_rebalance_result = Some(result.expect("rebalance activation task should not panic"));
2095 false
2096 }
2097 }
2098 }
2099 })
2100 .await
2101 .expect("competing activation should either finish or reach its commit")
2102 } else {
2103 false
2104 };
2105 if competing_reached_commit {
2106 competing_barrier.release();
2107 }
2108 drop(competing_barrier);
2109 first_barrier.release();
2110 drop(first_barrier);
2111
2112 let rebalance_result = match observed_rebalance_result {
2113 Some(result) => result,
2114 None => tokio::time::timeout(std::time::Duration::from_secs(15), &mut rebalance_task)
2115 .await
2116 .expect("real rebalance activation should finish")
2117 .expect("rebalance activation task should not panic"),
2118 };
2119 let decommission_result = match observed_decommission_result {
2120 Some(result) => result,
2121 None => tokio::time::timeout(std::time::Duration::from_secs(15), &mut decommission_task)
2122 .await
2123 .expect("real decommission activation should finish")
2124 .expect("decommission activation task should not panic"),
2125 };
2126 assert!(
2127 !competing_reached_commit,
2128 "the competing real start reached its commit while the other activation lock was held"
2129 );
2130 assert_ne!(
2131 rebalance_result.is_ok(),
2132 decommission_result.is_ok(),
2133 "exactly one real activation entry may commit"
2134 );
2135
2136 let mut persisted_rebalance = RebalanceMeta::new();
2137 let rebalance_committed = persisted_rebalance
2138 .load(rebalance_store.pools[0].clone())
2139 .await
2140 .is_ok_and(|_| is_rebalance_conflicting_with_decommission(&persisted_rebalance));
2141 let mut persisted_pool = PoolMeta::default();
2142 persisted_pool
2143 .load_no_lock_from_replicas(rebalance_store.pools.clone())
2144 .await
2145 .expect("persisted pool metadata should remain readable");
2146 let decommission_committed = pool_meta_has_active_decommission(&persisted_pool);
2147 assert_eq!(
2148 usize::from(rebalance_committed) + usize::from(decommission_committed),
2149 1,
2150 "at most one persisted activation state may be active"
2151 );
2152 assert_eq!(rebalance_committed, paused_kind == PoolActivationStartKind::Rebalance);
2153 assert_eq!(decommission_committed, paused_kind == PoolActivationStartKind::Decommission);
2154 }
2155
2156 #[tokio::test]
2157 #[serial_test::serial]
2158 async fn real_rebalance_and_decommission_starts_commit_at_most_one_side() {
2159 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
2160 assert_real_activation_start_race(PoolActivationStartKind::Rebalance).await;
2161 assert_real_activation_start_race(PoolActivationStartKind::Decommission).await;
2162 })
2163 .await;
2164 }
2165
2166 #[test]
2167 fn rebalance_activation_rejects_persisted_decommission_despite_idle_local_snapshot() {
2168 let persisted = PoolMeta {
2169 pools: vec![crate::core::pools::PoolStatus {
2170 id: 0,
2171 cmd_line: "pool-0".to_string(),
2172 last_update: OffsetDateTime::UNIX_EPOCH,
2173 decommission: Some(crate::core::pools::PoolDecommissionInfo {
2174 start_time: Some(OffsetDateTime::UNIX_EPOCH),
2175 ..Default::default()
2176 }),
2177 }],
2178 ..Default::default()
2179 };
2180
2181 let err = ensure_rebalance_activation_pool_meta_allowed(&persisted)
2182 .expect_err("persisted decommission must block a stale rebalance admission");
2183 assert!(matches!(err, Error::DecommissionAlreadyRunning));
2184 }
2185
2186 #[test]
2187 fn rebalance_activation_allows_persisted_idle_pool_meta() {
2188 assert!(ensure_rebalance_activation_pool_meta_allowed(&PoolMeta::default()).is_ok());
2189 }
2190
2191 #[tokio::test]
2192 async fn rebalance_merge_save_does_not_commit_after_namespace_fence_loss() {
2193 let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks_isolated(4).await;
2194
2195 let persisted = RebalanceMeta {
2196 id: "rebalance-a".to_string(),
2197 pool_stats: vec![RebalanceStats {
2198 participating: true,
2199 bytes: 1,
2200 info: RebalanceInfo {
2201 status: RebalStatus::Started,
2202 ..Default::default()
2203 },
2204 ..Default::default()
2205 }],
2206 ..Default::default()
2207 };
2208 persisted
2209 .save(set_disks.clone())
2210 .await
2211 .expect("baseline rebalance metadata should be saved");
2212
2213 let mut update = persisted.clone();
2214 update.pool_stats[0].bytes = 999;
2215 let err = merge_and_save_rebalance_meta_no_lock(
2216 set_disks.clone(),
2217 &update,
2218 "lost namespace fence test",
2219 ObjectOptions {
2220 no_lock: true,
2221 namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()),
2222 ..Default::default()
2223 },
2224 None,
2225 Some(persisted.id.as_str()),
2226 )
2227 .await
2228 .expect_err("lost namespace fence must reject the metadata commit");
2229 assert!(matches!(err, Error::NamespaceLockQuorumUnavailable { .. }));
2230
2231 let mut after = RebalanceMeta::new();
2232 after
2233 .load(set_disks)
2234 .await
2235 .expect("baseline rebalance metadata should remain readable");
2236 assert_eq!(after.id, persisted.id);
2237 assert_eq!(after.pool_stats[0].bytes, 1);
2238 }
2239
2240 #[tokio::test]
2241 async fn persisted_rebalance_run_fence_rejects_stale_active_node_after_remote_stop() {
2242 let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks_isolated(4).await;
2243 let active = RebalanceMeta {
2244 id: "rebalance-a".to_string(),
2245 cancel: Some(tokio_util::sync::CancellationToken::new()),
2246 pool_stats: vec![RebalanceStats {
2247 participating: true,
2248 info: RebalanceInfo {
2249 status: RebalStatus::Started,
2250 ..Default::default()
2251 },
2252 ..Default::default()
2253 }],
2254 ..Default::default()
2255 };
2256 active
2257 .save(set_disks.clone())
2258 .await
2259 .expect("active rebalance metadata should be saved");
2260 ensure_rebalance_worker_active(Some(&active), active.id.as_str(), "stale node precondition")
2261 .expect("the stale node snapshot should still look active locally");
2262
2263 let mut stopped = active.clone();
2264 stopped.stopped_at = Some(OffsetDateTime::now_utc());
2265 stopped.pool_stats[0].info.stopping = true;
2266 stopped
2267 .save(set_disks.clone())
2268 .await
2269 .expect("remote stop should replace persisted metadata");
2270
2271 let err = acquire_persisted_rebalance_run_guard(set_disks, active.id.as_str(), "cross-node stale snapshot")
2272 .await
2273 .expect_err("persisted stop must fence a node that missed stop propagation");
2274 assert!(
2275 matches!(err, Error::OperationCanceled),
2276 "a durable remote stop cancels the same run: {err}"
2277 );
2278 }
2279
2280 #[test]
2281 fn pool_rebalance_status_ignores_non_participating_pool_state() {
2282 let meta = RebalanceMeta {
2283 pool_stats: vec![
2284 RebalanceStats {
2285 participating: false,
2286 info: RebalanceInfo {
2287 status: RebalStatus::Started,
2288 stopping: true,
2289 ..Default::default()
2290 },
2291 ..Default::default()
2292 },
2293 RebalanceStats {
2294 participating: true,
2295 info: RebalanceInfo {
2296 status: RebalStatus::Started,
2297 ..Default::default()
2298 },
2299 ..Default::default()
2300 },
2301 ],
2302 ..Default::default()
2303 };
2304
2305 assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 0), (RebalStatus::None, false));
2306 assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 1), (RebalStatus::Started, false));
2307 assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 2), (RebalStatus::None, false));
2308 }
2309
2310 #[test]
2311 fn rebalance_status_refresh_applies_persisted_terminal_state() {
2312 let rebalance_id = "rebalance-id".to_string();
2313 let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
2314 let mut current = Some(RebalanceMeta {
2315 id: rebalance_id.clone(),
2316 pool_stats: vec![RebalanceStats {
2317 participating: true,
2318 info: RebalanceInfo {
2319 status: RebalStatus::Started,
2320 start_time: Some(now),
2321 ..Default::default()
2322 },
2323 ..Default::default()
2324 }],
2325 ..Default::default()
2326 });
2327 let persisted = RebalanceMeta {
2328 id: rebalance_id,
2329 pool_stats: vec![RebalanceStats {
2330 participating: true,
2331 info: RebalanceInfo {
2332 status: RebalStatus::Completed,
2333 start_time: Some(now),
2334 end_time: Some(now),
2335 ..Default::default()
2336 },
2337 ..Default::default()
2338 }],
2339 ..Default::default()
2340 };
2341
2342 assert!(merge_rebalance_status_refresh(&mut current, persisted));
2343
2344 let refreshed = current.as_ref().expect("refresh should keep rebalance metadata");
2345 assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
2346 assert_eq!(refreshed.pool_stats[0].info.end_time, Some(now));
2347 }
2348
2349 #[test]
2350 fn rebalance_status_refresh_preserves_runtime_cancel_token() {
2351 let rebalance_id = "rebalance-id".to_string();
2352 let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
2353 let mut current = Some(RebalanceMeta {
2354 id: rebalance_id.clone(),
2355 cancel: Some(tokio_util::sync::CancellationToken::new()),
2356 pool_stats: vec![RebalanceStats {
2357 participating: true,
2358 info: RebalanceInfo {
2359 status: RebalStatus::Started,
2360 start_time: Some(now),
2361 ..Default::default()
2362 },
2363 ..Default::default()
2364 }],
2365 ..Default::default()
2366 });
2367 let persisted = RebalanceMeta {
2368 id: rebalance_id,
2369 pool_stats: vec![RebalanceStats {
2370 participating: true,
2371 info: RebalanceInfo {
2372 status: RebalStatus::Started,
2373 start_time: Some(now),
2374 ..Default::default()
2375 },
2376 ..Default::default()
2377 }],
2378 ..Default::default()
2379 };
2380
2381 assert!(!merge_rebalance_status_refresh(&mut current, persisted));
2382
2383 assert!(
2384 current.as_ref().and_then(|meta| meta.cancel.as_ref()).is_some(),
2385 "status refresh must not drop the runtime cancellation token"
2386 );
2387 }
2388
2389 #[test]
2390 fn rebalance_status_refresh_preserves_local_active_different_id_conflict() {
2391 let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
2392 let mut current = Some(RebalanceMeta {
2393 id: "old-active-id".to_string(),
2394 cancel: Some(tokio_util::sync::CancellationToken::new()),
2395 pool_stats: vec![RebalanceStats {
2396 participating: true,
2397 info: RebalanceInfo {
2398 status: RebalStatus::Started,
2399 start_time: Some(now),
2400 ..Default::default()
2401 },
2402 ..Default::default()
2403 }],
2404 ..Default::default()
2405 });
2406 let persisted = RebalanceMeta {
2407 id: "new-terminal-id".to_string(),
2408 pool_stats: vec![RebalanceStats {
2409 participating: true,
2410 info: RebalanceInfo {
2411 status: RebalStatus::Completed,
2412 start_time: Some(now),
2413 end_time: Some(now),
2414 ..Default::default()
2415 },
2416 ..Default::default()
2417 }],
2418 ..Default::default()
2419 };
2420
2421 merge_rebalance_status_refresh(&mut current, persisted);
2422
2423 let refreshed = current.as_ref().expect("local active metadata should remain visible");
2424 assert_eq!(refreshed.id, "old-active-id");
2425 assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Started);
2426 assert!(
2427 refreshed.cancel.is_some(),
2428 "status refresh must not drop a live runtime cancellation token"
2429 );
2430 }
2431
2432 #[test]
2433 fn rebalance_status_refresh_replaces_stale_memory_without_runtime_token() {
2434 let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
2435 let mut current = Some(RebalanceMeta {
2436 id: "old-stale-id".to_string(),
2437 pool_stats: vec![RebalanceStats {
2438 participating: true,
2439 info: RebalanceInfo {
2440 status: RebalStatus::Started,
2441 start_time: Some(now),
2442 ..Default::default()
2443 },
2444 ..Default::default()
2445 }],
2446 ..Default::default()
2447 });
2448 let persisted = RebalanceMeta {
2449 id: "new-terminal-id".to_string(),
2450 pool_stats: vec![RebalanceStats {
2451 participating: true,
2452 info: RebalanceInfo {
2453 status: RebalStatus::Completed,
2454 start_time: Some(now),
2455 end_time: Some(now),
2456 ..Default::default()
2457 },
2458 ..Default::default()
2459 }],
2460 ..Default::default()
2461 };
2462
2463 merge_rebalance_status_refresh(&mut current, persisted);
2464
2465 let refreshed = current.as_ref().expect("persisted metadata should replace stale memory");
2466 assert_eq!(refreshed.id, "new-terminal-id");
2467 assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
2468 assert!(refreshed.cancel.is_none());
2469 }
2470}