1use super::meta::{
2 RebalanceMetaMergeOutcome, clone_first_arc, clone_rebalance_pool_stats, defer_bucket_in_rebalance_queue,
3 ensure_valid_rebalance_pool_index, invalid_rebalance_pool_index_error, is_rebalance_actively_running,
4 is_rebalance_conflicting_with_decommission, mark_rebalance_bucket_done, merge_rebalance_meta, percent_free_ratio,
5 rebalance_metadata_not_initialized_error, record_rebalance_cleanup_warning_in_meta,
6 record_rebalance_stop_propagation_snapshot, resolve_next_rebalance_bucket, rollback_rebalance_start_meta_snapshot_for_id,
7 should_accept_rebalance_stats_update, should_pool_participate, stop_rebalance_meta_snapshot_for_id,
8 validate_init_rebalance_state,
9};
10use super::worker::{
11 rebalance_max_attempts, rebalance_meta_lock_error, resolve_load_rebalance_stats_update_result,
12 resolve_rebalance_deferred_last_error, resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result,
13 retry_rebalance_metadata_access,
14};
15use super::{
16 DiskStat, EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
17 RebalStatus, RebalanceDeferKind, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord,
18 encode_rebalance_stop_propagation_record,
19};
20use crate::core::pools::{
21 PoolMeta, PoolRebalanceActivationFence, acquire_pool_activation_fleet_proof, acquire_pool_rebalance_activation_locks,
22 pool_meta_has_active_decommission,
23};
24use crate::error::{Error, Result};
25use crate::object_api::ObjectOptions;
26use crate::set_disk::get_lock_acquire_timeout;
27use crate::storage_api_contracts::{
28 admin::StorageAdminApi,
29 namespace::NamespaceLocking as StorageNamespaceLocking,
30 object::{EcstoreObjectIO, EcstoreObjectOperations},
31};
32use crate::store::ECStore;
33use rustfs_filemeta::FileInfo;
34use std::sync::Arc;
35use time::OffsetDateTime;
36use tracing::{debug, info};
37use uuid::Uuid;
38
39#[cfg(test)]
40static FAIL_NEXT_REBALANCE_ACTIVATION_SAVE: std::sync::Mutex<Option<String>> = std::sync::Mutex::new(None);
41
42#[cfg(test)]
43static REBALANCE_DISK_STATS_OVERRIDES: std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<Uuid, Vec<DiskStat>>>> =
44 std::sync::OnceLock::new();
45
46#[cfg(test)]
47pub(super) fn fail_next_rebalance_activation_save_for_test(rebalance_id: &str) {
48 *FAIL_NEXT_REBALANCE_ACTIVATION_SAVE
49 .lock()
50 .expect("rebalance activation save failure hook should not be poisoned") = Some(rebalance_id.to_string());
51}
52
53#[cfg(test)]
54fn set_rebalance_disk_stats_override_for_test(store_id: Uuid, disk_stats: Vec<DiskStat>) {
55 REBALANCE_DISK_STATS_OVERRIDES
56 .get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
57 .lock()
58 .expect("rebalance disk stats override should not be poisoned")
59 .insert(store_id, disk_stats);
60}
61
62#[cfg(test)]
63fn take_rebalance_disk_stats_override_for_test(store_id: Uuid) -> Option<Vec<DiskStat>> {
64 REBALANCE_DISK_STATS_OVERRIDES
65 .get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
66 .lock()
67 .expect("rebalance disk stats override should not be poisoned")
68 .remove(&store_id)
69}
70
71fn ensure_rebalance_activation_pool_meta_allowed(meta: &PoolMeta) -> Result<()> {
72 if pool_meta_has_active_decommission(meta) {
73 return Err(Error::DecommissionAlreadyRunning);
74 }
75
76 Ok(())
77}
78
79pub(super) enum RebalanceWorkerActivationFence {
80 Ready(Box<PoolRebalanceActivationFence>),
81 NotStartedTerminal,
82}
83
84pub(super) struct RebalanceRunGuard {
85 _local_guard: tokio::sync::OwnedRwLockReadGuard<()>,
86 persisted_guard: rustfs_lock::NamespaceLockGuard,
87}
88
89#[cfg(any(test, feature = "test-util"))]
90struct RebalanceStopWaitProbeState {
91 expected_id: String,
92 attempted: std::sync::atomic::AtomicBool,
93 notify: tokio::sync::Notify,
94}
95
96#[cfg(any(test, feature = "test-util"))]
97static REBALANCE_STOP_WAIT_PROBES: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<RebalanceStopWaitProbeState>>>> =
98 std::sync::OnceLock::new();
99
100#[cfg(any(test, feature = "test-util"))]
101pub(super) struct RebalanceStopWaitProbe {
102 state: Arc<RebalanceStopWaitProbeState>,
103}
104
105#[cfg(any(test, feature = "test-util"))]
106impl RebalanceStopWaitProbe {
107 pub(super) fn install(expected_id: &str) -> Self {
108 let state = Arc::new(RebalanceStopWaitProbeState {
109 expected_id: expected_id.to_string(),
110 attempted: std::sync::atomic::AtomicBool::new(false),
111 notify: tokio::sync::Notify::new(),
112 });
113 REBALANCE_STOP_WAIT_PROBES
114 .get_or_init(|| std::sync::Mutex::new(Vec::new()))
115 .lock()
116 .expect("rebalance stop wait probe should not be poisoned")
117 .push(Arc::clone(&state));
118 Self { state }
119 }
120
121 pub(super) async fn wait_until_attempted(&self) {
122 tokio::time::timeout(std::time::Duration::from_secs(30), async {
123 while !self.state.attempted.load(std::sync::atomic::Ordering::Acquire) {
124 self.state.notify.notified().await;
125 }
126 })
127 .await
128 .expect("rebalance stop should reach the deterministic activation wait probe");
129 }
130}
131
132#[cfg(any(test, feature = "test-util"))]
133impl Drop for RebalanceStopWaitProbe {
134 fn drop(&mut self) {
135 let mut probes = REBALANCE_STOP_WAIT_PROBES
136 .get_or_init(|| std::sync::Mutex::new(Vec::new()))
137 .lock()
138 .expect("rebalance stop wait probe should not be poisoned");
139 probes.retain(|state| !Arc::ptr_eq(state, &self.state));
140 }
141}
142
143#[cfg(any(test, feature = "test-util"))]
144fn observe_rebalance_stop_wait_attempt(expected_id: Option<&str>) {
145 let probes = REBALANCE_STOP_WAIT_PROBES
146 .get_or_init(|| std::sync::Mutex::new(Vec::new()))
147 .lock()
148 .expect("rebalance stop wait probe should not be poisoned")
149 .clone();
150 for state in probes {
151 if expected_id == Some(state.expected_id.as_str()) {
152 state.attempted.store(true, std::sync::atomic::Ordering::Release);
153 state.notify.notify_one();
154 }
155 }
156}
157
158impl RebalanceRunGuard {
159 pub(super) fn ensure_held(&self, stage: &str) -> Result<()> {
160 #[cfg(test)]
161 let forced_lost = self
162 .persisted_guard
163 .lock_lost_signal()
164 .is_some_and(|signal| crate::object_api::namespace_lock_signal_test_fence_is_lost(&signal));
165 #[cfg(not(test))]
166 let forced_lost = false;
167 if self.persisted_guard.is_lock_lost() || forced_lost {
168 return Err(Error::other(format!("rebalance distributed run fence lost during {stage}")));
169 }
170 Ok(())
171 }
172
173 pub(super) fn lock_lost_signal(&self) -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
174 self.persisted_guard.lock_lost_signal()
175 }
176}
177
178async fn acquire_persisted_rebalance_run_guard<S>(
179 pool: Arc<S>,
180 expected_id: &str,
181 stage: &str,
182) -> Result<rustfs_lock::NamespaceLockGuard>
183where
184 S: EcstoreObjectIO
185 + EcstoreObjectOperations
186 + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
187{
188 let ns_lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
189 let guard = ns_lock
190 .get_read_lock(get_lock_acquire_timeout())
191 .await
192 .map_err(|err| rebalance_meta_lock_error(err, "read"))?;
193 let mut opts = ObjectOptions {
194 no_lock: true,
195 ..Default::default()
196 };
197 opts.add_namespace_lock_guard(&guard);
198 let object_info = pool
199 .get_object_info(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME, &opts)
200 .await;
201 let persisted_run_id = match object_info {
202 Ok(object_info) => {
203 let metadata = object_info.user_defined.as_ref();
204 let run_id = rustfs_utils::http::metadata_compat::get_consistent_str(
205 metadata,
206 rustfs_utils::http::metadata_compat::SUFFIX_REBALANCE_RUN_ID,
207 );
208 if run_id.is_none()
209 && rustfs_utils::http::metadata_compat::contains_key_str(
210 metadata,
211 rustfs_utils::http::metadata_compat::SUFFIX_REBALANCE_RUN_ID,
212 )
213 {
214 return Err(Error::other(format!("rebalance run marker is inconsistent during {stage}")));
215 }
216 run_id.map(str::to_string)
217 }
218 Err(err) if crate::error::is_err_object_not_found(&err) => None,
219 Err(err) => return Err(err),
220 };
221 if let Some(persisted_run_id) = persisted_run_id {
222 if persisted_run_id != expected_id {
223 return Err(Error::other(format!(
224 "stale rebalance run rejected during {stage}: expected {expected_id}, found {persisted_run_id}"
225 )));
226 }
227 } else {
228 let mut persisted = RebalanceMeta::new();
231 persisted.load_with_opts(pool, opts).await?;
232 ensure_rebalance_worker_active(Some(&persisted), expected_id, stage)?;
233 }
234 if guard.is_lock_lost() {
235 return Err(Error::other(format!("rebalance distributed run fence lost during {stage}")));
236 }
237 Ok(guard)
238}
239
240pub(super) async fn merge_and_save_rebalance_meta_no_lock<S>(
241 pool: Arc<S>,
242 local_snapshot: &RebalanceMeta,
243 stage: &str,
244 mut opts: ObjectOptions,
245 activation_fence: Option<&PoolRebalanceActivationFence>,
246 expected_id: Option<&str>,
247) -> Result<()>
248where
249 S: EcstoreObjectIO,
250{
251 let mut merged = RebalanceMeta::new();
252 match merged.load_with_opts(pool.clone(), opts.clone()).await {
253 Ok(()) => {
254 if let Some(expected_id) = expected_id {
255 ensure_rebalance_run_id(Some(&merged), expected_id, stage)?;
256 }
257 if merge_rebalance_meta(&mut merged, local_snapshot) == RebalanceMetaMergeOutcome::RejectedActiveConflict {
258 return Err(Error::RebalanceAlreadyRunning);
259 }
260 }
261 Err(Error::ConfigNotFound) => {
262 if expected_id.is_some() {
263 return Err(rebalance_metadata_not_initialized_error(stage));
264 }
265 merged = local_snapshot.clone();
266 }
267 Err(err) => return Err(Error::other(format!("rebalance meta load before save failed during {stage}: {err}"))),
268 }
269
270 if let Some(fence) = activation_fence {
271 fence.add_namespace_lock_fence(&mut opts);
272 fence.ensure_held()?;
273 }
274 #[cfg(test)]
275 let barrier_pool = pool.clone();
276 merged.save_with_opts(pool, opts).await?;
277 #[cfg(test)]
278 if let Some(fence) = activation_fence {
279 crate::core::pools::pause_pool_activation_after_durable_save(&barrier_pool, fence).await;
280 }
281 Ok(())
284}
285
286pub(super) fn ensure_rebalance_run_id(meta: Option<&RebalanceMeta>, expected_id: &str, stage: &str) -> Result<()> {
287 let Some(meta) = meta else {
288 return Err(rebalance_metadata_not_initialized_error(stage));
289 };
290 if meta.id != expected_id {
291 return Err(Error::other(format!(
292 "stale rebalance worker rejected during {stage}: expected {expected_id}, found {}",
293 meta.id
294 )));
295 }
296 Ok(())
297}
298
299pub(super) fn ensure_rebalance_worker_active(meta: Option<&RebalanceMeta>, expected_id: &str, stage: &str) -> Result<()> {
300 ensure_rebalance_run_id(meta, expected_id, stage)?;
301 let Some(meta) = meta else {
302 return Err(rebalance_metadata_not_initialized_error(stage));
303 };
304 if meta.stopped_at.is_some() || meta.stop_requested {
305 return Err(Error::OperationCanceled);
306 }
307 if meta
308 .cancel
309 .as_ref()
310 .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
311 || !is_rebalance_conflicting_with_decommission(meta)
312 {
313 return Err(Error::other(format!("inactive rebalance worker rejected during {stage}: {expected_id}")));
314 }
315 Ok(())
316}
317
318pub(super) fn validate_rebalance_disk_stats_coverage(disk_stats: &[DiskStat]) -> Result<()> {
319 for (idx, disk_stat) in disk_stats.iter().enumerate() {
320 if disk_stat.total_space == 0 {
321 return Err(Error::other(format!(
322 "rebalance storage info is incomplete: pool {idx} has no reported capacity"
323 )));
324 }
325 }
326
327 Ok(())
328}
329
330fn pool_rebalance_status_from_meta(meta: Option<&RebalanceMeta>, pool_index: usize) -> (RebalStatus, bool) {
331 meta.and_then(|meta| meta.pool_stats.get(pool_index))
332 .filter(|pool_stat| pool_stat.participating)
333 .map(|pool_stat| (pool_stat.info.status, pool_stat.info.stopping))
334 .unwrap_or_default()
335}
336
337fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted: RebalanceMeta) -> bool {
338 if persisted.id.is_empty() && persisted.pool_stats.is_empty() {
339 return clear_rebalance_status_refresh(current);
340 }
341
342 let before = current.clone();
343 match current.as_mut() {
344 Some(current_meta) => {
345 if merge_rebalance_meta(current_meta, &persisted) == RebalanceMetaMergeOutcome::RejectedActiveConflict
346 && !is_rebalance_actively_running(current_meta)
347 {
348 *current = Some(persisted);
349 }
350 }
351 None => {
352 *current = Some(persisted);
353 }
354 }
355
356 match (before.as_ref(), current.as_ref()) {
357 (None, None) => false,
358 (None, Some(_)) | (Some(_), None) => true,
359 (Some(before), Some(after)) => rebalance_movement_snapshot_changed(Some(before), after),
360 }
361}
362
363fn clear_rebalance_status_refresh(current: &mut Option<RebalanceMeta>) -> bool {
364 if current.as_ref().is_none_or(|meta| !is_rebalance_actively_running(meta)) {
365 current.take().is_some()
366 } else {
367 false
368 }
369}
370
371fn rebalance_movement_snapshot_changed(current: Option<&RebalanceMeta>, persisted: &RebalanceMeta) -> bool {
372 let Some(current) = current else {
373 return true;
374 };
375
376 current.id != persisted.id
377 || current.stopped_at != persisted.stopped_at
378 || current.pool_stats.len() != persisted.pool_stats.len()
379 || current
380 .pool_stats
381 .iter()
382 .zip(persisted.pool_stats.iter())
383 .any(|(current, persisted)| {
384 current.participating != persisted.participating
385 || current.info.status != persisted.info.status
386 || current.info.stopping != persisted.info.stopping
387 })
388}
389
390impl ECStore {
391 pub(super) async fn rebalance_activation_write_guard(
394 &self,
395 expected_id: Option<&str>,
396 stage: &str,
397 ) -> Result<Option<tokio::sync::OwnedRwLockWriteGuard<()>>> {
398 let activation_gate = {
399 let meta = self.rebalance_meta.read().await;
400 if let Some(expected_id) = expected_id {
401 ensure_rebalance_run_id(meta.as_ref(), expected_id, stage)?;
402 }
403 meta.as_ref().map(|meta| Arc::clone(&meta.activation_gate))
404 };
405 Ok(match activation_gate {
406 Some(gate) => Some(gate.write_owned().await),
407 None => None,
408 })
409 }
410
411 pub(super) async fn rebalance_run_guard(&self, expected_id: &str, stage: &str) -> Result<RebalanceRunGuard> {
412 let activation_gate = {
414 let meta = self.rebalance_meta.read().await;
415 ensure_rebalance_worker_active(meta.as_ref(), expected_id, stage)?;
416 Arc::clone(
417 &meta
418 .as_ref()
419 .ok_or_else(|| rebalance_metadata_not_initialized_error(stage))?
420 .activation_gate,
421 )
422 };
423 let guard = Arc::clone(&activation_gate).read_owned().await;
424 let meta = self.rebalance_meta.read().await;
425 ensure_rebalance_worker_active(meta.as_ref(), expected_id, stage)?;
426 let current_gate = &meta
427 .as_ref()
428 .ok_or_else(|| rebalance_metadata_not_initialized_error(stage))?
429 .activation_gate;
430 if !Arc::ptr_eq(&activation_gate, current_gate) {
431 return Err(Error::other(format!(
432 "stale rebalance activation gate rejected during {stage}: {expected_id}"
433 )));
434 }
435 drop(meta);
436 let pool = clone_first_arc(self.pools.as_slice(), "rebalance run fence: no pools available")?;
437 let persisted_guard = acquire_persisted_rebalance_run_guard(pool, expected_id, stage).await?;
438 Ok(RebalanceRunGuard {
439 _local_guard: guard,
440 persisted_guard,
441 })
442 }
443
444 pub(super) async fn save_rebalance_meta_with_merge<S>(
445 &self,
446 pool: Arc<S>,
447 local_snapshot: &RebalanceMeta,
448 stage: &str,
449 ) -> Result<()>
450 where
451 S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
452 {
453 self.save_rebalance_meta_with_merge_for_id(pool, local_snapshot, stage, None)
454 .await
455 }
456
457 pub(super) async fn save_rebalance_meta_for_id_with_merge<S>(
458 &self,
459 pool: Arc<S>,
460 local_snapshot: &RebalanceMeta,
461 stage: &str,
462 expected_id: &str,
463 ) -> Result<()>
464 where
465 S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
466 {
467 self.save_rebalance_meta_with_merge_for_id(pool, local_snapshot, stage, Some(expected_id))
468 .await
469 }
470
471 pub(super) async fn save_rebalance_meta_under_activation_fence<S>(
472 &self,
473 pool: Arc<S>,
474 local_snapshot: &RebalanceMeta,
475 stage: &str,
476 activation_fence: &PoolRebalanceActivationFence,
477 expected_id: &str,
478 ) -> Result<()>
479 where
480 S: EcstoreObjectIO,
481 {
482 #[cfg(test)]
483 {
484 let mut fail_id = FAIL_NEXT_REBALANCE_ACTIVATION_SAVE
485 .lock()
486 .expect("rebalance activation save failure hook should not be poisoned");
487 if fail_id.as_deref() == Some(local_snapshot.id.as_str()) {
488 fail_id.take();
489 return Err(Error::other("injected rebalance activation save failure"));
490 }
491 }
492 merge_and_save_rebalance_meta_no_lock(
493 pool,
494 local_snapshot,
495 stage,
496 ObjectOptions {
497 no_lock: true,
498 ..Default::default()
499 },
500 Some(activation_fence),
501 Some(expected_id),
502 )
503 .await
504 }
505
506 async fn save_rebalance_meta_with_merge_for_id<S>(
507 &self,
508 pool: Arc<S>,
509 local_snapshot: &RebalanceMeta,
510 stage: &str,
511 expected_id: Option<&str>,
512 ) -> Result<()>
513 where
514 S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
515 {
516 let ns_lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
517 let guard = retry_rebalance_metadata_access(None, rebalance_max_attempts(), || async {
518 ns_lock
519 .get_write_lock(get_lock_acquire_timeout())
520 .await
521 .map_err(|err| rebalance_meta_lock_error(err, "write"))
522 })
523 .await?;
524 let mut opts = ObjectOptions {
525 no_lock: true,
526 ..Default::default()
527 };
528 opts.add_namespace_lock_guard(&guard);
529
530 merge_and_save_rebalance_meta_no_lock(pool, local_snapshot, stage, opts, None, expected_id).await?;
531 if guard.is_lock_lost() {
532 return Err(Error::other("rebalance metadata lock lost during metadata commit"));
533 }
534 Ok(())
535 }
536
537 async fn save_rebalance_activation_meta_with_merge<S>(
538 &self,
539 pool: Arc<S>,
540 local_snapshot: &RebalanceMeta,
541 stage: &str,
542 ) -> Result<()>
543 where
544 S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
545 {
546 #[cfg(test)]
547 crate::core::pools::observe_pool_activation_start_attempt(crate::core::pools::PoolActivationStartKind::Rebalance);
548 let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
549 let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
550 pool_meta_guard.ensure_write_safe(stage)?;
551 let activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), fleet_proof).await?;
552 let pool_meta = self
553 .load_runtime_pool_meta_under_activation_fence(&mut pool_meta_guard, &activation_fence, stage)
554 .await?;
555 ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?;
556
557 merge_and_save_rebalance_meta_no_lock(
558 pool,
559 local_snapshot,
560 stage,
561 ObjectOptions {
562 no_lock: true,
563 ..Default::default()
564 },
565 Some(&activation_fence),
566 None,
567 )
568 .await
569 }
570
571 pub(super) async fn fence_rebalance_worker_activation<S>(
572 &self,
573 pool: Arc<S>,
574 expected_id: &str,
575 ) -> Result<RebalanceWorkerActivationFence>
576 where
577 S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
578 {
579 let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
581 pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
582 let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
586 let pool_meta = self
587 .load_runtime_pool_meta_under_activation_fence(&mut pool_meta_guard, &activation_fence, "rebalance worker activation")
588 .await?;
589 ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?;
590
591 let mut persisted = RebalanceMeta::new();
592 persisted
593 .load_with_opts(
594 pool,
595 ObjectOptions {
596 no_lock: true,
597 ..Default::default()
598 },
599 )
600 .await?;
601 if persisted.id != expected_id {
602 return Err(Error::other(format!(
603 "rebalance metadata changed before worker activation: expected {expected_id}, found {}",
604 persisted.id
605 )));
606 }
607
608 activation_fence.ensure_held()?;
609 if !crate::services::rebalance::rebalance_requires_worker_activation(&persisted) {
610 return Ok(RebalanceWorkerActivationFence::NotStartedTerminal);
611 }
612
613 let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
617 activation_fence.set_fleet_proof(fleet_proof);
618 activation_fence.ensure_held()?;
619
620 Ok(RebalanceWorkerActivationFence::Ready(Box::new(activation_fence)))
621 }
622
623 #[tracing::instrument(skip_all)]
624 pub async fn load_rebalance_meta(&self) -> Result<()> {
625 let _start_guard = self.start_gate.lock().await;
626 self.load_rebalance_meta_under_start_gate().await
627 }
628
629 pub async fn prepare_rebalance_stop(&self) -> Result<Option<String>> {
631 let _start_guard = self.start_gate.lock().await;
632
633 let (local_stop_id, activation_gate) = {
634 let mut rebalance_meta = self.rebalance_meta.write().await;
635 if let Some(meta) = rebalance_meta.as_mut()
636 && is_rebalance_conflicting_with_decommission(meta)
637 {
638 meta.stop_requested = true;
639 meta.cancel
640 .get_or_insert_with(tokio_util::sync::CancellationToken::new)
641 .cancel();
642 #[cfg(any(test, feature = "test-util"))]
643 observe_rebalance_stop_wait_attempt(Some(meta.id.as_str()));
644 (Some(meta.id.clone()), Some(Arc::clone(&meta.activation_gate)))
645 } else {
646 (None, None)
647 }
648 };
649
650 if let Some(gate) = activation_gate
657 && gate.try_write_owned().is_err()
658 {
659 return Ok(local_stop_id);
660 }
661
662 self.load_rebalance_meta_under_start_gate().await?;
663
664 let mut rebalance_meta = self.rebalance_meta.write().await;
665 let Some(meta) = rebalance_meta.as_mut() else {
666 return Ok(None);
667 };
668 if meta.stopped_at.is_some() || (!is_rebalance_conflicting_with_decommission(meta) && !meta.stop_requested) {
669 return Ok(None);
670 }
671 if meta.id.is_empty() {
672 return Err(Error::other("active rebalance metadata has no activation id"));
673 }
674 meta.stop_requested = true;
675 meta.cancel
676 .get_or_insert_with(tokio_util::sync::CancellationToken::new)
677 .cancel();
678 Ok(Some(meta.id.clone()))
679 }
680
681 pub(crate) async fn load_rebalance_meta_under_start_gate(&self) -> Result<()> {
682 let mut meta = RebalanceMeta::new();
683 debug!(
684 event = EVENT_REBALANCE_STATE,
685 component = LOG_COMPONENT_ECSTORE,
686 subsystem = LOG_SUBSYSTEM_REBALANCE,
687 state = "metadata_loading",
688 "Loading rebalance metadata"
689 );
690 let pool = clone_first_arc(&self.pools, "rebalanceMeta: no pools available")?;
691 let _activation_guard = self.rebalance_activation_write_guard(None, "load rebalance metadata").await?;
692 let movement_gate = self.ctx.data_movement_operation_gate();
693 let _movement_guard = movement_gate.write().await;
694 let loaded = resolve_rebalance_meta_load_result(meta.load(pool).await)?;
695 if loaded {
696 let movement_changed = rebalance_movement_snapshot_changed(self.rebalance_meta.read().await.as_ref(), &meta);
697 {
698 let mut rebalance_meta = self.rebalance_meta.write().await;
699 if let Some(current) = rebalance_meta.as_ref()
700 && current.id == meta.id
701 {
702 meta.cancel = current.cancel.clone();
703 meta.activation_gate = Arc::clone(¤t.activation_gate);
704 meta.stop_requested = current.stop_requested;
705 }
706 *rebalance_meta = Some(meta);
707
708 drop(rebalance_meta);
709 }
710
711 if movement_changed {
712 self.ctx.advance_data_movement_operation_epoch();
713 }
714 drop(_movement_guard);
715 resolve_load_rebalance_stats_update_result(self.update_rebalance_stats().await)?;
716 debug!(
717 event = EVENT_REBALANCE_STATE,
718 component = LOG_COMPONENT_ECSTORE,
719 subsystem = LOG_SUBSYSTEM_REBALANCE,
720 state = "metadata_loaded",
721 "Loaded rebalance metadata"
722 );
723 } else {
724 let movement_changed = self.rebalance_meta.read().await.is_some();
725 {
726 let mut rebalance_meta = self.rebalance_meta.write().await;
727 *rebalance_meta = None;
728 }
729 if movement_changed {
730 self.ctx.advance_data_movement_operation_epoch();
731 }
732 drop(_movement_guard);
733 debug!(
734 event = EVENT_REBALANCE_STATE,
735 component = LOG_COMPONENT_ECSTORE,
736 subsystem = LOG_SUBSYSTEM_REBALANCE,
737 state = "metadata_missing",
738 reason = "rebalance_not_started",
739 "Rebalance metadata not found"
740 );
741 }
742
743 Ok(())
744 }
745
746 #[tracing::instrument(skip_all)]
747 pub async fn refresh_rebalance_status_meta(&self) -> Result<()> {
748 let _start_guard = self.start_gate.lock().await;
749 let pool = clone_first_arc(&self.pools, "refresh_rebalance_status_meta: no pools available")?;
750 let mut persisted = RebalanceMeta::new();
751 let _activation_guard = self
752 .rebalance_activation_write_guard(None, "refresh rebalance metadata")
753 .await?;
754 let movement_gate = self.ctx.data_movement_operation_gate();
755 let _movement_guard = movement_gate.write().await;
756 let loaded = persisted.load(pool).await;
757 match loaded {
758 Ok(()) => {
759 let mut rebalance_meta = self.rebalance_meta.write().await;
760 if merge_rebalance_status_refresh(&mut rebalance_meta, persisted) {
761 self.ctx.advance_data_movement_operation_epoch();
762 }
763 }
764 Err(Error::ConfigNotFound) => {
765 let mut rebalance_meta = self.rebalance_meta.write().await;
766 if clear_rebalance_status_refresh(&mut rebalance_meta) {
767 self.ctx.advance_data_movement_operation_epoch();
768 }
769 }
770 Err(err) => {
771 return Err(Error::other(format!("rebalance metadata refresh failed during pool status: {err}")));
772 }
773 }
774
775 Ok(())
776 }
777
778 #[tracing::instrument(skip_all)]
779 pub async fn update_rebalance_stats(&self) -> Result<()> {
780 let mut ok = false;
781
782 let pool_stats = {
783 let rebalance_meta = self.rebalance_meta.read().await;
784 clone_rebalance_pool_stats(rebalance_meta.as_ref())?
785 };
786
787 debug!(
788 event = EVENT_REBALANCE_STATE,
789 component = LOG_COMPONENT_ECSTORE,
790 subsystem = LOG_SUBSYSTEM_REBALANCE,
791 pool_count = pool_stats.len(),
792 "Refreshing rebalance stats snapshot"
793 );
794
795 for i in 0..self.pools.len() {
796 if pool_stats.get(i).is_none() {
797 let mut rebalance_meta = self.rebalance_meta.write().await;
798 debug!(
799 event = EVENT_REBALANCE_STATE,
800 component = LOG_COMPONENT_ECSTORE,
801 subsystem = LOG_SUBSYSTEM_REBALANCE,
802 pool_index = i,
803 state = "pool_stat_missing",
804 "Adding missing rebalance pool stats entry"
805 );
806 if let Some(meta) = rebalance_meta.as_mut() {
807 meta.pool_stats.push(RebalanceStats::default());
808 }
809 ok = true;
810 drop(rebalance_meta);
811 }
812 }
813
814 if ok {
815 debug!(
816 event = EVENT_REBALANCE_STATE,
817 component = LOG_COMPONENT_ECSTORE,
818 subsystem = LOG_SUBSYSTEM_REBALANCE,
819 state = "metadata_saving",
820 "Saving rebalance metadata after stats refresh"
821 );
822
823 let rebalance_meta = self.rebalance_meta.read().await;
824 if let Some(meta) = rebalance_meta.as_ref() {
825 let pool = clone_first_arc(&self.pools, "update_rebalance_stats: no pools available")?;
826 resolve_rebalance_meta_save_result(
827 self.save_rebalance_meta_for_id_with_merge(pool, meta, "update_rebalance_stats", meta.id.as_str())
828 .await,
829 "update_rebalance_stats",
830 )?;
831 }
832 }
833
834 Ok(())
835 }
836
837 #[tracing::instrument(skip(self))]
838 pub async fn init_rebalance_meta(&self, bucktes: Vec<String>) -> Result<String> {
839 info!(
840 event = EVENT_REBALANCE_STATE,
841 component = LOG_COMPONENT_ECSTORE,
842 subsystem = LOG_SUBSYSTEM_REBALANCE,
843 state = "initializing",
844 bucket_count = bucktes.len(),
845 "Initializing rebalance metadata"
846 );
847 let si = StorageAdminApi::storage_info(self).await;
848
849 let mut disk_stats = vec![DiskStat::default(); self.pools.len()];
850
851 let mut total_cap = 0;
852 let mut total_free = 0;
853 for disk in si.disks.iter() {
854 if disk.pool_index < 0 || disk_stats.len() <= disk.pool_index as usize {
855 continue;
856 }
857
858 total_cap += disk.total_space;
859 total_free += disk.available_space;
860
861 disk_stats[disk.pool_index as usize].total_space += disk.total_space;
862 disk_stats[disk.pool_index as usize].available_space += disk.available_space;
863 }
864
865 #[cfg(test)]
866 if let Some(overridden) = take_rebalance_disk_stats_override_for_test(self.id) {
867 disk_stats = overridden;
868 total_cap = disk_stats.iter().map(|stat| stat.total_space).sum();
869 total_free = disk_stats.iter().map(|stat| stat.available_space).sum();
870 }
871
872 let percent_free_goal = percent_free_ratio(total_free, total_cap);
873 validate_rebalance_disk_stats_coverage(&disk_stats)?;
874
875 let mut pool_stats = Vec::with_capacity(self.pools.len());
876
877 let now = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
878
879 for disk_stat in disk_stats.iter() {
880 let mut pool_stat = RebalanceStats {
881 init_free_space: disk_stat.available_space,
882 init_capacity: disk_stat.total_space,
883 buckets: bucktes.clone(),
884 rebalanced_buckets: Vec::with_capacity(bucktes.len()),
885 ..Default::default()
886 };
887
888 if should_pool_participate(disk_stat.available_space, disk_stat.total_space, percent_free_goal) {
889 pool_stat.participating = true;
890 pool_stat.info = RebalanceInfo {
891 start_time: Some(now),
892 status: RebalStatus::Started,
893 ..Default::default()
894 };
895 }
896
897 pool_stats.push(pool_stat);
898 }
899
900 let has_participating_pool = pool_stats.iter().any(|pool_stat| pool_stat.participating);
901 let meta = RebalanceMeta {
902 id: Uuid::new_v4().to_string(),
903 stopped_at: (!has_participating_pool).then_some(now),
904 percent_free_goal,
905 pool_stats,
906 ..Default::default()
907 };
908
909 let pool = clone_first_arc(&self.pools, "init_rebalance_meta: no pools available")?;
910 resolve_rebalance_meta_save_result(
911 self.save_rebalance_activation_meta_with_merge(pool, &meta, "init_rebalance_meta")
912 .await,
913 "init_rebalance_meta",
914 )?;
915
916 info!(
917 event = EVENT_REBALANCE_STATE,
918 component = LOG_COMPONENT_ECSTORE,
919 subsystem = LOG_SUBSYSTEM_REBALANCE,
920 state = "metadata_initialized",
921 bucket_count = bucktes.len(),
922 "Rebalance metadata initialized"
923 );
924
925 let id = meta.id.clone();
926
927 {
928 let mut rebalance_meta = self.rebalance_meta.write().await;
929 *rebalance_meta = Some(meta);
930 drop(rebalance_meta);
931 }
932
933 Ok(id)
934 }
935
936 #[tracing::instrument(skip(self, bucktes))]
937 pub async fn init_and_start_rebalance(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
938 let id = self.init_rebalance_start(bucktes).await?;
939 if let Err(start_err) = self.start_rebalance_for_id(&id).await {
940 if let Err(rollback_err) = self
941 .rollback_rebalance_start_without_worker_for_id(Some(&id), start_err.to_string())
942 .await
943 {
944 return Err(Error::other(format!(
945 "failed to start rebalance after metadata initialized for {id}: {start_err}; rollback failed: {rollback_err}"
946 )));
947 }
948
949 return Err(Error::other(format!(
950 "failed to start rebalance after metadata initialized for {id}; local metadata was finalized as failed: {start_err}"
951 )));
952 }
953
954 Ok(id)
955 }
956
957 #[tracing::instrument(skip(self, bucktes))]
958 pub async fn init_rebalance_start(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
959 let _start_guard = self.start_gate.lock().await;
960 {
961 let decommission_running = self.is_decommission_running().await;
962 let rebalance_meta = self.rebalance_meta.read().await;
963 validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
964 }
965 let _activation_guard = self
966 .rebalance_activation_write_guard(None, "initialize replacement rebalance")
967 .await?;
968 let movement_gate = self.ctx.data_movement_operation_gate();
969 let _movement_guard = movement_gate.write().await;
970
971 let id = self.init_rebalance_meta(bucktes).await?;
972 self.ctx.advance_data_movement_operation_epoch();
973 Ok(id)
974 }
975
976 #[tracing::instrument(skip(self))]
977 pub async fn start_rebalance_for_id(self: &Arc<Self>, expected_id: &str) -> Result<()> {
978 let _start_guard = self.start_gate.lock().await;
979 let _activation_guard = self
980 .rebalance_activation_write_guard(Some(expected_id), "start rebalance")
981 .await?;
982 let movement_gate = self.ctx.data_movement_operation_gate();
983 let _movement_guard = movement_gate.write().await;
984
985 {
986 let rebalance_meta = self.rebalance_meta.read().await;
987 let Some(meta) = rebalance_meta.as_ref() else {
988 return Err(Error::ConfigNotFound);
989 };
990 if meta.id != expected_id {
991 return Err(Error::other(format!(
992 "rebalance metadata changed before start: expected {expected_id}, found {}",
993 meta.id
994 )));
995 }
996 if meta.stopped_at.is_some() {
997 if !is_rebalance_conflicting_with_decommission(meta) {
998 debug!(
999 event = EVENT_REBALANCE_STATE,
1000 component = LOG_COMPONENT_ECSTORE,
1001 subsystem = LOG_SUBSYSTEM_REBALANCE,
1002 state = "start_skipped",
1003 reason = "not_started_terminal",
1004 rebalance_id = %expected_id,
1005 "Skipped rebalance start because metadata is already terminal"
1006 );
1007 return Ok(());
1008 }
1009 return Err(Error::other(format!("rebalance {expected_id} was stopped before start")));
1010 }
1011 }
1012
1013 if self.start_rebalance_under_gate().await? {
1014 self.ctx.advance_data_movement_operation_epoch();
1015 }
1016 Ok(())
1017 }
1018
1019 pub async fn rollback_rebalance_start_for_id(self: &Arc<Self>, expected_id: Option<&str>, start_error: String) -> Result<()> {
1020 self.rollback_rebalance_start_without_worker_for_id(expected_id, start_error)
1021 .await
1022 }
1023
1024 #[tracing::instrument(skip(self, fi))]
1025 pub async fn update_pool_stats(&self, pool_index: usize, bucket: String, fi: &FileInfo) -> Result<()> {
1026 self.update_pool_stats_batch(pool_index, bucket, &[fi]).await
1027 }
1028
1029 #[tracing::instrument(skip(self, versions))]
1030 pub async fn update_pool_stats_batch(&self, pool_index: usize, bucket: String, versions: &[&FileInfo]) -> Result<()> {
1031 self.update_pool_stats_batch_for_id(pool_index, bucket, versions, None).await
1032 }
1033
1034 pub(super) async fn update_pool_stats_batch_for_rebalance(
1035 &self,
1036 pool_index: usize,
1037 bucket: String,
1038 versions: &[&FileInfo],
1039 expected_id: &str,
1040 ) -> Result<()> {
1041 self.update_pool_stats_batch_for_id(pool_index, bucket, versions, Some(expected_id))
1042 .await
1043 }
1044
1045 async fn update_pool_stats_batch_for_id(
1046 &self,
1047 pool_index: usize,
1048 bucket: String,
1049 versions: &[&FileInfo],
1050 expected_id: Option<&str>,
1051 ) -> Result<()> {
1052 if versions.is_empty() {
1053 return Ok(());
1054 }
1055
1056 let mut rebalance_meta = self.rebalance_meta.write().await;
1057 if let Some(expected_id) = expected_id {
1058 ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "update rebalance pool stats")?;
1059 }
1060 if let Some(meta) = rebalance_meta.as_mut() {
1061 if !should_accept_rebalance_stats_update(meta, pool_index) {
1062 return Ok(());
1063 }
1064
1065 if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
1066 pool_stat.update_batch(bucket, versions);
1067 }
1068 }
1069
1070 Ok(())
1071 }
1072
1073 #[tracing::instrument(skip(self))]
1074 pub async fn next_rebal_bucket(&self, pool_index: usize, expected_id: &str) -> Result<Option<String>> {
1075 let rebalance_meta = self.rebalance_meta.read().await;
1076 ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "next rebalance bucket")?;
1077 debug!(
1078 event = EVENT_REBALANCE_BUCKET,
1079 component = LOG_COMPONENT_ECSTORE,
1080 subsystem = LOG_SUBSYSTEM_REBALANCE,
1081 pool_index,
1082 has_meta = rebalance_meta.is_some(),
1083 state = "next_bucket_lookup",
1084 "Rebalance next bucket lookup"
1085 );
1086 resolve_next_rebalance_bucket(rebalance_meta.as_ref(), pool_index)
1087 }
1088
1089 #[tracing::instrument(skip(self))]
1090 pub async fn bucket_rebalance_done(&self, pool_index: usize, bucket: String, expected_id: &str) -> Result<()> {
1091 let mut rebalance_meta = self.rebalance_meta.write().await;
1092 ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "mark rebalance bucket done")?;
1093 mark_rebalance_bucket_done(rebalance_meta.as_mut(), pool_index, &bucket)
1094 }
1095
1096 pub(super) async fn record_rebalance_cleanup_warning(
1097 &self,
1098 pool_index: usize,
1099 bucket: &str,
1100 object: &str,
1101 message: String,
1102 expected_id: &str,
1103 ) -> Result<()> {
1104 let mut rebalance_meta = self.rebalance_meta.write().await;
1105 ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "record rebalance cleanup warning")?;
1106 record_rebalance_cleanup_warning_in_meta(
1107 rebalance_meta.as_mut(),
1108 pool_index,
1109 bucket,
1110 object,
1111 message,
1112 OffsetDateTime::now_utc(),
1113 )
1114 }
1115
1116 pub(super) async fn defer_rebalance_bucket(
1117 &self,
1118 pool_index: usize,
1119 bucket: String,
1120 last_error: String,
1121 expected_id: &str,
1122 kind: RebalanceDeferKind,
1123 ) -> Result<()> {
1124 let mut rebalance_meta = self.rebalance_meta.write().await;
1125 ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "defer rebalance bucket")?;
1126 let Some(meta) = rebalance_meta.as_mut() else {
1127 return Err(rebalance_metadata_not_initialized_error("defer rebalance bucket"));
1128 };
1129 let pool_count = meta.pool_stats.len();
1130 ensure_valid_rebalance_pool_index(pool_count, pool_index)?;
1131 let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) else {
1132 return Err(invalid_rebalance_pool_index_error(pool_index, pool_count));
1133 };
1134
1135 defer_bucket_in_rebalance_queue(pool_stat, &bucket)?;
1136 let pending_entry_defer = pool_stat.info.last_error.clone();
1137 pool_stat.info.last_error = resolve_rebalance_deferred_last_error(kind, pending_entry_defer.as_deref(), &last_error);
1138 meta.last_refreshed_at = Some(OffsetDateTime::now_utc());
1139 Ok(())
1140 }
1141
1142 pub async fn is_rebalance_started(&self) -> bool {
1143 let rebalance_meta = self.rebalance_meta.read().await;
1144 if let Some(meta) = rebalance_meta.as_ref() {
1145 meta.pool_stats.iter().enumerate().for_each(|(i, v)| {
1146 debug!(
1147 event = EVENT_REBALANCE_STATE,
1148 component = LOG_COMPONENT_ECSTORE,
1149 subsystem = LOG_SUBSYSTEM_REBALANCE,
1150 pool_index = i,
1151 participating = v.participating,
1152 status = ?v.info.status,
1153 state = "status_inspected",
1154 "Rebalance status inspected"
1155 );
1156 });
1157
1158 let started = is_rebalance_conflicting_with_decommission(meta);
1159 if started {
1160 debug!(
1161 event = EVENT_REBALANCE_STATE,
1162 component = LOG_COMPONENT_ECSTORE,
1163 subsystem = LOG_SUBSYSTEM_REBALANCE,
1164 state = "running",
1165 "Rebalance is running"
1166 );
1167 return true;
1168 }
1169 }
1170
1171 debug!(
1172 event = EVENT_REBALANCE_STATE,
1173 component = LOG_COMPONENT_ECSTORE,
1174 subsystem = LOG_SUBSYSTEM_REBALANCE,
1175 state = "not_running",
1176 "Rebalance is not running"
1177 );
1178 false
1179 }
1180
1181 pub async fn is_rebalance_conflicting_with_decommission(&self) -> bool {
1182 let rebalance_meta = self.rebalance_meta.read().await;
1183 rebalance_meta
1184 .as_ref()
1185 .is_some_and(is_rebalance_conflicting_with_decommission)
1186 }
1187
1188 pub async fn is_pool_rebalancing(&self, pool_index: usize) -> bool {
1189 let rebalance_meta = self.rebalance_meta.read().await;
1190 if let Some(ref meta) = *rebalance_meta {
1191 if meta.stopped_at.is_some() {
1192 return false;
1193 }
1194
1195 if let Some(pool_stat) = meta.pool_stats.get(pool_index) {
1196 return pool_stat.participating && pool_stat.info.status == RebalStatus::Started;
1197 }
1198 }
1199
1200 false
1201 }
1202
1203 pub async fn pool_rebalance_status(&self, pool_index: usize) -> (RebalStatus, bool) {
1204 let rebalance_meta = self.rebalance_meta.read().await;
1205 pool_rebalance_status_from_meta(rebalance_meta.as_ref(), pool_index)
1206 }
1207
1208 pub async fn current_rebalance_id(&self) -> Option<String> {
1209 let rebalance_meta = self.rebalance_meta.read().await;
1210 rebalance_meta
1211 .as_ref()
1212 .and_then(|meta| (!meta.id.is_empty()).then(|| meta.id.clone()))
1213 }
1214
1215 pub async fn cancel_rebalance_admission_for_id(&self, expected_id: &str) -> Result<()> {
1216 let _start_guard = self.start_gate.lock().await;
1217 let mut rebalance_meta = self.rebalance_meta.write().await;
1218 ensure_rebalance_run_id(rebalance_meta.as_ref(), expected_id, "cancel rebalance admission")?;
1219 let meta = rebalance_meta
1220 .as_mut()
1221 .ok_or_else(|| rebalance_metadata_not_initialized_error("cancel rebalance admission"))?;
1222 if meta.stopped_at.is_some() || (!is_rebalance_conflicting_with_decommission(meta) && !meta.stop_requested) {
1223 return Err(Error::other(format!(
1224 "inactive rebalance rejected while cancelling admission: {expected_id}"
1225 )));
1226 }
1227 meta.stop_requested = true;
1228 meta.cancel
1229 .get_or_insert_with(tokio_util::sync::CancellationToken::new)
1230 .cancel();
1231 Ok(())
1232 }
1233
1234 #[tracing::instrument(skip(self))]
1235 pub async fn stop_rebalance(self: &Arc<Self>) -> Result<()> {
1236 self.stop_rebalance_for_id(None).await
1237 }
1238
1239 #[tracing::instrument(skip(self))]
1240 pub async fn stop_rebalance_for_id(self: &Arc<Self>, expected_id: Option<&str>) -> Result<()> {
1241 let _start_guard = self.start_gate.lock().await;
1242 let activation_gate = {
1243 let mut rebalance_meta = self.rebalance_meta.write().await;
1244 if let Some(expected_id) = expected_id {
1245 ensure_rebalance_run_id(rebalance_meta.as_ref(), expected_id, "stop rebalance")?;
1246 }
1247 rebalance_meta.as_mut().map(|meta| {
1248 meta.stop_requested |= is_rebalance_conflicting_with_decommission(meta);
1249 let cancel = meta.cancel.get_or_insert_with(tokio_util::sync::CancellationToken::new);
1250 cancel.cancel();
1251 Arc::clone(&meta.activation_gate)
1252 })
1253 };
1254 let _activation_guard = match activation_gate {
1255 Some(gate) => {
1256 #[cfg(any(test, feature = "test-util"))]
1257 observe_rebalance_stop_wait_attempt(expected_id);
1258 Some(gate.write_owned().await)
1259 }
1260 None => None,
1261 };
1262 let movement_gate = self.ctx.data_movement_operation_gate();
1263 let _movement_guard = movement_gate.write().await;
1264 let stopped_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
1265 let (previous_meta, meta_to_save) = {
1266 let mut rebalance_meta = self.rebalance_meta.write().await;
1267 let previous_meta = rebalance_meta.clone();
1268 let meta_to_save = stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), stopped_at, expected_id)?;
1269 (previous_meta, meta_to_save)
1270 };
1271
1272 if let Some(meta_to_save) = meta_to_save {
1273 let pool = clone_first_arc(self.pools.as_slice(), "stop_rebalance: no pools available")?;
1274 let save_result = resolve_rebalance_meta_save_result(
1275 self.save_rebalance_meta_for_id_with_merge(pool, &meta_to_save, "stop_rebalance", meta_to_save.id.as_str())
1276 .await,
1277 "stop_rebalance",
1278 );
1279 if let Err(err) = save_result {
1280 *self.rebalance_meta.write().await = previous_meta;
1281 return Err(err);
1282 }
1283 self.ctx.advance_data_movement_operation_epoch();
1284 }
1285
1286 Ok(())
1287 }
1288
1289 async fn rollback_rebalance_start_without_worker_for_id(
1290 self: &Arc<Self>,
1291 expected_id: Option<&str>,
1292 start_error: String,
1293 ) -> Result<()> {
1294 let _start_guard = self.start_gate.lock().await;
1295 let _activation_guard = self
1296 .rebalance_activation_write_guard(expected_id, "rollback rebalance start")
1297 .await?;
1298 let movement_gate = self.ctx.data_movement_operation_gate();
1299 let _movement_guard = movement_gate.write().await;
1300 let failed_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
1301 let meta_to_save = {
1302 let mut rebalance_meta = self.rebalance_meta.write().await;
1303 rollback_rebalance_start_meta_snapshot_for_id(rebalance_meta.as_mut(), failed_at, expected_id, start_error)
1304 };
1305
1306 if let Some(meta_to_save) = meta_to_save {
1307 let pool = clone_first_arc(self.pools.as_slice(), "rollback_rebalance_start: no pools available")?;
1308 resolve_rebalance_meta_save_result(
1309 self.save_rebalance_meta_for_id_with_merge(
1310 pool,
1311 &meta_to_save,
1312 "rollback_rebalance_start",
1313 meta_to_save.id.as_str(),
1314 )
1315 .await,
1316 "rollback_rebalance_start",
1317 )?;
1318 self.ctx.advance_data_movement_operation_epoch();
1319 }
1320
1321 Ok(())
1322 }
1323
1324 pub async fn record_rebalance_stop_propagation(
1325 self: &Arc<Self>,
1326 expected_id: &str,
1327 record: RebalanceStopPropagationRecord,
1328 ) -> Result<()> {
1329 if !record.has_failures() {
1330 return Ok(());
1331 }
1332
1333 let _start_guard = self.start_gate.lock().await;
1334 let _activation_guard = self
1335 .rebalance_activation_write_guard(Some(expected_id), "record rebalance stop propagation")
1336 .await?;
1337 let movement_gate = self.ctx.data_movement_operation_gate();
1338 let _movement_guard = movement_gate.write().await;
1339 let encoded_error = encode_rebalance_stop_propagation_record(&record);
1340 let meta_to_save = {
1341 let mut rebalance_meta = self.rebalance_meta.write().await;
1342 ensure_rebalance_run_id(rebalance_meta.as_ref(), expected_id, "record rebalance stop propagation")?;
1343 record_rebalance_stop_propagation_snapshot(rebalance_meta.as_mut(), encoded_error, OffsetDateTime::now_utc())
1344 };
1345
1346 if let Some(meta_to_save) = meta_to_save {
1347 let pool = clone_first_arc(self.pools.as_slice(), "record_rebalance_stop_propagation: no pools available")?;
1348 resolve_rebalance_meta_save_result(
1349 self.save_rebalance_meta_for_id_with_merge(pool, &meta_to_save, "record_rebalance_stop_propagation", expected_id)
1350 .await,
1351 "record_rebalance_stop_propagation",
1352 )?;
1353 self.ctx.advance_data_movement_operation_epoch();
1354 }
1355
1356 Ok(())
1357 }
1358}
1359
1360#[cfg(test)]
1361mod tests {
1362 use super::*;
1363 use crate::config::com::delete_config;
1364 use crate::core::pools::{
1365 DecommissionErasureLayout, DecommissionPoolCapacityInfo, POOL_META_NAME, PoolActivationDurableSaveBarrier,
1366 PoolActivationStartKind, PoolActivationStartProbe, PoolMetaWriteState, persist_pool_meta_identity_for_startup,
1367 set_decommission_capacity_info_overrides_for_test,
1368 };
1369 use crate::object_api::NamespaceLockFence;
1370 use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause, hermetic_set_disks_isolated};
1371
1372 #[tokio::test]
1373 async fn rebalance_status_read_recovers_after_metadata_lock_timeout() {
1374 let id = "rebalance-status-lock-retry";
1375 let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1376 id: id.to_string(),
1377 pool_stats: vec![RebalanceStats::default()],
1378 ..Default::default()
1379 })
1380 .await;
1381 let ns_lock = store.pools[0]
1382 .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME)
1383 .await
1384 .expect("metadata lock should be available");
1385 let writer = ns_lock
1386 .get_write_lock(get_lock_acquire_timeout())
1387 .await
1388 .expect("hold metadata writer");
1389 let retried = Arc::new(tokio::sync::Notify::new());
1390 let refresh = super::super::worker::REBALANCE_METADATA_RETRY_PROBE
1391 .scope(Arc::clone(&retried), store.refresh_rebalance_status_meta());
1392 tokio::pin!(refresh);
1393 tokio::time::timeout(std::time::Duration::from_secs(30), async {
1394 tokio::select! {
1395 _ = retried.notified() => {},
1396 result = &mut refresh => panic!("status read must retry its contended metadata lock: {result:?}"),
1397 }
1398 })
1399 .await
1400 .expect("status read should encounter a real lock timeout");
1401 drop(writer);
1402 tokio::time::timeout(std::time::Duration::from_secs(30), refresh)
1403 .await
1404 .expect("status refresh should finish after the writer releases")
1405 .expect("transient metadata contention must not fail status refresh");
1406 assert_eq!(
1407 store
1408 .rebalance_meta
1409 .read()
1410 .await
1411 .as_ref()
1412 .expect("metadata must remain present")
1413 .id,
1414 id
1415 );
1416 }
1417
1418 #[tokio::test]
1419 async fn rebalance_stats_save_recovers_after_metadata_lock_timeout() {
1420 let id = "rebalance-stats-lock-retry";
1421 let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1422 id: id.to_string(),
1423 pool_stats: vec![RebalanceStats::default()],
1424 ..Default::default()
1425 })
1426 .await;
1427 store
1428 .rebalance_meta
1429 .write()
1430 .await
1431 .as_mut()
1432 .expect("local metadata must exist")
1433 .pool_stats[0]
1434 .num_objects = 7;
1435 let ns_lock = store.pools[0]
1436 .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME)
1437 .await
1438 .expect("metadata lock should be available");
1439 let reader = ns_lock
1440 .get_read_lock(get_lock_acquire_timeout())
1441 .await
1442 .expect("hold a migration read fence");
1443 let retried = Arc::new(tokio::sync::Notify::new());
1444 let save = super::super::worker::REBALANCE_METADATA_RETRY_PROBE.scope(
1445 Arc::clone(&retried),
1446 store.save_rebalance_stats_for_id(0, super::super::RebalSaveOpt::Stats, id),
1447 );
1448 tokio::pin!(save);
1449 tokio::time::timeout(std::time::Duration::from_secs(30), async {
1450 tokio::select! {
1451 _ = retried.notified() => {},
1452 result = &mut save => panic!("stats save must retry its contended metadata lock: {result:?}"),
1453 }
1454 })
1455 .await
1456 .expect("stats save should encounter a real lock timeout");
1457 drop(reader);
1458 tokio::time::timeout(std::time::Duration::from_secs(30), save)
1459 .await
1460 .expect("stats save should finish after the migration read fence releases")
1461 .expect("transient metadata contention must not lose stats");
1462 let mut persisted = RebalanceMeta::new();
1463 persisted
1464 .load(Arc::clone(&store.pools[0]))
1465 .await
1466 .expect("persisted stats must be readable");
1467 assert_eq!(persisted.id, id);
1468 assert_eq!(persisted.pool_stats[0].num_objects, 7);
1469 }
1470
1471 async fn persist_initialized_identity_then_remove_pool_meta(store: &Arc<ECStore>) {
1472 let deployment_id = store
1473 .ctx
1474 .deployment_id()
1475 .expect("test store should have a deployment identity");
1476 let mut write_state = PoolMetaWriteState::for_startup(deployment_id, false);
1477 persist_pool_meta_identity_for_startup(store.pools.clone(), &mut write_state, true)
1478 .await
1479 .expect("initialized pool metadata identity should persist");
1480 *store.pool_meta_save_gate.lock().await = write_state;
1481 for pool in &store.pools {
1482 delete_config(pool.clone(), POOL_META_NAME)
1483 .await
1484 .expect("every pool metadata replica should be removed");
1485 }
1486 }
1487
1488 async fn assert_activation_locks_released(store: &Arc<ECStore>) {
1489 let fleet_proof = acquire_pool_activation_fleet_proof(&store.ctx)
1490 .await
1491 .expect("fleet proof should remain available");
1492 tokio::time::timeout(
1493 std::time::Duration::from_secs(5),
1494 acquire_pool_rebalance_activation_locks(store.pools[0].clone(), fleet_proof),
1495 )
1496 .await
1497 .expect("a rejected activation must release namespace fences promptly")
1498 .expect("a rejected activation must release both namespace fences");
1499 }
1500
1501 #[tokio::test]
1502 async fn rebalance_stop_wait_probe_matches_run_id() {
1503 let probe = RebalanceStopWaitProbe::install("rebalance-stop-current");
1504
1505 observe_rebalance_stop_wait_attempt(Some("rebalance-stop-stale"));
1506 assert!(!probe.state.attempted.load(std::sync::atomic::Ordering::Acquire));
1507
1508 observe_rebalance_stop_wait_attempt(Some("rebalance-stop-current"));
1509 probe.wait_until_attempted().await;
1510 }
1511
1512 #[test]
1513 fn rebalance_stop_classification_checks_identity_and_explicit_intent() {
1514 let mut meta = RebalanceMeta {
1515 id: "current".to_string(),
1516 cancel: Some(tokio_util::sync::CancellationToken::new()),
1517 pool_stats: vec![RebalanceStats {
1518 participating: true,
1519 info: RebalanceInfo {
1520 status: RebalStatus::Started,
1521 ..Default::default()
1522 },
1523 ..Default::default()
1524 }],
1525 ..Default::default()
1526 };
1527 ensure_rebalance_worker_active(Some(&meta), "current", "test").expect("active worker");
1528 meta.cancel.as_ref().unwrap().cancel();
1529 assert!(
1530 !matches!(
1531 ensure_rebalance_worker_active(Some(&meta), "current", "test"),
1532 Err(Error::OperationCanceled)
1533 ),
1534 "a sibling failure is not an operator stop"
1535 );
1536 meta.stop_requested = true;
1537 assert!(matches!(
1538 ensure_rebalance_worker_active(Some(&meta), "current", "test"),
1539 Err(Error::OperationCanceled)
1540 ));
1541 assert!(
1542 !matches!(ensure_rebalance_worker_active(Some(&meta), "old", "test"), Err(Error::OperationCanceled)),
1543 "stale identity remains a failure even during stop"
1544 );
1545 assert!(!matches!(
1546 ensure_rebalance_worker_active(None, "current", "test"),
1547 Err(Error::OperationCanceled)
1548 ));
1549 }
1550
1551 #[tokio::test]
1552 async fn rebalance_stop_intent_does_not_survive_replacement_run_reload() {
1553 let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1554 id: "replacement".to_string(),
1555 pool_stats: vec![RebalanceStats {
1556 participating: true,
1557 info: RebalanceInfo {
1558 status: RebalStatus::Completed,
1559 ..Default::default()
1560 },
1561 ..Default::default()
1562 }],
1563 ..Default::default()
1564 })
1565 .await;
1566 let previous_gate = {
1567 let mut meta = store.rebalance_meta.write().await;
1568 let meta = meta.as_mut().unwrap();
1569 meta.id = "previous".to_string();
1570 meta.stop_requested = true;
1571 let cancel = tokio_util::sync::CancellationToken::new();
1572 cancel.cancel();
1573 meta.cancel = Some(cancel);
1574 Arc::clone(&meta.activation_gate)
1575 };
1576 store.load_rebalance_meta().await.expect("reload replacement run");
1577 let meta = store.rebalance_meta.read().await;
1578 let meta = meta.as_ref().unwrap();
1579 assert_eq!(meta.id, "replacement");
1580 assert!(!meta.stop_requested);
1581 assert!(meta.cancel.is_none());
1582 assert!(!Arc::ptr_eq(&previous_gate, &meta.activation_gate));
1583 }
1584
1585 #[tokio::test]
1586 async fn cancel_rebalance_admission_is_id_checked_and_idempotent() {
1587 let rebalance_id = "rebalance-admission-current";
1588 let cancel = tokio_util::sync::CancellationToken::new();
1589 let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1590 id: rebalance_id.to_string(),
1591 percent_free_goal: 1.0,
1592 cancel: Some(cancel.clone()),
1593 pool_stats: vec![RebalanceStats {
1594 participating: true,
1595 init_capacity: 100,
1596 info: RebalanceInfo {
1597 start_time: Some(OffsetDateTime::now_utc()),
1598 status: RebalStatus::Started,
1599 ..Default::default()
1600 },
1601 ..Default::default()
1602 }],
1603 ..Default::default()
1604 })
1605 .await;
1606
1607 let error = store
1608 .cancel_rebalance_admission_for_id("rebalance-admission-replacement")
1609 .await
1610 .expect_err("a stale admin stop must not cancel the current run");
1611 assert!(error.to_string().contains("expected rebalance-admission-replacement"));
1612 assert!(!cancel.is_cancelled());
1613
1614 store
1615 .cancel_rebalance_admission_for_id(rebalance_id)
1616 .await
1617 .expect("the current run admission should close");
1618 store
1619 .cancel_rebalance_admission_for_id(rebalance_id)
1620 .await
1621 .expect("retrying admission cancellation should be idempotent");
1622 assert!(cancel.is_cancelled());
1623 let err = store
1624 .update_pool_stats_batch_for_rebalance(0, "bucket".to_string(), &[&FileInfo::default()], rebalance_id)
1625 .await
1626 .expect_err("stop racing with a final stats update must cancel that update");
1627 assert!(matches!(err, Error::OperationCanceled), "operator stop lost its cancellation type: {err}");
1628 }
1629
1630 #[tokio::test]
1631 async fn prepare_rebalance_stop_preserves_intent_when_worker_stops_before_reload() {
1632 let id = "stop-worker-before-reload";
1633 let cancel = tokio_util::sync::CancellationToken::new();
1634 let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1635 id: id.to_string(),
1636 cancel: Some(cancel.clone()),
1637 pool_stats: vec![RebalanceStats {
1638 participating: true,
1639 info: RebalanceInfo {
1640 status: RebalStatus::Stopped,
1641 ..Default::default()
1642 },
1643 ..Default::default()
1644 }],
1645 ..Default::default()
1646 })
1647 .await;
1648 let gate = {
1649 let mut meta = store.rebalance_meta.write().await;
1650 let meta = meta.as_mut().expect("local rebalance metadata");
1651 meta.pool_stats[0].info.status = RebalStatus::Started;
1652 Arc::clone(&meta.activation_gate)
1653 };
1654 assert_eq!(
1655 store.prepare_rebalance_stop().await.expect("prepare the same run stop"),
1656 Some(id.to_string())
1657 );
1658 {
1659 let meta = store.rebalance_meta.read().await;
1660 let meta = meta.as_ref().expect("reloaded stop target");
1661 assert!(Arc::ptr_eq(&gate, &meta.activation_gate), "reload must retain the drained run's gate");
1662 assert!(meta.cancel.as_ref().is_some_and(|token| token.is_cancelled()));
1663 }
1664 store
1665 .stop_rebalance_for_id(Some(id))
1666 .await
1667 .expect("finish the stop after the worker's terminal event");
1668 store
1669 .load_rebalance_meta()
1670 .await
1671 .expect("reload the acknowledged durable stop");
1672 let meta = store.rebalance_meta.read().await;
1673 let meta = meta.as_ref().expect("durable stopped metadata");
1674 assert!(meta.stopped_at.is_some(), "a successful stop must retain its durable timestamp");
1675 assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped);
1676 }
1677
1678 #[tokio::test]
1679 #[serial_test::serial]
1680 async fn equal_free_ratio_admin_no_participant_rebalance_succeeds_and_persists_terminal_generation_after_restart() {
1681 let (_temp_dirs, store, restarted) =
1682 crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
1683 let movement_floor = OffsetDateTime::from_unix_timestamp(4_100_000_000).expect("future test timestamp should be valid");
1684 *store.rebalance_meta.write().await = Some(RebalanceMeta {
1685 id: "previous-terminal-rebalance".to_string(),
1686 stopped_at: Some(movement_floor),
1687 ..Default::default()
1688 });
1689 set_rebalance_disk_stats_override_for_test(
1690 store.id,
1691 vec![
1692 DiskStat {
1693 total_space: 100,
1694 available_space: 50,
1695 },
1696 DiskStat {
1697 total_space: 100,
1698 available_space: 50,
1699 },
1700 ],
1701 );
1702
1703 let rebalance_id = store
1704 .init_and_start_rebalance(vec!["equal-ratio-no-op".to_string()])
1705 .await
1706 .expect("equal free ratio admin rebalance should succeed as a terminal no-op");
1707 let stopped_at = {
1708 let local = store.rebalance_meta.read().await;
1709 let local = local.as_ref().expect("no-op rebalance metadata should remain available");
1710 assert_eq!(local.id, rebalance_id);
1711 assert!(local.pool_stats.iter().all(|pool_stat| !pool_stat.participating));
1712 let stopped_at = local.stopped_at.expect("no-op rebalance must persist a terminal timestamp");
1713 assert_eq!(stopped_at, movement_floor + time::Duration::nanoseconds(1));
1714 stopped_at
1715 };
1716
1717 let stopped_generation =
1718 u64::try_from(stopped_at.unix_timestamp_nanos()).expect("terminal timestamp should map to scanner generation");
1719 let live_status = store.scanner_data_movement_pause_status().await;
1720 assert!(!live_status.paused);
1721 assert_eq!(live_status.movement_generation, stopped_generation);
1722
1723 restarted
1724 .load_rebalance_meta()
1725 .await
1726 .expect("restarted store should load the persisted no-op rebalance metadata");
1727 let status = restarted.scanner_data_movement_pause_status().await;
1728
1729 assert!(!status.paused);
1730 assert_eq!(status.movement_generation, stopped_generation);
1731 assert_eq!(restarted.scanner_data_movement_generation(), stopped_generation);
1732 }
1733
1734 #[tokio::test]
1735 #[serial_test::serial]
1736 async fn rebalance_activation_rejects_initialized_cluster_with_all_pool_meta_missing() {
1737 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
1738 persist_initialized_identity_then_remove_pool_meta(&store).await;
1739 set_rebalance_disk_stats_override_for_test(
1740 store.id,
1741 vec![
1742 DiskStat {
1743 total_space: 100,
1744 available_space: 0,
1745 },
1746 DiskStat {
1747 total_space: 100,
1748 available_space: 100,
1749 },
1750 ],
1751 );
1752
1753 let err = store
1754 .init_rebalance_start(vec!["missing-pool-meta".to_string()])
1755 .await
1756 .expect_err("rebalance activation must fail closed when every pool.bin is missing");
1757 assert!(err.to_string().contains("initialized cluster identity exists"));
1758 store
1759 .ensure_pool_meta_side_effects_safe("rebalance activation after missing pool metadata")
1760 .await
1761 .expect_err("the missing metadata observation must latch the shared runtime gate");
1762
1763 let mut persisted = RebalanceMeta::new();
1764 assert!(
1765 matches!(persisted.load(store.pools[0].clone()).await, Err(Error::ConfigNotFound)),
1766 "rejected activation must not create rebalance metadata"
1767 );
1768 assert_activation_locks_released(&store).await;
1769 }
1770
1771 #[tokio::test]
1772 #[serial_test::serial]
1773 async fn rebalance_worker_rejects_initialized_cluster_with_all_pool_meta_missing() {
1774 let rebalance_id = "missing-pool-meta-worker";
1775 let active = RebalanceMeta {
1776 id: rebalance_id.to_string(),
1777 percent_free_goal: 0.5,
1778 pool_stats: vec![
1779 RebalanceStats {
1780 participating: true,
1781 init_capacity: 100,
1782 info: RebalanceInfo {
1783 status: RebalStatus::Started,
1784 ..Default::default()
1785 },
1786 ..Default::default()
1787 },
1788 RebalanceStats {
1789 participating: true,
1790 init_capacity: 100,
1791 info: RebalanceInfo {
1792 status: RebalStatus::Started,
1793 ..Default::default()
1794 },
1795 ..Default::default()
1796 },
1797 ],
1798 ..Default::default()
1799 };
1800 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(active)).await;
1801 persist_initialized_identity_then_remove_pool_meta(&store).await;
1802
1803 let err = match store
1804 .fence_rebalance_worker_activation(store.pools[0].clone(), rebalance_id)
1805 .await
1806 {
1807 Ok(_) => panic!("worker activation must not return a fence when every pool.bin is missing"),
1808 Err(err) => err,
1809 };
1810 assert!(err.to_string().contains("initialized cluster identity exists"));
1811 store
1812 .ensure_pool_meta_side_effects_safe("rebalance worker after missing pool metadata")
1813 .await
1814 .expect_err("worker validation must latch the shared runtime gate");
1815 assert_activation_locks_released(&store).await;
1816 }
1817
1818 #[tokio::test]
1819 #[serial_test::serial]
1820 async fn rebalance_worker_skips_terminal_metadata_without_fleet_proof() {
1821 let rebalance_id = "terminal-metadata-without-proof";
1822 let completed = RebalanceMeta {
1823 id: rebalance_id.to_string(),
1824 pool_stats: vec![RebalanceStats {
1825 participating: true,
1826 info: RebalanceInfo {
1827 status: RebalStatus::Completed,
1828 ..Default::default()
1829 },
1830 ..Default::default()
1831 }],
1832 ..Default::default()
1833 };
1834 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(completed)).await;
1835 let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
1836
1837 let activation = store
1838 .fence_rebalance_worker_activation(store.pools[0].clone(), rebalance_id)
1839 .await
1840 .expect("terminal metadata should not require a fleet proof");
1841 assert!(matches!(activation, RebalanceWorkerActivationFence::NotStartedTerminal));
1842 }
1843
1844 #[tokio::test]
1845 #[serial_test::serial]
1846 async fn rebalance_worker_still_requires_fleet_proof_for_active_metadata() {
1847 let rebalance_id = "active-metadata-without-proof";
1848 let active = RebalanceMeta {
1849 id: rebalance_id.to_string(),
1850 pool_stats: vec![RebalanceStats {
1851 participating: true,
1852 info: RebalanceInfo {
1853 status: RebalStatus::Started,
1854 ..Default::default()
1855 },
1856 ..Default::default()
1857 }],
1858 ..Default::default()
1859 };
1860 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(active)).await;
1861 let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
1862
1863 let err = match store
1864 .fence_rebalance_worker_activation(store.pools[0].clone(), rebalance_id)
1865 .await
1866 {
1867 Ok(_) => panic!("active metadata must not be admitted without a fleet proof"),
1868 Err(err) => err,
1869 };
1870 assert!(
1871 err.to_string()
1872 .contains("pool activation requires a live fleet capability proof")
1873 );
1874 }
1875
1876 #[tokio::test]
1877 #[serial_test::serial]
1878 async fn rebalance_activation_adopts_commit_after_post_save_fence_loss() {
1879 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
1880 set_rebalance_disk_stats_override_for_test(
1881 store.id,
1882 vec![
1883 DiskStat {
1884 total_space: 100,
1885 available_space: 0,
1886 },
1887 DiskStat {
1888 total_space: 100,
1889 available_space: 100,
1890 },
1891 ],
1892 );
1893 let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
1894 let start_store = Arc::clone(&store);
1895 let start_task = tokio::spawn(async move {
1896 start_store
1897 .init_rebalance_start(vec!["post-commit-fence-loss".to_string()])
1898 .await
1899 });
1900
1901 barrier.wait_until_paused().await;
1902 barrier.release_after_fence_loss();
1903 let rebalance_id = tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
1904 .await
1905 .expect("rebalance activation should finish after its durable commit")
1906 .expect("rebalance activation task should not panic")
1907 .expect("post-commit fence loss must not report the committed activation as failed");
1908
1909 let local_meta = store.rebalance_meta.read().await;
1910 let local = local_meta
1911 .as_ref()
1912 .expect("the committed rebalance metadata should be installed locally");
1913 assert_eq!(local.id, rebalance_id);
1914 assert!(is_rebalance_conflicting_with_decommission(local));
1915 drop(local_meta);
1916
1917 let mut persisted = RebalanceMeta::new();
1918 persisted
1919 .load(store.pools[0].clone())
1920 .await
1921 .expect("the committed rebalance metadata should remain readable");
1922 assert_eq!(persisted.id, rebalance_id);
1923 assert!(is_rebalance_conflicting_with_decommission(&persisted));
1924 }
1925
1926 #[tokio::test]
1927 #[serial_test::serial]
1928 async fn rebalance_worker_admission_installs_committed_candidate_after_post_save_fence_loss() {
1929 let rebalance_id = "post-save-worker-admission";
1930 let active = RebalanceMeta {
1931 id: rebalance_id.to_string(),
1932 percent_free_goal: 0.5,
1933 pool_stats: vec![
1934 RebalanceStats {
1935 participating: true,
1936 init_free_space: 50,
1937 init_capacity: 100,
1938 buckets: vec!["completed-pool-bucket".to_string()],
1939 info: RebalanceInfo {
1940 status: RebalStatus::Started,
1941 ..Default::default()
1942 },
1943 ..Default::default()
1944 },
1945 RebalanceStats {
1946 participating: true,
1947 init_capacity: 100,
1948 buckets: vec!["active-pool-bucket".to_string()],
1949 info: RebalanceInfo {
1950 status: RebalStatus::Started,
1951 ..Default::default()
1952 },
1953 ..Default::default()
1954 },
1955 ],
1956 ..Default::default()
1957 };
1958 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(active)).await;
1959 let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
1960 let start_store = Arc::clone(&store);
1961 let start_task = tokio::spawn(async move { start_store.start_rebalance().await });
1962
1963 barrier.wait_until_paused().await;
1964 barrier.release_after_fence_loss();
1965 tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
1966 .await
1967 .expect("rebalance worker admission should finish after its durable commit")
1968 .expect("rebalance worker admission task should not panic")
1969 .expect("post-commit fence loss must not reject the committed worker candidate");
1970
1971 let local_meta = store.rebalance_meta.read().await;
1972 let local = local_meta
1973 .as_ref()
1974 .expect("the committed worker candidate should remain installed locally");
1975 assert_eq!(local.id, rebalance_id);
1976 assert_eq!(local.pool_stats[0].info.status, RebalStatus::Completed);
1977 let admitted_cancel = local
1978 .cancel
1979 .clone()
1980 .expect("the committed worker candidate should install its cancellation token");
1981 assert!(!admitted_cancel.is_cancelled());
1982 drop(local_meta);
1983
1984 admitted_cancel.cancel();
1985 assert!(admitted_cancel.is_cancelled());
1986
1987 let mut persisted = RebalanceMeta::new();
1988 persisted
1989 .load(store.pools[0].clone())
1990 .await
1991 .expect("the committed worker candidate should remain readable");
1992 assert_eq!(persisted.id, rebalance_id);
1993 assert_eq!(persisted.pool_stats[0].info.status, RebalStatus::Completed);
1994 }
1995
1996 async fn assert_real_activation_start_race(paused_kind: PoolActivationStartKind) {
1997 let (_temp_dirs, rebalance_store, decommission_store) =
1998 crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
1999 crate::services::rebalance::promote_test_pool_meta_to_v2(&rebalance_store).await;
2000 crate::services::rebalance::promote_test_pool_meta_to_v2(&decommission_store).await;
2001 let disk_stats = vec![
2002 DiskStat {
2003 total_space: 100,
2004 available_space: 0,
2005 },
2006 DiskStat {
2007 total_space: 100,
2008 available_space: 100,
2009 },
2010 ];
2011 set_rebalance_disk_stats_override_for_test(rebalance_store.id, disk_stats.clone());
2012 set_rebalance_disk_stats_override_for_test(decommission_store.id, disk_stats);
2013 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2014 let capacity_snapshot = vec![
2015 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
2016 DecommissionPoolCapacityInfo::for_test(1, layout, 200, 200, 0),
2017 ];
2018 set_decommission_capacity_info_overrides_for_test(
2020 decommission_store.id,
2021 vec![capacity_snapshot.clone(), capacity_snapshot],
2022 );
2023 let (first_object, competing_object, competing_kind) = match paused_kind {
2024 PoolActivationStartKind::Rebalance => {
2025 (REBAL_META_NAME, crate::core::pools::POOL_META_NAME, PoolActivationStartKind::Decommission)
2026 }
2027 PoolActivationStartKind::Decommission => {
2028 (crate::core::pools::POOL_META_NAME, REBAL_META_NAME, PoolActivationStartKind::Rebalance)
2029 }
2030 };
2031 let first_barrier = PutObjectCommitBarrier::install(
2032 crate::disk::RUSTFS_META_BUCKET,
2033 first_object,
2034 PutObjectCommitPause::BeforeQuotaRename,
2035 );
2036 let competing_barrier = PutObjectCommitBarrier::install(
2037 crate::disk::RUSTFS_META_BUCKET,
2038 competing_object,
2039 PutObjectCommitPause::BeforeQuotaRename,
2040 );
2041
2042 let mut rebalance_task;
2043 let mut decommission_task;
2044 let mut observed_rebalance_result = None;
2045 let mut observed_decommission_result = None;
2046 let reached_activation = if paused_kind == PoolActivationStartKind::Rebalance {
2047 let store = Arc::clone(&rebalance_store);
2048 rebalance_task =
2049 tokio::spawn(async move { store.init_rebalance_start(vec!["bucket".to_string()]).await.map(|_| ()) });
2050 tokio::select! {
2051 _ = first_barrier.wait_until_paused() => {}
2052 result = &mut rebalance_task => {
2053 panic!("real rebalance start finished before its activation commit: {:?}", result.expect("rebalance activation task should not panic"));
2054 }
2055 }
2056 let probe = PoolActivationStartProbe::install(competing_kind);
2057 let store = Arc::clone(&decommission_store);
2058 decommission_task = tokio::spawn(async move { store.start_decommission(vec![0]).await });
2059 tokio::time::timeout(std::time::Duration::from_secs(15), async {
2060 tokio::select! {
2061 _ = probe.wait_until_attempted() => true,
2062 result = &mut decommission_task => {
2063 observed_decommission_result = Some(result.expect("decommission activation task should not panic"));
2064 false
2065 }
2066 }
2067 })
2068 .await
2069 .unwrap_or(false)
2070 } else {
2071 let store = Arc::clone(&decommission_store);
2072 decommission_task = tokio::spawn(async move { store.start_decommission(vec![0]).await });
2073 tokio::select! {
2074 _ = first_barrier.wait_until_paused() => {}
2075 result = &mut decommission_task => {
2076 panic!("real decommission start finished before its activation commit: {:?}", result.expect("decommission activation task should not panic"));
2077 }
2078 }
2079 let probe = PoolActivationStartProbe::install(competing_kind);
2080 let store = Arc::clone(&rebalance_store);
2081 rebalance_task =
2082 tokio::spawn(async move { store.init_rebalance_start(vec!["bucket".to_string()]).await.map(|_| ()) });
2083 tokio::time::timeout(std::time::Duration::from_secs(15), async {
2084 tokio::select! {
2085 _ = probe.wait_until_attempted() => true,
2086 result = &mut rebalance_task => {
2087 observed_rebalance_result = Some(result.expect("rebalance activation task should not panic"));
2088 false
2089 }
2090 }
2091 })
2092 .await
2093 .unwrap_or(false)
2094 };
2095
2096 let competing_reached_commit = if reached_activation {
2099 tokio::time::timeout(std::time::Duration::from_secs(15), async {
2100 if paused_kind == PoolActivationStartKind::Rebalance {
2101 tokio::select! {
2102 _ = competing_barrier.wait_until_paused() => true,
2103 result = &mut decommission_task => {
2104 observed_decommission_result = Some(result.expect("decommission activation task should not panic"));
2105 false
2106 }
2107 }
2108 } else {
2109 tokio::select! {
2110 _ = competing_barrier.wait_until_paused() => true,
2111 result = &mut rebalance_task => {
2112 observed_rebalance_result = Some(result.expect("rebalance activation task should not panic"));
2113 false
2114 }
2115 }
2116 }
2117 })
2118 .await
2119 .expect("competing activation should either finish or reach its commit")
2120 } else {
2121 false
2122 };
2123 if competing_reached_commit {
2124 competing_barrier.release();
2125 }
2126 drop(competing_barrier);
2127 first_barrier.release();
2128 drop(first_barrier);
2129
2130 let rebalance_result = match observed_rebalance_result {
2131 Some(result) => result,
2132 None => tokio::time::timeout(std::time::Duration::from_secs(15), &mut rebalance_task)
2133 .await
2134 .expect("real rebalance activation should finish")
2135 .expect("rebalance activation task should not panic"),
2136 };
2137 let decommission_result = match observed_decommission_result {
2138 Some(result) => result,
2139 None => tokio::time::timeout(std::time::Duration::from_secs(15), &mut decommission_task)
2140 .await
2141 .expect("real decommission activation should finish")
2142 .expect("decommission activation task should not panic"),
2143 };
2144 assert!(
2145 !competing_reached_commit,
2146 "the competing real start reached its commit while the other activation lock was held"
2147 );
2148 assert_ne!(
2149 rebalance_result.is_ok(),
2150 decommission_result.is_ok(),
2151 "exactly one real activation entry may commit"
2152 );
2153
2154 let mut persisted_rebalance = RebalanceMeta::new();
2155 let rebalance_committed = persisted_rebalance
2156 .load(rebalance_store.pools[0].clone())
2157 .await
2158 .is_ok_and(|_| is_rebalance_conflicting_with_decommission(&persisted_rebalance));
2159 let mut persisted_pool = PoolMeta::default();
2160 persisted_pool
2161 .load_no_lock_from_replicas(rebalance_store.pools.clone())
2162 .await
2163 .expect("persisted pool metadata should remain readable");
2164 let decommission_committed = pool_meta_has_active_decommission(&persisted_pool);
2165 assert_eq!(
2166 usize::from(rebalance_committed) + usize::from(decommission_committed),
2167 1,
2168 "at most one persisted activation state may be active"
2169 );
2170 assert_eq!(rebalance_committed, paused_kind == PoolActivationStartKind::Rebalance);
2171 assert_eq!(decommission_committed, paused_kind == PoolActivationStartKind::Decommission);
2172 }
2173
2174 #[tokio::test]
2175 #[serial_test::serial]
2176 async fn real_rebalance_and_decommission_starts_commit_at_most_one_side() {
2177 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
2178 assert_real_activation_start_race(PoolActivationStartKind::Rebalance).await;
2179 assert_real_activation_start_race(PoolActivationStartKind::Decommission).await;
2180 })
2181 .await;
2182 }
2183
2184 #[test]
2185 fn rebalance_activation_rejects_persisted_decommission_despite_idle_local_snapshot() {
2186 let persisted = PoolMeta {
2187 pools: vec![crate::core::pools::PoolStatus {
2188 id: 0,
2189 cmd_line: "pool-0".to_string(),
2190 last_update: OffsetDateTime::UNIX_EPOCH,
2191 decommission: Some(crate::core::pools::PoolDecommissionInfo {
2192 start_time: Some(OffsetDateTime::UNIX_EPOCH),
2193 ..Default::default()
2194 }),
2195 }],
2196 ..Default::default()
2197 };
2198
2199 let err = ensure_rebalance_activation_pool_meta_allowed(&persisted)
2200 .expect_err("persisted decommission must block a stale rebalance admission");
2201 assert!(matches!(err, Error::DecommissionAlreadyRunning));
2202 }
2203
2204 #[test]
2205 fn rebalance_activation_allows_persisted_idle_pool_meta() {
2206 assert!(ensure_rebalance_activation_pool_meta_allowed(&PoolMeta::default()).is_ok());
2207 }
2208
2209 #[tokio::test]
2210 async fn rebalance_merge_save_does_not_commit_after_namespace_fence_loss() {
2211 let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks_isolated(4).await;
2212
2213 let persisted = RebalanceMeta {
2214 id: "rebalance-a".to_string(),
2215 pool_stats: vec![RebalanceStats {
2216 participating: true,
2217 bytes: 1,
2218 info: RebalanceInfo {
2219 status: RebalStatus::Started,
2220 ..Default::default()
2221 },
2222 ..Default::default()
2223 }],
2224 ..Default::default()
2225 };
2226 persisted
2227 .save(set_disks.clone())
2228 .await
2229 .expect("baseline rebalance metadata should be saved");
2230
2231 let mut update = persisted.clone();
2232 update.pool_stats[0].bytes = 999;
2233 let err = merge_and_save_rebalance_meta_no_lock(
2234 set_disks.clone(),
2235 &update,
2236 "lost namespace fence test",
2237 ObjectOptions {
2238 no_lock: true,
2239 namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()),
2240 ..Default::default()
2241 },
2242 None,
2243 Some(persisted.id.as_str()),
2244 )
2245 .await
2246 .expect_err("lost namespace fence must reject the metadata commit");
2247 assert!(matches!(err, Error::NamespaceLockQuorumUnavailable { .. }));
2248
2249 let mut after = RebalanceMeta::new();
2250 after
2251 .load(set_disks)
2252 .await
2253 .expect("baseline rebalance metadata should remain readable");
2254 assert_eq!(after.id, persisted.id);
2255 assert_eq!(after.pool_stats[0].bytes, 1);
2256 }
2257
2258 #[tokio::test]
2259 async fn persisted_rebalance_run_fence_rejects_stale_active_node_after_remote_stop() {
2260 let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks_isolated(4).await;
2261 let active = RebalanceMeta {
2262 id: "rebalance-a".to_string(),
2263 cancel: Some(tokio_util::sync::CancellationToken::new()),
2264 pool_stats: vec![RebalanceStats {
2265 participating: true,
2266 info: RebalanceInfo {
2267 status: RebalStatus::Started,
2268 ..Default::default()
2269 },
2270 ..Default::default()
2271 }],
2272 ..Default::default()
2273 };
2274 active
2275 .save(set_disks.clone())
2276 .await
2277 .expect("active rebalance metadata should be saved");
2278 ensure_rebalance_worker_active(Some(&active), active.id.as_str(), "stale node precondition")
2279 .expect("the stale node snapshot should still look active locally");
2280
2281 let mut stopped = active.clone();
2282 stopped.stopped_at = Some(OffsetDateTime::now_utc());
2283 stopped.pool_stats[0].info.stopping = true;
2284 stopped
2285 .save(set_disks.clone())
2286 .await
2287 .expect("remote stop should replace persisted metadata");
2288
2289 let err = acquire_persisted_rebalance_run_guard(set_disks, active.id.as_str(), "cross-node stale snapshot")
2290 .await
2291 .expect_err("persisted stop must fence a node that missed stop propagation");
2292 assert!(
2293 matches!(err, Error::OperationCanceled),
2294 "a durable remote stop cancels the same run: {err}"
2295 );
2296 }
2297
2298 #[test]
2299 fn pool_rebalance_status_ignores_non_participating_pool_state() {
2300 let meta = RebalanceMeta {
2301 pool_stats: vec![
2302 RebalanceStats {
2303 participating: false,
2304 info: RebalanceInfo {
2305 status: RebalStatus::Started,
2306 stopping: true,
2307 ..Default::default()
2308 },
2309 ..Default::default()
2310 },
2311 RebalanceStats {
2312 participating: true,
2313 info: RebalanceInfo {
2314 status: RebalStatus::Started,
2315 ..Default::default()
2316 },
2317 ..Default::default()
2318 },
2319 ],
2320 ..Default::default()
2321 };
2322
2323 assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 0), (RebalStatus::None, false));
2324 assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 1), (RebalStatus::Started, false));
2325 assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 2), (RebalStatus::None, false));
2326 }
2327
2328 #[test]
2329 fn rebalance_status_refresh_applies_persisted_terminal_state() {
2330 let rebalance_id = "rebalance-id".to_string();
2331 let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
2332 let mut current = Some(RebalanceMeta {
2333 id: rebalance_id.clone(),
2334 pool_stats: vec![RebalanceStats {
2335 participating: true,
2336 info: RebalanceInfo {
2337 status: RebalStatus::Started,
2338 start_time: Some(now),
2339 ..Default::default()
2340 },
2341 ..Default::default()
2342 }],
2343 ..Default::default()
2344 });
2345 let persisted = RebalanceMeta {
2346 id: rebalance_id,
2347 pool_stats: vec![RebalanceStats {
2348 participating: true,
2349 info: RebalanceInfo {
2350 status: RebalStatus::Completed,
2351 start_time: Some(now),
2352 end_time: Some(now),
2353 ..Default::default()
2354 },
2355 ..Default::default()
2356 }],
2357 ..Default::default()
2358 };
2359
2360 assert!(merge_rebalance_status_refresh(&mut current, persisted));
2361
2362 let refreshed = current.as_ref().expect("refresh should keep rebalance metadata");
2363 assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
2364 assert_eq!(refreshed.pool_stats[0].info.end_time, Some(now));
2365 }
2366
2367 #[test]
2368 fn rebalance_status_refresh_preserves_runtime_cancel_token() {
2369 let rebalance_id = "rebalance-id".to_string();
2370 let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
2371 let mut current = Some(RebalanceMeta {
2372 id: rebalance_id.clone(),
2373 cancel: Some(tokio_util::sync::CancellationToken::new()),
2374 pool_stats: vec![RebalanceStats {
2375 participating: true,
2376 info: RebalanceInfo {
2377 status: RebalStatus::Started,
2378 start_time: Some(now),
2379 ..Default::default()
2380 },
2381 ..Default::default()
2382 }],
2383 ..Default::default()
2384 });
2385 let persisted = RebalanceMeta {
2386 id: rebalance_id,
2387 pool_stats: vec![RebalanceStats {
2388 participating: true,
2389 info: RebalanceInfo {
2390 status: RebalStatus::Started,
2391 start_time: Some(now),
2392 ..Default::default()
2393 },
2394 ..Default::default()
2395 }],
2396 ..Default::default()
2397 };
2398
2399 assert!(!merge_rebalance_status_refresh(&mut current, persisted));
2400
2401 assert!(
2402 current.as_ref().and_then(|meta| meta.cancel.as_ref()).is_some(),
2403 "status refresh must not drop the runtime cancellation token"
2404 );
2405 }
2406
2407 #[test]
2408 fn rebalance_status_refresh_preserves_local_active_different_id_conflict() {
2409 let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
2410 let mut current = Some(RebalanceMeta {
2411 id: "old-active-id".to_string(),
2412 cancel: Some(tokio_util::sync::CancellationToken::new()),
2413 pool_stats: vec![RebalanceStats {
2414 participating: true,
2415 info: RebalanceInfo {
2416 status: RebalStatus::Started,
2417 start_time: Some(now),
2418 ..Default::default()
2419 },
2420 ..Default::default()
2421 }],
2422 ..Default::default()
2423 });
2424 let persisted = RebalanceMeta {
2425 id: "new-terminal-id".to_string(),
2426 pool_stats: vec![RebalanceStats {
2427 participating: true,
2428 info: RebalanceInfo {
2429 status: RebalStatus::Completed,
2430 start_time: Some(now),
2431 end_time: Some(now),
2432 ..Default::default()
2433 },
2434 ..Default::default()
2435 }],
2436 ..Default::default()
2437 };
2438
2439 merge_rebalance_status_refresh(&mut current, persisted);
2440
2441 let refreshed = current.as_ref().expect("local active metadata should remain visible");
2442 assert_eq!(refreshed.id, "old-active-id");
2443 assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Started);
2444 assert!(
2445 refreshed.cancel.is_some(),
2446 "status refresh must not drop a live runtime cancellation token"
2447 );
2448 }
2449
2450 #[test]
2451 fn rebalance_status_refresh_replaces_stale_memory_without_runtime_token() {
2452 let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
2453 let mut current = Some(RebalanceMeta {
2454 id: "old-stale-id".to_string(),
2455 pool_stats: vec![RebalanceStats {
2456 participating: true,
2457 info: RebalanceInfo {
2458 status: RebalStatus::Started,
2459 start_time: Some(now),
2460 ..Default::default()
2461 },
2462 ..Default::default()
2463 }],
2464 ..Default::default()
2465 });
2466 let persisted = RebalanceMeta {
2467 id: "new-terminal-id".to_string(),
2468 pool_stats: vec![RebalanceStats {
2469 participating: true,
2470 info: RebalanceInfo {
2471 status: RebalStatus::Completed,
2472 start_time: Some(now),
2473 end_time: Some(now),
2474 ..Default::default()
2475 },
2476 ..Default::default()
2477 }],
2478 ..Default::default()
2479 };
2480
2481 merge_rebalance_status_refresh(&mut current, persisted);
2482
2483 let refreshed = current.as_ref().expect("persisted metadata should replace stale memory");
2484 assert_eq!(refreshed.id, "new-terminal-id");
2485 assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
2486 assert!(refreshed.cancel.is_none());
2487 }
2488}