Skip to main content

rustfs_ecstore/services/rebalance/
control.rs

1use super::meta::{
2    RebalanceMetaMergeOutcome, clone_first_arc, clone_rebalance_pool_stats, defer_bucket_in_rebalance_queue,
3    ensure_valid_rebalance_pool_index, invalid_rebalance_pool_index_error, is_rebalance_actively_running,
4    is_rebalance_conflicting_with_decommission, mark_rebalance_bucket_done, merge_rebalance_meta, percent_free_ratio,
5    rebalance_metadata_not_initialized_error, record_rebalance_cleanup_warning_in_meta,
6    record_rebalance_stop_propagation_snapshot, resolve_next_rebalance_bucket, rollback_rebalance_start_meta_snapshot_for_id,
7    should_accept_rebalance_stats_update, should_pool_participate, stop_rebalance_meta_snapshot_for_id,
8    validate_init_rebalance_state,
9};
10use super::worker::{
11    rebalance_max_attempts, rebalance_meta_lock_error, resolve_load_rebalance_stats_update_result,
12    resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result, retry_rebalance_metadata_access,
13};
14use super::{
15    DiskStat, EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
16    RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord,
17    encode_rebalance_stop_propagation_record,
18};
19use crate::core::pools::{
20    PoolMeta, PoolRebalanceActivationFence, acquire_pool_activation_fleet_proof, acquire_pool_rebalance_activation_locks,
21    pool_meta_has_active_decommission,
22};
23use crate::error::{Error, Result};
24use crate::object_api::ObjectOptions;
25use crate::set_disk::get_lock_acquire_timeout;
26use crate::storage_api_contracts::{
27    admin::StorageAdminApi,
28    namespace::NamespaceLocking as StorageNamespaceLocking,
29    object::{EcstoreObjectIO, EcstoreObjectOperations},
30};
31use crate::store::ECStore;
32use rustfs_filemeta::FileInfo;
33use std::sync::Arc;
34use time::OffsetDateTime;
35use tracing::{debug, info};
36use uuid::Uuid;
37
38#[cfg(test)]
39static FAIL_NEXT_REBALANCE_ACTIVATION_SAVE: std::sync::Mutex<Option<String>> = std::sync::Mutex::new(None);
40
41#[cfg(test)]
42static REBALANCE_DISK_STATS_OVERRIDES: std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<Uuid, Vec<DiskStat>>>> =
43    std::sync::OnceLock::new();
44
45#[cfg(test)]
46pub(super) fn fail_next_rebalance_activation_save_for_test(rebalance_id: &str) {
47    *FAIL_NEXT_REBALANCE_ACTIVATION_SAVE
48        .lock()
49        .expect("rebalance activation save failure hook should not be poisoned") = Some(rebalance_id.to_string());
50}
51
52#[cfg(test)]
53fn set_rebalance_disk_stats_override_for_test(store_id: Uuid, disk_stats: Vec<DiskStat>) {
54    REBALANCE_DISK_STATS_OVERRIDES
55        .get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
56        .lock()
57        .expect("rebalance disk stats override should not be poisoned")
58        .insert(store_id, disk_stats);
59}
60
61#[cfg(test)]
62fn take_rebalance_disk_stats_override_for_test(store_id: Uuid) -> Option<Vec<DiskStat>> {
63    REBALANCE_DISK_STATS_OVERRIDES
64        .get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
65        .lock()
66        .expect("rebalance disk stats override should not be poisoned")
67        .remove(&store_id)
68}
69
70fn ensure_rebalance_activation_pool_meta_allowed(meta: &PoolMeta) -> Result<()> {
71    if pool_meta_has_active_decommission(meta) {
72        return Err(Error::DecommissionAlreadyRunning);
73    }
74
75    Ok(())
76}
77
78pub(super) enum RebalanceWorkerActivationFence {
79    Ready(Box<PoolRebalanceActivationFence>),
80    NotStartedTerminal,
81}
82
83pub(super) struct RebalanceRunGuard {
84    _local_guard: tokio::sync::OwnedRwLockReadGuard<()>,
85    persisted_guard: rustfs_lock::NamespaceLockGuard,
86}
87
88#[cfg(any(test, feature = "test-util"))]
89struct RebalanceStopWaitProbeState {
90    expected_id: String,
91    attempted: std::sync::atomic::AtomicBool,
92    notify: tokio::sync::Notify,
93}
94
95#[cfg(any(test, feature = "test-util"))]
96static REBALANCE_STOP_WAIT_PROBES: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<RebalanceStopWaitProbeState>>>> =
97    std::sync::OnceLock::new();
98
99#[cfg(any(test, feature = "test-util"))]
100pub(super) struct RebalanceStopWaitProbe {
101    state: Arc<RebalanceStopWaitProbeState>,
102}
103
104#[cfg(any(test, feature = "test-util"))]
105impl RebalanceStopWaitProbe {
106    pub(super) fn install(expected_id: &str) -> Self {
107        let state = Arc::new(RebalanceStopWaitProbeState {
108            expected_id: expected_id.to_string(),
109            attempted: std::sync::atomic::AtomicBool::new(false),
110            notify: tokio::sync::Notify::new(),
111        });
112        REBALANCE_STOP_WAIT_PROBES
113            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
114            .lock()
115            .expect("rebalance stop wait probe should not be poisoned")
116            .push(Arc::clone(&state));
117        Self { state }
118    }
119
120    pub(super) async fn wait_until_attempted(&self) {
121        tokio::time::timeout(std::time::Duration::from_secs(30), async {
122            while !self.state.attempted.load(std::sync::atomic::Ordering::Acquire) {
123                self.state.notify.notified().await;
124            }
125        })
126        .await
127        .expect("rebalance stop should reach the deterministic activation wait probe");
128    }
129}
130
131#[cfg(any(test, feature = "test-util"))]
132impl Drop for RebalanceStopWaitProbe {
133    fn drop(&mut self) {
134        let mut probes = REBALANCE_STOP_WAIT_PROBES
135            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
136            .lock()
137            .expect("rebalance stop wait probe should not be poisoned");
138        probes.retain(|state| !Arc::ptr_eq(state, &self.state));
139    }
140}
141
142#[cfg(any(test, feature = "test-util"))]
143fn observe_rebalance_stop_wait_attempt(expected_id: Option<&str>) {
144    let probes = REBALANCE_STOP_WAIT_PROBES
145        .get_or_init(|| std::sync::Mutex::new(Vec::new()))
146        .lock()
147        .expect("rebalance stop wait probe should not be poisoned")
148        .clone();
149    for state in probes {
150        if expected_id == Some(state.expected_id.as_str()) {
151            state.attempted.store(true, std::sync::atomic::Ordering::Release);
152            state.notify.notify_one();
153        }
154    }
155}
156
157impl RebalanceRunGuard {
158    pub(super) fn ensure_held(&self, stage: &str) -> Result<()> {
159        #[cfg(test)]
160        let forced_lost = self
161            .persisted_guard
162            .lock_lost_signal()
163            .is_some_and(|signal| crate::object_api::namespace_lock_signal_test_fence_is_lost(&signal));
164        #[cfg(not(test))]
165        let forced_lost = false;
166        if self.persisted_guard.is_lock_lost() || forced_lost {
167            return Err(Error::other(format!("rebalance distributed run fence lost during {stage}")));
168        }
169        Ok(())
170    }
171
172    pub(super) fn lock_lost_signal(&self) -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
173        self.persisted_guard.lock_lost_signal()
174    }
175}
176
177async fn acquire_persisted_rebalance_run_guard<S>(
178    pool: Arc<S>,
179    expected_id: &str,
180    stage: &str,
181) -> Result<rustfs_lock::NamespaceLockGuard>
182where
183    S: EcstoreObjectIO
184        + EcstoreObjectOperations
185        + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
186{
187    let ns_lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
188    let guard = ns_lock
189        .get_read_lock(get_lock_acquire_timeout())
190        .await
191        .map_err(|err| rebalance_meta_lock_error(err, "read"))?;
192    let mut opts = ObjectOptions {
193        no_lock: true,
194        ..Default::default()
195    };
196    opts.add_namespace_lock_guard(&guard);
197    let object_info = pool
198        .get_object_info(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME, &opts)
199        .await;
200    let persisted_run_id = match object_info {
201        Ok(object_info) => {
202            let metadata = object_info.user_defined.as_ref();
203            let run_id = rustfs_utils::http::metadata_compat::get_consistent_str(
204                metadata,
205                rustfs_utils::http::metadata_compat::SUFFIX_REBALANCE_RUN_ID,
206            );
207            if run_id.is_none()
208                && rustfs_utils::http::metadata_compat::contains_key_str(
209                    metadata,
210                    rustfs_utils::http::metadata_compat::SUFFIX_REBALANCE_RUN_ID,
211                )
212            {
213                return Err(Error::other(format!("rebalance run marker is inconsistent during {stage}")));
214            }
215            run_id.map(str::to_string)
216        }
217        Err(err) if crate::error::is_err_object_not_found(&err) => None,
218        Err(err) => return Err(err),
219    };
220    if let Some(persisted_run_id) = persisted_run_id {
221        if persisted_run_id != expected_id {
222            return Err(Error::other(format!(
223                "stale rebalance run rejected during {stage}: expected {expected_id}, found {persisted_run_id}"
224            )));
225        }
226    } else {
227        // Metadata written before the fixed-size run marker was introduced
228        // remains readable during rolling upgrades.
229        let mut persisted = RebalanceMeta::new();
230        persisted.load_with_opts(pool, opts).await?;
231        ensure_rebalance_worker_active(Some(&persisted), expected_id, stage)?;
232    }
233    if guard.is_lock_lost() {
234        return Err(Error::other(format!("rebalance distributed run fence lost during {stage}")));
235    }
236    Ok(guard)
237}
238
239pub(super) async fn merge_and_save_rebalance_meta_no_lock<S>(
240    pool: Arc<S>,
241    local_snapshot: &RebalanceMeta,
242    stage: &str,
243    mut opts: ObjectOptions,
244    activation_fence: Option<&PoolRebalanceActivationFence>,
245    expected_id: Option<&str>,
246) -> Result<()>
247where
248    S: EcstoreObjectIO,
249{
250    let mut merged = RebalanceMeta::new();
251    match merged.load_with_opts(pool.clone(), opts.clone()).await {
252        Ok(()) => {
253            if let Some(expected_id) = expected_id {
254                ensure_rebalance_run_id(Some(&merged), expected_id, stage)?;
255            }
256            if merge_rebalance_meta(&mut merged, local_snapshot) == RebalanceMetaMergeOutcome::RejectedActiveConflict {
257                return Err(Error::RebalanceAlreadyRunning);
258            }
259        }
260        Err(Error::ConfigNotFound) => {
261            if expected_id.is_some() {
262                return Err(rebalance_metadata_not_initialized_error(stage));
263            }
264            merged = local_snapshot.clone();
265        }
266        Err(err) => return Err(Error::other(format!("rebalance meta load before save failed during {stage}: {err}"))),
267    }
268
269    if let Some(fence) = activation_fence {
270        fence.add_namespace_lock_fence(&mut opts);
271        fence.ensure_held()?;
272    }
273    #[cfg(test)]
274    let barrier_pool = pool.clone();
275    merged.save_with_opts(pool, opts).await?;
276    #[cfg(test)]
277    if let Some(fence) = activation_fence {
278        crate::core::pools::pause_pool_activation_after_durable_save(&barrier_pool, fence).await;
279    }
280    // With an activation fence, a successful save is the commit point. Lease
281    // loss after this point cannot make the durable activation uncommitted.
282    Ok(())
283}
284
285pub(super) fn ensure_rebalance_run_id(meta: Option<&RebalanceMeta>, expected_id: &str, stage: &str) -> Result<()> {
286    let Some(meta) = meta else {
287        return Err(rebalance_metadata_not_initialized_error(stage));
288    };
289    if meta.id != expected_id {
290        return Err(Error::other(format!(
291            "stale rebalance worker rejected during {stage}: expected {expected_id}, found {}",
292            meta.id
293        )));
294    }
295    Ok(())
296}
297
298pub(super) fn ensure_rebalance_worker_active(meta: Option<&RebalanceMeta>, expected_id: &str, stage: &str) -> Result<()> {
299    ensure_rebalance_run_id(meta, expected_id, stage)?;
300    let Some(meta) = meta else {
301        return Err(rebalance_metadata_not_initialized_error(stage));
302    };
303    if meta.stopped_at.is_some() || meta.stop_requested {
304        return Err(Error::OperationCanceled);
305    }
306    if meta
307        .cancel
308        .as_ref()
309        .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
310        || !is_rebalance_conflicting_with_decommission(meta)
311    {
312        return Err(Error::other(format!("inactive rebalance worker rejected during {stage}: {expected_id}")));
313    }
314    Ok(())
315}
316
317pub(super) fn validate_rebalance_disk_stats_coverage(disk_stats: &[DiskStat]) -> Result<()> {
318    for (idx, disk_stat) in disk_stats.iter().enumerate() {
319        if disk_stat.total_space == 0 {
320            return Err(Error::other(format!(
321                "rebalance storage info is incomplete: pool {idx} has no reported capacity"
322            )));
323        }
324    }
325
326    Ok(())
327}
328
329fn pool_rebalance_status_from_meta(meta: Option<&RebalanceMeta>, pool_index: usize) -> (RebalStatus, bool) {
330    meta.and_then(|meta| meta.pool_stats.get(pool_index))
331        .filter(|pool_stat| pool_stat.participating)
332        .map(|pool_stat| (pool_stat.info.status, pool_stat.info.stopping))
333        .unwrap_or_default()
334}
335
336fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted: RebalanceMeta) -> bool {
337    if persisted.id.is_empty() && persisted.pool_stats.is_empty() {
338        return clear_rebalance_status_refresh(current);
339    }
340
341    let before = current.clone();
342    match current.as_mut() {
343        Some(current_meta) => {
344            if merge_rebalance_meta(current_meta, &persisted) == RebalanceMetaMergeOutcome::RejectedActiveConflict
345                && !is_rebalance_actively_running(current_meta)
346            {
347                *current = Some(persisted);
348            }
349        }
350        None => {
351            *current = Some(persisted);
352        }
353    }
354
355    match (before.as_ref(), current.as_ref()) {
356        (None, None) => false,
357        (None, Some(_)) | (Some(_), None) => true,
358        (Some(before), Some(after)) => rebalance_movement_snapshot_changed(Some(before), after),
359    }
360}
361
362fn clear_rebalance_status_refresh(current: &mut Option<RebalanceMeta>) -> bool {
363    if current.as_ref().is_none_or(|meta| !is_rebalance_actively_running(meta)) {
364        current.take().is_some()
365    } else {
366        false
367    }
368}
369
370fn rebalance_movement_snapshot_changed(current: Option<&RebalanceMeta>, persisted: &RebalanceMeta) -> bool {
371    let Some(current) = current else {
372        return true;
373    };
374
375    current.id != persisted.id
376        || current.stopped_at != persisted.stopped_at
377        || current.pool_stats.len() != persisted.pool_stats.len()
378        || current
379            .pool_stats
380            .iter()
381            .zip(persisted.pool_stats.iter())
382            .any(|(current, persisted)| {
383                current.participating != persisted.participating
384                    || current.info.status != persisted.info.status
385                    || current.info.stopping != persisted.info.stopping
386            })
387}
388
389impl ECStore {
390    // Transition order is start_gate -> activation_gate -> movement gate -> rebalance_meta;
391    // the probe read below is released before the activation gate.
392    pub(super) async fn rebalance_activation_write_guard(
393        &self,
394        expected_id: Option<&str>,
395        stage: &str,
396    ) -> Result<Option<tokio::sync::OwnedRwLockWriteGuard<()>>> {
397        let activation_gate = {
398            let meta = self.rebalance_meta.read().await;
399            if let Some(expected_id) = expected_id {
400                ensure_rebalance_run_id(meta.as_ref(), expected_id, stage)?;
401            }
402            meta.as_ref().map(|meta| Arc::clone(&meta.activation_gate))
403        };
404        Ok(match activation_gate {
405            Some(gate) => Some(gate.write_owned().await),
406            None => None,
407        })
408    }
409
410    pub(super) async fn rebalance_run_guard(&self, expected_id: &str, stage: &str) -> Result<RebalanceRunGuard> {
411        // Runtime fence order is activation_gate -> rebalance.bin.
412        let activation_gate = {
413            let meta = self.rebalance_meta.read().await;
414            ensure_rebalance_worker_active(meta.as_ref(), expected_id, stage)?;
415            Arc::clone(
416                &meta
417                    .as_ref()
418                    .ok_or_else(|| rebalance_metadata_not_initialized_error(stage))?
419                    .activation_gate,
420            )
421        };
422        let guard = Arc::clone(&activation_gate).read_owned().await;
423        let meta = self.rebalance_meta.read().await;
424        ensure_rebalance_worker_active(meta.as_ref(), expected_id, stage)?;
425        let current_gate = &meta
426            .as_ref()
427            .ok_or_else(|| rebalance_metadata_not_initialized_error(stage))?
428            .activation_gate;
429        if !Arc::ptr_eq(&activation_gate, current_gate) {
430            return Err(Error::other(format!(
431                "stale rebalance activation gate rejected during {stage}: {expected_id}"
432            )));
433        }
434        drop(meta);
435        let pool = clone_first_arc(self.pools.as_slice(), "rebalance run fence: no pools available")?;
436        let persisted_guard = acquire_persisted_rebalance_run_guard(pool, expected_id, stage).await?;
437        Ok(RebalanceRunGuard {
438            _local_guard: guard,
439            persisted_guard,
440        })
441    }
442
443    pub(super) async fn save_rebalance_meta_with_merge<S>(
444        &self,
445        pool: Arc<S>,
446        local_snapshot: &RebalanceMeta,
447        stage: &str,
448    ) -> Result<()>
449    where
450        S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
451    {
452        self.save_rebalance_meta_with_merge_for_id(pool, local_snapshot, stage, None)
453            .await
454    }
455
456    pub(super) async fn save_rebalance_meta_for_id_with_merge<S>(
457        &self,
458        pool: Arc<S>,
459        local_snapshot: &RebalanceMeta,
460        stage: &str,
461        expected_id: &str,
462    ) -> Result<()>
463    where
464        S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
465    {
466        self.save_rebalance_meta_with_merge_for_id(pool, local_snapshot, stage, Some(expected_id))
467            .await
468    }
469
470    pub(super) async fn save_rebalance_meta_under_activation_fence<S>(
471        &self,
472        pool: Arc<S>,
473        local_snapshot: &RebalanceMeta,
474        stage: &str,
475        activation_fence: &PoolRebalanceActivationFence,
476        expected_id: &str,
477    ) -> Result<()>
478    where
479        S: EcstoreObjectIO,
480    {
481        #[cfg(test)]
482        {
483            let mut fail_id = FAIL_NEXT_REBALANCE_ACTIVATION_SAVE
484                .lock()
485                .expect("rebalance activation save failure hook should not be poisoned");
486            if fail_id.as_deref() == Some(local_snapshot.id.as_str()) {
487                fail_id.take();
488                return Err(Error::other("injected rebalance activation save failure"));
489            }
490        }
491        merge_and_save_rebalance_meta_no_lock(
492            pool,
493            local_snapshot,
494            stage,
495            ObjectOptions {
496                no_lock: true,
497                ..Default::default()
498            },
499            Some(activation_fence),
500            Some(expected_id),
501        )
502        .await
503    }
504
505    async fn save_rebalance_meta_with_merge_for_id<S>(
506        &self,
507        pool: Arc<S>,
508        local_snapshot: &RebalanceMeta,
509        stage: &str,
510        expected_id: Option<&str>,
511    ) -> Result<()>
512    where
513        S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
514    {
515        let ns_lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
516        let guard = retry_rebalance_metadata_access(None, rebalance_max_attempts(), || async {
517            ns_lock
518                .get_write_lock(get_lock_acquire_timeout())
519                .await
520                .map_err(|err| rebalance_meta_lock_error(err, "write"))
521        })
522        .await?;
523        let mut opts = ObjectOptions {
524            no_lock: true,
525            ..Default::default()
526        };
527        opts.add_namespace_lock_guard(&guard);
528
529        merge_and_save_rebalance_meta_no_lock(pool, local_snapshot, stage, opts, None, expected_id).await?;
530        if guard.is_lock_lost() {
531            return Err(Error::other("rebalance metadata lock lost during metadata commit"));
532        }
533        Ok(())
534    }
535
536    async fn save_rebalance_activation_meta_with_merge<S>(
537        &self,
538        pool: Arc<S>,
539        local_snapshot: &RebalanceMeta,
540        stage: &str,
541    ) -> Result<()>
542    where
543        S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
544    {
545        #[cfg(test)]
546        crate::core::pools::observe_pool_activation_start_attempt(crate::core::pools::PoolActivationStartKind::Rebalance);
547        let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
548        let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
549        pool_meta_guard.ensure_write_safe(stage)?;
550        let activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), fleet_proof).await?;
551        let pool_meta = self
552            .load_runtime_pool_meta_under_activation_fence(&mut pool_meta_guard, &activation_fence, stage)
553            .await?;
554        ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?;
555
556        merge_and_save_rebalance_meta_no_lock(
557            pool,
558            local_snapshot,
559            stage,
560            ObjectOptions {
561                no_lock: true,
562                ..Default::default()
563            },
564            Some(&activation_fence),
565            None,
566        )
567        .await
568    }
569
570    pub(super) async fn fence_rebalance_worker_activation<S>(
571        &self,
572        pool: Arc<S>,
573        expected_id: &str,
574    ) -> Result<RebalanceWorkerActivationFence>
575    where
576        S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
577    {
578        // Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin.
579        let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
580        pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
581        // Classify the durable rebalance record while holding both namespace
582        // fences. A terminal record is a no-op and must not depend on the
583        // notification subsystem having published a fleet proof yet.
584        let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
585        let pool_meta = self
586            .load_runtime_pool_meta_under_activation_fence(&mut pool_meta_guard, &activation_fence, "rebalance worker activation")
587            .await?;
588        ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?;
589
590        let mut persisted = RebalanceMeta::new();
591        persisted
592            .load_with_opts(
593                pool,
594                ObjectOptions {
595                    no_lock: true,
596                    ..Default::default()
597                },
598            )
599            .await?;
600        if persisted.id != expected_id {
601            return Err(Error::other(format!(
602                "rebalance metadata changed before worker activation: expected {expected_id}, found {}",
603                persisted.id
604            )));
605        }
606
607        activation_fence.ensure_held()?;
608        if !crate::services::rebalance::rebalance_requires_worker_activation(&persisted) {
609            return Ok(RebalanceWorkerActivationFence::NotStartedTerminal);
610        }
611
612        // Active worker admission still requires the fail-closed fleet proof.
613        // Attach it immediately before the final fence validation so expiry or
614        // topology changes are checked again at every later commit boundary.
615        let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
616        activation_fence.set_fleet_proof(fleet_proof);
617        activation_fence.ensure_held()?;
618
619        Ok(RebalanceWorkerActivationFence::Ready(Box::new(activation_fence)))
620    }
621
622    #[tracing::instrument(skip_all)]
623    pub async fn load_rebalance_meta(&self) -> Result<()> {
624        let _start_guard = self.start_gate.lock().await;
625        self.load_rebalance_meta_under_start_gate().await
626    }
627
628    /// Cancels local admission before refreshing the persisted stop target under the start gate.
629    pub async fn prepare_rebalance_stop(&self) -> Result<Option<String>> {
630        let _start_guard = self.start_gate.lock().await;
631
632        {
633            let mut rebalance_meta = self.rebalance_meta.write().await;
634            if let Some(meta) = rebalance_meta.as_mut()
635                && is_rebalance_conflicting_with_decommission(meta)
636            {
637                meta.stop_requested = true;
638                meta.cancel
639                    .get_or_insert_with(tokio_util::sync::CancellationToken::new)
640                    .cancel();
641                #[cfg(any(test, feature = "test-util"))]
642                observe_rebalance_stop_wait_attempt(Some(meta.id.as_str()));
643            }
644        }
645
646        self.load_rebalance_meta_under_start_gate().await?;
647
648        let mut rebalance_meta = self.rebalance_meta.write().await;
649        let Some(meta) = rebalance_meta.as_mut() else {
650            return Ok(None);
651        };
652        if meta.stopped_at.is_some() || (!is_rebalance_conflicting_with_decommission(meta) && !meta.stop_requested) {
653            return Ok(None);
654        }
655        if meta.id.is_empty() {
656            return Err(Error::other("active rebalance metadata has no activation id"));
657        }
658        meta.stop_requested = true;
659        meta.cancel
660            .get_or_insert_with(tokio_util::sync::CancellationToken::new)
661            .cancel();
662        Ok(Some(meta.id.clone()))
663    }
664
665    pub(crate) async fn load_rebalance_meta_under_start_gate(&self) -> Result<()> {
666        let mut meta = RebalanceMeta::new();
667        debug!(
668            event = EVENT_REBALANCE_STATE,
669            component = LOG_COMPONENT_ECSTORE,
670            subsystem = LOG_SUBSYSTEM_REBALANCE,
671            state = "metadata_loading",
672            "Loading rebalance metadata"
673        );
674        let pool = clone_first_arc(&self.pools, "rebalanceMeta: no pools available")?;
675        let _activation_guard = self.rebalance_activation_write_guard(None, "load rebalance metadata").await?;
676        let movement_gate = self.ctx.data_movement_operation_gate();
677        let _movement_guard = movement_gate.write().await;
678        let loaded = resolve_rebalance_meta_load_result(meta.load(pool).await)?;
679        if loaded {
680            let movement_changed = rebalance_movement_snapshot_changed(self.rebalance_meta.read().await.as_ref(), &meta);
681            {
682                let mut rebalance_meta = self.rebalance_meta.write().await;
683                if let Some(current) = rebalance_meta.as_ref()
684                    && current.id == meta.id
685                {
686                    meta.cancel = current.cancel.clone();
687                    meta.activation_gate = Arc::clone(&current.activation_gate);
688                    meta.stop_requested = current.stop_requested;
689                }
690                *rebalance_meta = Some(meta);
691
692                drop(rebalance_meta);
693            }
694
695            if movement_changed {
696                self.ctx.advance_data_movement_operation_epoch();
697            }
698            drop(_movement_guard);
699            resolve_load_rebalance_stats_update_result(self.update_rebalance_stats().await)?;
700            debug!(
701                event = EVENT_REBALANCE_STATE,
702                component = LOG_COMPONENT_ECSTORE,
703                subsystem = LOG_SUBSYSTEM_REBALANCE,
704                state = "metadata_loaded",
705                "Loaded rebalance metadata"
706            );
707        } else {
708            let movement_changed = self.rebalance_meta.read().await.is_some();
709            {
710                let mut rebalance_meta = self.rebalance_meta.write().await;
711                *rebalance_meta = None;
712            }
713            if movement_changed {
714                self.ctx.advance_data_movement_operation_epoch();
715            }
716            drop(_movement_guard);
717            debug!(
718                event = EVENT_REBALANCE_STATE,
719                component = LOG_COMPONENT_ECSTORE,
720                subsystem = LOG_SUBSYSTEM_REBALANCE,
721                state = "metadata_missing",
722                reason = "rebalance_not_started",
723                "Rebalance metadata not found"
724            );
725        }
726
727        Ok(())
728    }
729
730    #[tracing::instrument(skip_all)]
731    pub async fn refresh_rebalance_status_meta(&self) -> Result<()> {
732        let _start_guard = self.start_gate.lock().await;
733        let pool = clone_first_arc(&self.pools, "refresh_rebalance_status_meta: no pools available")?;
734        let mut persisted = RebalanceMeta::new();
735        let _activation_guard = self
736            .rebalance_activation_write_guard(None, "refresh rebalance metadata")
737            .await?;
738        let movement_gate = self.ctx.data_movement_operation_gate();
739        let _movement_guard = movement_gate.write().await;
740        let loaded = persisted.load(pool).await;
741        match loaded {
742            Ok(()) => {
743                let mut rebalance_meta = self.rebalance_meta.write().await;
744                if merge_rebalance_status_refresh(&mut rebalance_meta, persisted) {
745                    self.ctx.advance_data_movement_operation_epoch();
746                }
747            }
748            Err(Error::ConfigNotFound) => {
749                let mut rebalance_meta = self.rebalance_meta.write().await;
750                if clear_rebalance_status_refresh(&mut rebalance_meta) {
751                    self.ctx.advance_data_movement_operation_epoch();
752                }
753            }
754            Err(err) => {
755                return Err(Error::other(format!("rebalance metadata refresh failed during pool status: {err}")));
756            }
757        }
758
759        Ok(())
760    }
761
762    #[tracing::instrument(skip_all)]
763    pub async fn update_rebalance_stats(&self) -> Result<()> {
764        let mut ok = false;
765
766        let pool_stats = {
767            let rebalance_meta = self.rebalance_meta.read().await;
768            clone_rebalance_pool_stats(rebalance_meta.as_ref())?
769        };
770
771        debug!(
772            event = EVENT_REBALANCE_STATE,
773            component = LOG_COMPONENT_ECSTORE,
774            subsystem = LOG_SUBSYSTEM_REBALANCE,
775            pool_count = pool_stats.len(),
776            "Refreshing rebalance stats snapshot"
777        );
778
779        for i in 0..self.pools.len() {
780            if pool_stats.get(i).is_none() {
781                let mut rebalance_meta = self.rebalance_meta.write().await;
782                debug!(
783                    event = EVENT_REBALANCE_STATE,
784                    component = LOG_COMPONENT_ECSTORE,
785                    subsystem = LOG_SUBSYSTEM_REBALANCE,
786                    pool_index = i,
787                    state = "pool_stat_missing",
788                    "Adding missing rebalance pool stats entry"
789                );
790                if let Some(meta) = rebalance_meta.as_mut() {
791                    meta.pool_stats.push(RebalanceStats::default());
792                }
793                ok = true;
794                drop(rebalance_meta);
795            }
796        }
797
798        if ok {
799            debug!(
800                event = EVENT_REBALANCE_STATE,
801                component = LOG_COMPONENT_ECSTORE,
802                subsystem = LOG_SUBSYSTEM_REBALANCE,
803                state = "metadata_saving",
804                "Saving rebalance metadata after stats refresh"
805            );
806
807            let rebalance_meta = self.rebalance_meta.read().await;
808            if let Some(meta) = rebalance_meta.as_ref() {
809                let pool = clone_first_arc(&self.pools, "update_rebalance_stats: no pools available")?;
810                resolve_rebalance_meta_save_result(
811                    self.save_rebalance_meta_for_id_with_merge(pool, meta, "update_rebalance_stats", meta.id.as_str())
812                        .await,
813                    "update_rebalance_stats",
814                )?;
815            }
816        }
817
818        Ok(())
819    }
820
821    #[tracing::instrument(skip(self))]
822    pub async fn init_rebalance_meta(&self, bucktes: Vec<String>) -> Result<String> {
823        info!(
824            event = EVENT_REBALANCE_STATE,
825            component = LOG_COMPONENT_ECSTORE,
826            subsystem = LOG_SUBSYSTEM_REBALANCE,
827            state = "initializing",
828            bucket_count = bucktes.len(),
829            "Initializing rebalance metadata"
830        );
831        let si = StorageAdminApi::storage_info(self).await;
832
833        let mut disk_stats = vec![DiskStat::default(); self.pools.len()];
834
835        let mut total_cap = 0;
836        let mut total_free = 0;
837        for disk in si.disks.iter() {
838            if disk.pool_index < 0 || disk_stats.len() <= disk.pool_index as usize {
839                continue;
840            }
841
842            total_cap += disk.total_space;
843            total_free += disk.available_space;
844
845            disk_stats[disk.pool_index as usize].total_space += disk.total_space;
846            disk_stats[disk.pool_index as usize].available_space += disk.available_space;
847        }
848
849        #[cfg(test)]
850        if let Some(overridden) = take_rebalance_disk_stats_override_for_test(self.id) {
851            disk_stats = overridden;
852            total_cap = disk_stats.iter().map(|stat| stat.total_space).sum();
853            total_free = disk_stats.iter().map(|stat| stat.available_space).sum();
854        }
855
856        let percent_free_goal = percent_free_ratio(total_free, total_cap);
857        validate_rebalance_disk_stats_coverage(&disk_stats)?;
858
859        let mut pool_stats = Vec::with_capacity(self.pools.len());
860
861        let now = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
862
863        for disk_stat in disk_stats.iter() {
864            let mut pool_stat = RebalanceStats {
865                init_free_space: disk_stat.available_space,
866                init_capacity: disk_stat.total_space,
867                buckets: bucktes.clone(),
868                rebalanced_buckets: Vec::with_capacity(bucktes.len()),
869                ..Default::default()
870            };
871
872            if should_pool_participate(disk_stat.available_space, disk_stat.total_space, percent_free_goal) {
873                pool_stat.participating = true;
874                pool_stat.info = RebalanceInfo {
875                    start_time: Some(now),
876                    status: RebalStatus::Started,
877                    ..Default::default()
878                };
879            }
880
881            pool_stats.push(pool_stat);
882        }
883
884        let has_participating_pool = pool_stats.iter().any(|pool_stat| pool_stat.participating);
885        let meta = RebalanceMeta {
886            id: Uuid::new_v4().to_string(),
887            stopped_at: (!has_participating_pool).then_some(now),
888            percent_free_goal,
889            pool_stats,
890            ..Default::default()
891        };
892
893        let pool = clone_first_arc(&self.pools, "init_rebalance_meta: no pools available")?;
894        resolve_rebalance_meta_save_result(
895            self.save_rebalance_activation_meta_with_merge(pool, &meta, "init_rebalance_meta")
896                .await,
897            "init_rebalance_meta",
898        )?;
899
900        info!(
901            event = EVENT_REBALANCE_STATE,
902            component = LOG_COMPONENT_ECSTORE,
903            subsystem = LOG_SUBSYSTEM_REBALANCE,
904            state = "metadata_initialized",
905            bucket_count = bucktes.len(),
906            "Rebalance metadata initialized"
907        );
908
909        let id = meta.id.clone();
910
911        {
912            let mut rebalance_meta = self.rebalance_meta.write().await;
913            *rebalance_meta = Some(meta);
914            drop(rebalance_meta);
915        }
916
917        Ok(id)
918    }
919
920    #[tracing::instrument(skip(self, bucktes))]
921    pub async fn init_and_start_rebalance(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
922        let id = self.init_rebalance_start(bucktes).await?;
923        if let Err(start_err) = self.start_rebalance_for_id(&id).await {
924            if let Err(rollback_err) = self
925                .rollback_rebalance_start_without_worker_for_id(Some(&id), start_err.to_string())
926                .await
927            {
928                return Err(Error::other(format!(
929                    "failed to start rebalance after metadata initialized for {id}: {start_err}; rollback failed: {rollback_err}"
930                )));
931            }
932
933            return Err(Error::other(format!(
934                "failed to start rebalance after metadata initialized for {id}; local metadata was finalized as failed: {start_err}"
935            )));
936        }
937
938        Ok(id)
939    }
940
941    #[tracing::instrument(skip(self, bucktes))]
942    pub async fn init_rebalance_start(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
943        let _start_guard = self.start_gate.lock().await;
944        {
945            let decommission_running = self.is_decommission_running().await;
946            let rebalance_meta = self.rebalance_meta.read().await;
947            validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
948        }
949        let _activation_guard = self
950            .rebalance_activation_write_guard(None, "initialize replacement rebalance")
951            .await?;
952        let movement_gate = self.ctx.data_movement_operation_gate();
953        let _movement_guard = movement_gate.write().await;
954
955        let id = self.init_rebalance_meta(bucktes).await?;
956        self.ctx.advance_data_movement_operation_epoch();
957        Ok(id)
958    }
959
960    #[tracing::instrument(skip(self))]
961    pub async fn start_rebalance_for_id(self: &Arc<Self>, expected_id: &str) -> Result<()> {
962        let _start_guard = self.start_gate.lock().await;
963        let _activation_guard = self
964            .rebalance_activation_write_guard(Some(expected_id), "start rebalance")
965            .await?;
966        let movement_gate = self.ctx.data_movement_operation_gate();
967        let _movement_guard = movement_gate.write().await;
968
969        {
970            let rebalance_meta = self.rebalance_meta.read().await;
971            let Some(meta) = rebalance_meta.as_ref() else {
972                return Err(Error::ConfigNotFound);
973            };
974            if meta.id != expected_id {
975                return Err(Error::other(format!(
976                    "rebalance metadata changed before start: expected {expected_id}, found {}",
977                    meta.id
978                )));
979            }
980            if meta.stopped_at.is_some() {
981                if !is_rebalance_conflicting_with_decommission(meta) {
982                    debug!(
983                        event = EVENT_REBALANCE_STATE,
984                        component = LOG_COMPONENT_ECSTORE,
985                        subsystem = LOG_SUBSYSTEM_REBALANCE,
986                        state = "start_skipped",
987                        reason = "not_started_terminal",
988                        rebalance_id = %expected_id,
989                        "Skipped rebalance start because metadata is already terminal"
990                    );
991                    return Ok(());
992                }
993                return Err(Error::other(format!("rebalance {expected_id} was stopped before start")));
994            }
995        }
996
997        if self.start_rebalance_under_gate().await? {
998            self.ctx.advance_data_movement_operation_epoch();
999        }
1000        Ok(())
1001    }
1002
1003    pub async fn rollback_rebalance_start_for_id(self: &Arc<Self>, expected_id: Option<&str>, start_error: String) -> Result<()> {
1004        self.rollback_rebalance_start_without_worker_for_id(expected_id, start_error)
1005            .await
1006    }
1007
1008    #[tracing::instrument(skip(self, fi))]
1009    pub async fn update_pool_stats(&self, pool_index: usize, bucket: String, fi: &FileInfo) -> Result<()> {
1010        self.update_pool_stats_batch(pool_index, bucket, &[fi]).await
1011    }
1012
1013    #[tracing::instrument(skip(self, versions))]
1014    pub async fn update_pool_stats_batch(&self, pool_index: usize, bucket: String, versions: &[&FileInfo]) -> Result<()> {
1015        self.update_pool_stats_batch_for_id(pool_index, bucket, versions, None).await
1016    }
1017
1018    pub(super) async fn update_pool_stats_batch_for_rebalance(
1019        &self,
1020        pool_index: usize,
1021        bucket: String,
1022        versions: &[&FileInfo],
1023        expected_id: &str,
1024    ) -> Result<()> {
1025        self.update_pool_stats_batch_for_id(pool_index, bucket, versions, Some(expected_id))
1026            .await
1027    }
1028
1029    async fn update_pool_stats_batch_for_id(
1030        &self,
1031        pool_index: usize,
1032        bucket: String,
1033        versions: &[&FileInfo],
1034        expected_id: Option<&str>,
1035    ) -> Result<()> {
1036        if versions.is_empty() {
1037            return Ok(());
1038        }
1039
1040        let mut rebalance_meta = self.rebalance_meta.write().await;
1041        if let Some(expected_id) = expected_id {
1042            ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "update rebalance pool stats")?;
1043        }
1044        if let Some(meta) = rebalance_meta.as_mut() {
1045            if !should_accept_rebalance_stats_update(meta, pool_index) {
1046                return Ok(());
1047            }
1048
1049            if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
1050                pool_stat.update_batch(bucket, versions);
1051            }
1052        }
1053
1054        Ok(())
1055    }
1056
1057    #[tracing::instrument(skip(self))]
1058    pub async fn next_rebal_bucket(&self, pool_index: usize, expected_id: &str) -> Result<Option<String>> {
1059        let rebalance_meta = self.rebalance_meta.read().await;
1060        ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "next rebalance bucket")?;
1061        debug!(
1062            event = EVENT_REBALANCE_BUCKET,
1063            component = LOG_COMPONENT_ECSTORE,
1064            subsystem = LOG_SUBSYSTEM_REBALANCE,
1065            pool_index,
1066            has_meta = rebalance_meta.is_some(),
1067            state = "next_bucket_lookup",
1068            "Rebalance next bucket lookup"
1069        );
1070        resolve_next_rebalance_bucket(rebalance_meta.as_ref(), pool_index)
1071    }
1072
1073    #[tracing::instrument(skip(self))]
1074    pub async fn bucket_rebalance_done(&self, pool_index: usize, bucket: String, expected_id: &str) -> Result<()> {
1075        let mut rebalance_meta = self.rebalance_meta.write().await;
1076        ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "mark rebalance bucket done")?;
1077        mark_rebalance_bucket_done(rebalance_meta.as_mut(), pool_index, &bucket)
1078    }
1079
1080    pub(super) async fn record_rebalance_cleanup_warning(
1081        &self,
1082        pool_index: usize,
1083        bucket: &str,
1084        object: &str,
1085        message: String,
1086        expected_id: &str,
1087    ) -> Result<()> {
1088        let mut rebalance_meta = self.rebalance_meta.write().await;
1089        ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "record rebalance cleanup warning")?;
1090        record_rebalance_cleanup_warning_in_meta(
1091            rebalance_meta.as_mut(),
1092            pool_index,
1093            bucket,
1094            object,
1095            message,
1096            OffsetDateTime::now_utc(),
1097        )
1098    }
1099
1100    pub(super) async fn defer_rebalance_bucket(
1101        &self,
1102        pool_index: usize,
1103        bucket: String,
1104        last_error: String,
1105        expected_id: &str,
1106    ) -> Result<()> {
1107        let mut rebalance_meta = self.rebalance_meta.write().await;
1108        ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "defer rebalance bucket")?;
1109        let Some(meta) = rebalance_meta.as_mut() else {
1110            return Err(rebalance_metadata_not_initialized_error("defer rebalance bucket"));
1111        };
1112        let pool_count = meta.pool_stats.len();
1113        ensure_valid_rebalance_pool_index(pool_count, pool_index)?;
1114        let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) else {
1115            return Err(invalid_rebalance_pool_index_error(pool_index, pool_count));
1116        };
1117
1118        defer_bucket_in_rebalance_queue(pool_stat, &bucket)?;
1119        pool_stat.info.last_error = Some(last_error);
1120        meta.last_refreshed_at = Some(OffsetDateTime::now_utc());
1121        Ok(())
1122    }
1123
1124    pub async fn is_rebalance_started(&self) -> bool {
1125        let rebalance_meta = self.rebalance_meta.read().await;
1126        if let Some(meta) = rebalance_meta.as_ref() {
1127            meta.pool_stats.iter().enumerate().for_each(|(i, v)| {
1128                debug!(
1129                    event = EVENT_REBALANCE_STATE,
1130                    component = LOG_COMPONENT_ECSTORE,
1131                    subsystem = LOG_SUBSYSTEM_REBALANCE,
1132                    pool_index = i,
1133                    participating = v.participating,
1134                    status = ?v.info.status,
1135                    state = "status_inspected",
1136                    "Rebalance status inspected"
1137                );
1138            });
1139
1140            let started = is_rebalance_conflicting_with_decommission(meta);
1141            if started {
1142                debug!(
1143                    event = EVENT_REBALANCE_STATE,
1144                    component = LOG_COMPONENT_ECSTORE,
1145                    subsystem = LOG_SUBSYSTEM_REBALANCE,
1146                    state = "running",
1147                    "Rebalance is running"
1148                );
1149                return true;
1150            }
1151        }
1152
1153        debug!(
1154            event = EVENT_REBALANCE_STATE,
1155            component = LOG_COMPONENT_ECSTORE,
1156            subsystem = LOG_SUBSYSTEM_REBALANCE,
1157            state = "not_running",
1158            "Rebalance is not running"
1159        );
1160        false
1161    }
1162
1163    pub async fn is_rebalance_conflicting_with_decommission(&self) -> bool {
1164        let rebalance_meta = self.rebalance_meta.read().await;
1165        rebalance_meta
1166            .as_ref()
1167            .is_some_and(is_rebalance_conflicting_with_decommission)
1168    }
1169
1170    pub async fn is_pool_rebalancing(&self, pool_index: usize) -> bool {
1171        let rebalance_meta = self.rebalance_meta.read().await;
1172        if let Some(ref meta) = *rebalance_meta {
1173            if meta.stopped_at.is_some() {
1174                return false;
1175            }
1176
1177            if let Some(pool_stat) = meta.pool_stats.get(pool_index) {
1178                return pool_stat.participating && pool_stat.info.status == RebalStatus::Started;
1179            }
1180        }
1181
1182        false
1183    }
1184
1185    pub async fn pool_rebalance_status(&self, pool_index: usize) -> (RebalStatus, bool) {
1186        let rebalance_meta = self.rebalance_meta.read().await;
1187        pool_rebalance_status_from_meta(rebalance_meta.as_ref(), pool_index)
1188    }
1189
1190    pub async fn current_rebalance_id(&self) -> Option<String> {
1191        let rebalance_meta = self.rebalance_meta.read().await;
1192        rebalance_meta
1193            .as_ref()
1194            .and_then(|meta| (!meta.id.is_empty()).then(|| meta.id.clone()))
1195    }
1196
1197    pub async fn cancel_rebalance_admission_for_id(&self, expected_id: &str) -> Result<()> {
1198        let _start_guard = self.start_gate.lock().await;
1199        let mut rebalance_meta = self.rebalance_meta.write().await;
1200        ensure_rebalance_run_id(rebalance_meta.as_ref(), expected_id, "cancel rebalance admission")?;
1201        let meta = rebalance_meta
1202            .as_mut()
1203            .ok_or_else(|| rebalance_metadata_not_initialized_error("cancel rebalance admission"))?;
1204        if meta.stopped_at.is_some() || (!is_rebalance_conflicting_with_decommission(meta) && !meta.stop_requested) {
1205            return Err(Error::other(format!(
1206                "inactive rebalance rejected while cancelling admission: {expected_id}"
1207            )));
1208        }
1209        meta.stop_requested = true;
1210        meta.cancel
1211            .get_or_insert_with(tokio_util::sync::CancellationToken::new)
1212            .cancel();
1213        Ok(())
1214    }
1215
1216    #[tracing::instrument(skip(self))]
1217    pub async fn stop_rebalance(self: &Arc<Self>) -> Result<()> {
1218        self.stop_rebalance_for_id(None).await
1219    }
1220
1221    #[tracing::instrument(skip(self))]
1222    pub async fn stop_rebalance_for_id(self: &Arc<Self>, expected_id: Option<&str>) -> Result<()> {
1223        let _start_guard = self.start_gate.lock().await;
1224        let activation_gate = {
1225            let mut rebalance_meta = self.rebalance_meta.write().await;
1226            if let Some(expected_id) = expected_id {
1227                ensure_rebalance_run_id(rebalance_meta.as_ref(), expected_id, "stop rebalance")?;
1228            }
1229            rebalance_meta.as_mut().map(|meta| {
1230                meta.stop_requested |= is_rebalance_conflicting_with_decommission(meta);
1231                let cancel = meta.cancel.get_or_insert_with(tokio_util::sync::CancellationToken::new);
1232                cancel.cancel();
1233                Arc::clone(&meta.activation_gate)
1234            })
1235        };
1236        let _activation_guard = match activation_gate {
1237            Some(gate) => {
1238                #[cfg(any(test, feature = "test-util"))]
1239                observe_rebalance_stop_wait_attempt(expected_id);
1240                Some(gate.write_owned().await)
1241            }
1242            None => None,
1243        };
1244        let movement_gate = self.ctx.data_movement_operation_gate();
1245        let _movement_guard = movement_gate.write().await;
1246        let stopped_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
1247        let (previous_meta, meta_to_save) = {
1248            let mut rebalance_meta = self.rebalance_meta.write().await;
1249            let previous_meta = rebalance_meta.clone();
1250            let meta_to_save = stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), stopped_at, expected_id)?;
1251            (previous_meta, meta_to_save)
1252        };
1253
1254        if let Some(meta_to_save) = meta_to_save {
1255            let pool = clone_first_arc(self.pools.as_slice(), "stop_rebalance: no pools available")?;
1256            let save_result = resolve_rebalance_meta_save_result(
1257                self.save_rebalance_meta_for_id_with_merge(pool, &meta_to_save, "stop_rebalance", meta_to_save.id.as_str())
1258                    .await,
1259                "stop_rebalance",
1260            );
1261            if let Err(err) = save_result {
1262                *self.rebalance_meta.write().await = previous_meta;
1263                return Err(err);
1264            }
1265            self.ctx.advance_data_movement_operation_epoch();
1266        }
1267
1268        Ok(())
1269    }
1270
1271    async fn rollback_rebalance_start_without_worker_for_id(
1272        self: &Arc<Self>,
1273        expected_id: Option<&str>,
1274        start_error: String,
1275    ) -> Result<()> {
1276        let _start_guard = self.start_gate.lock().await;
1277        let _activation_guard = self
1278            .rebalance_activation_write_guard(expected_id, "rollback rebalance start")
1279            .await?;
1280        let movement_gate = self.ctx.data_movement_operation_gate();
1281        let _movement_guard = movement_gate.write().await;
1282        let failed_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
1283        let meta_to_save = {
1284            let mut rebalance_meta = self.rebalance_meta.write().await;
1285            rollback_rebalance_start_meta_snapshot_for_id(rebalance_meta.as_mut(), failed_at, expected_id, start_error)
1286        };
1287
1288        if let Some(meta_to_save) = meta_to_save {
1289            let pool = clone_first_arc(self.pools.as_slice(), "rollback_rebalance_start: no pools available")?;
1290            resolve_rebalance_meta_save_result(
1291                self.save_rebalance_meta_for_id_with_merge(
1292                    pool,
1293                    &meta_to_save,
1294                    "rollback_rebalance_start",
1295                    meta_to_save.id.as_str(),
1296                )
1297                .await,
1298                "rollback_rebalance_start",
1299            )?;
1300            self.ctx.advance_data_movement_operation_epoch();
1301        }
1302
1303        Ok(())
1304    }
1305
1306    pub async fn record_rebalance_stop_propagation(
1307        self: &Arc<Self>,
1308        expected_id: &str,
1309        record: RebalanceStopPropagationRecord,
1310    ) -> Result<()> {
1311        if !record.has_failures() {
1312            return Ok(());
1313        }
1314
1315        let _start_guard = self.start_gate.lock().await;
1316        let _activation_guard = self
1317            .rebalance_activation_write_guard(Some(expected_id), "record rebalance stop propagation")
1318            .await?;
1319        let movement_gate = self.ctx.data_movement_operation_gate();
1320        let _movement_guard = movement_gate.write().await;
1321        let encoded_error = encode_rebalance_stop_propagation_record(&record);
1322        let meta_to_save = {
1323            let mut rebalance_meta = self.rebalance_meta.write().await;
1324            ensure_rebalance_run_id(rebalance_meta.as_ref(), expected_id, "record rebalance stop propagation")?;
1325            record_rebalance_stop_propagation_snapshot(rebalance_meta.as_mut(), encoded_error, OffsetDateTime::now_utc())
1326        };
1327
1328        if let Some(meta_to_save) = meta_to_save {
1329            let pool = clone_first_arc(self.pools.as_slice(), "record_rebalance_stop_propagation: no pools available")?;
1330            resolve_rebalance_meta_save_result(
1331                self.save_rebalance_meta_for_id_with_merge(pool, &meta_to_save, "record_rebalance_stop_propagation", expected_id)
1332                    .await,
1333                "record_rebalance_stop_propagation",
1334            )?;
1335            self.ctx.advance_data_movement_operation_epoch();
1336        }
1337
1338        Ok(())
1339    }
1340}
1341
1342#[cfg(test)]
1343mod tests {
1344    use super::*;
1345    use crate::config::com::delete_config;
1346    use crate::core::pools::{
1347        DecommissionErasureLayout, DecommissionPoolCapacityInfo, POOL_META_NAME, PoolActivationDurableSaveBarrier,
1348        PoolActivationStartKind, PoolActivationStartProbe, PoolMetaWriteState, persist_pool_meta_identity_for_startup,
1349        set_decommission_capacity_info_overrides_for_test,
1350    };
1351    use crate::object_api::NamespaceLockFence;
1352    use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause, hermetic_set_disks_isolated};
1353
1354    #[tokio::test]
1355    async fn rebalance_status_read_recovers_after_metadata_lock_timeout() {
1356        let id = "rebalance-status-lock-retry";
1357        let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1358            id: id.to_string(),
1359            pool_stats: vec![RebalanceStats::default()],
1360            ..Default::default()
1361        })
1362        .await;
1363        let ns_lock = store.pools[0]
1364            .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME)
1365            .await
1366            .expect("metadata lock should be available");
1367        let writer = ns_lock
1368            .get_write_lock(get_lock_acquire_timeout())
1369            .await
1370            .expect("hold metadata writer");
1371        let retried = Arc::new(tokio::sync::Notify::new());
1372        let refresh = super::super::worker::REBALANCE_METADATA_RETRY_PROBE
1373            .scope(Arc::clone(&retried), store.refresh_rebalance_status_meta());
1374        tokio::pin!(refresh);
1375        tokio::time::timeout(std::time::Duration::from_secs(30), async {
1376            tokio::select! {
1377                _ = retried.notified() => {},
1378                result = &mut refresh => panic!("status read must retry its contended metadata lock: {result:?}"),
1379            }
1380        })
1381        .await
1382        .expect("status read should encounter a real lock timeout");
1383        drop(writer);
1384        tokio::time::timeout(std::time::Duration::from_secs(30), refresh)
1385            .await
1386            .expect("status refresh should finish after the writer releases")
1387            .expect("transient metadata contention must not fail status refresh");
1388        assert_eq!(
1389            store
1390                .rebalance_meta
1391                .read()
1392                .await
1393                .as_ref()
1394                .expect("metadata must remain present")
1395                .id,
1396            id
1397        );
1398    }
1399
1400    #[tokio::test]
1401    async fn rebalance_stats_save_recovers_after_metadata_lock_timeout() {
1402        let id = "rebalance-stats-lock-retry";
1403        let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1404            id: id.to_string(),
1405            pool_stats: vec![RebalanceStats::default()],
1406            ..Default::default()
1407        })
1408        .await;
1409        store
1410            .rebalance_meta
1411            .write()
1412            .await
1413            .as_mut()
1414            .expect("local metadata must exist")
1415            .pool_stats[0]
1416            .num_objects = 7;
1417        let ns_lock = store.pools[0]
1418            .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME)
1419            .await
1420            .expect("metadata lock should be available");
1421        let reader = ns_lock
1422            .get_read_lock(get_lock_acquire_timeout())
1423            .await
1424            .expect("hold a migration read fence");
1425        let retried = Arc::new(tokio::sync::Notify::new());
1426        let save = super::super::worker::REBALANCE_METADATA_RETRY_PROBE.scope(
1427            Arc::clone(&retried),
1428            store.save_rebalance_stats_for_id(0, super::super::RebalSaveOpt::Stats, id),
1429        );
1430        tokio::pin!(save);
1431        tokio::time::timeout(std::time::Duration::from_secs(30), async {
1432            tokio::select! {
1433                _ = retried.notified() => {},
1434                result = &mut save => panic!("stats save must retry its contended metadata lock: {result:?}"),
1435            }
1436        })
1437        .await
1438        .expect("stats save should encounter a real lock timeout");
1439        drop(reader);
1440        tokio::time::timeout(std::time::Duration::from_secs(30), save)
1441            .await
1442            .expect("stats save should finish after the migration read fence releases")
1443            .expect("transient metadata contention must not lose stats");
1444        let mut persisted = RebalanceMeta::new();
1445        persisted
1446            .load(Arc::clone(&store.pools[0]))
1447            .await
1448            .expect("persisted stats must be readable");
1449        assert_eq!(persisted.id, id);
1450        assert_eq!(persisted.pool_stats[0].num_objects, 7);
1451    }
1452
1453    async fn persist_initialized_identity_then_remove_pool_meta(store: &Arc<ECStore>) {
1454        let deployment_id = store
1455            .ctx
1456            .deployment_id()
1457            .expect("test store should have a deployment identity");
1458        let mut write_state = PoolMetaWriteState::for_startup(deployment_id, false);
1459        persist_pool_meta_identity_for_startup(store.pools.clone(), &mut write_state, true)
1460            .await
1461            .expect("initialized pool metadata identity should persist");
1462        *store.pool_meta_save_gate.lock().await = write_state;
1463        for pool in &store.pools {
1464            delete_config(pool.clone(), POOL_META_NAME)
1465                .await
1466                .expect("every pool metadata replica should be removed");
1467        }
1468    }
1469
1470    async fn assert_activation_locks_released(store: &Arc<ECStore>) {
1471        let fleet_proof = acquire_pool_activation_fleet_proof(&store.ctx)
1472            .await
1473            .expect("fleet proof should remain available");
1474        tokio::time::timeout(
1475            std::time::Duration::from_secs(5),
1476            acquire_pool_rebalance_activation_locks(store.pools[0].clone(), fleet_proof),
1477        )
1478        .await
1479        .expect("a rejected activation must release namespace fences promptly")
1480        .expect("a rejected activation must release both namespace fences");
1481    }
1482
1483    #[tokio::test]
1484    async fn rebalance_stop_wait_probe_matches_run_id() {
1485        let probe = RebalanceStopWaitProbe::install("rebalance-stop-current");
1486
1487        observe_rebalance_stop_wait_attempt(Some("rebalance-stop-stale"));
1488        assert!(!probe.state.attempted.load(std::sync::atomic::Ordering::Acquire));
1489
1490        observe_rebalance_stop_wait_attempt(Some("rebalance-stop-current"));
1491        probe.wait_until_attempted().await;
1492    }
1493
1494    #[test]
1495    fn rebalance_stop_classification_checks_identity_and_explicit_intent() {
1496        let mut meta = RebalanceMeta {
1497            id: "current".to_string(),
1498            cancel: Some(tokio_util::sync::CancellationToken::new()),
1499            pool_stats: vec![RebalanceStats {
1500                participating: true,
1501                info: RebalanceInfo {
1502                    status: RebalStatus::Started,
1503                    ..Default::default()
1504                },
1505                ..Default::default()
1506            }],
1507            ..Default::default()
1508        };
1509        ensure_rebalance_worker_active(Some(&meta), "current", "test").expect("active worker");
1510        meta.cancel.as_ref().unwrap().cancel();
1511        assert!(
1512            !matches!(
1513                ensure_rebalance_worker_active(Some(&meta), "current", "test"),
1514                Err(Error::OperationCanceled)
1515            ),
1516            "a sibling failure is not an operator stop"
1517        );
1518        meta.stop_requested = true;
1519        assert!(matches!(
1520            ensure_rebalance_worker_active(Some(&meta), "current", "test"),
1521            Err(Error::OperationCanceled)
1522        ));
1523        assert!(
1524            !matches!(ensure_rebalance_worker_active(Some(&meta), "old", "test"), Err(Error::OperationCanceled)),
1525            "stale identity remains a failure even during stop"
1526        );
1527        assert!(!matches!(
1528            ensure_rebalance_worker_active(None, "current", "test"),
1529            Err(Error::OperationCanceled)
1530        ));
1531    }
1532
1533    #[tokio::test]
1534    async fn rebalance_stop_intent_does_not_survive_replacement_run_reload() {
1535        let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1536            id: "replacement".to_string(),
1537            pool_stats: vec![RebalanceStats {
1538                participating: true,
1539                info: RebalanceInfo {
1540                    status: RebalStatus::Completed,
1541                    ..Default::default()
1542                },
1543                ..Default::default()
1544            }],
1545            ..Default::default()
1546        })
1547        .await;
1548        let previous_gate = {
1549            let mut meta = store.rebalance_meta.write().await;
1550            let meta = meta.as_mut().unwrap();
1551            meta.id = "previous".to_string();
1552            meta.stop_requested = true;
1553            let cancel = tokio_util::sync::CancellationToken::new();
1554            cancel.cancel();
1555            meta.cancel = Some(cancel);
1556            Arc::clone(&meta.activation_gate)
1557        };
1558        store.load_rebalance_meta().await.expect("reload replacement run");
1559        let meta = store.rebalance_meta.read().await;
1560        let meta = meta.as_ref().unwrap();
1561        assert_eq!(meta.id, "replacement");
1562        assert!(!meta.stop_requested);
1563        assert!(meta.cancel.is_none());
1564        assert!(!Arc::ptr_eq(&previous_gate, &meta.activation_gate));
1565    }
1566
1567    #[tokio::test]
1568    async fn cancel_rebalance_admission_is_id_checked_and_idempotent() {
1569        let rebalance_id = "rebalance-admission-current";
1570        let cancel = tokio_util::sync::CancellationToken::new();
1571        let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1572            id: rebalance_id.to_string(),
1573            percent_free_goal: 1.0,
1574            cancel: Some(cancel.clone()),
1575            pool_stats: vec![RebalanceStats {
1576                participating: true,
1577                init_capacity: 100,
1578                info: RebalanceInfo {
1579                    start_time: Some(OffsetDateTime::now_utc()),
1580                    status: RebalStatus::Started,
1581                    ..Default::default()
1582                },
1583                ..Default::default()
1584            }],
1585            ..Default::default()
1586        })
1587        .await;
1588
1589        let error = store
1590            .cancel_rebalance_admission_for_id("rebalance-admission-replacement")
1591            .await
1592            .expect_err("a stale admin stop must not cancel the current run");
1593        assert!(error.to_string().contains("expected rebalance-admission-replacement"));
1594        assert!(!cancel.is_cancelled());
1595
1596        store
1597            .cancel_rebalance_admission_for_id(rebalance_id)
1598            .await
1599            .expect("the current run admission should close");
1600        store
1601            .cancel_rebalance_admission_for_id(rebalance_id)
1602            .await
1603            .expect("retrying admission cancellation should be idempotent");
1604        assert!(cancel.is_cancelled());
1605        let err = store
1606            .update_pool_stats_batch_for_rebalance(0, "bucket".to_string(), &[&FileInfo::default()], rebalance_id)
1607            .await
1608            .expect_err("stop racing with a final stats update must cancel that update");
1609        assert!(matches!(err, Error::OperationCanceled), "operator stop lost its cancellation type: {err}");
1610    }
1611
1612    #[tokio::test]
1613    async fn prepare_rebalance_stop_preserves_intent_when_worker_stops_before_reload() {
1614        let id = "stop-worker-before-reload";
1615        let cancel = tokio_util::sync::CancellationToken::new();
1616        let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1617            id: id.to_string(),
1618            cancel: Some(cancel.clone()),
1619            pool_stats: vec![RebalanceStats {
1620                participating: true,
1621                info: RebalanceInfo {
1622                    status: RebalStatus::Stopped,
1623                    ..Default::default()
1624                },
1625                ..Default::default()
1626            }],
1627            ..Default::default()
1628        })
1629        .await;
1630        let gate = {
1631            let mut meta = store.rebalance_meta.write().await;
1632            let meta = meta.as_mut().expect("local rebalance metadata");
1633            meta.pool_stats[0].info.status = RebalStatus::Started;
1634            Arc::clone(&meta.activation_gate)
1635        };
1636        assert_eq!(
1637            store.prepare_rebalance_stop().await.expect("prepare the same run stop"),
1638            Some(id.to_string())
1639        );
1640        {
1641            let meta = store.rebalance_meta.read().await;
1642            let meta = meta.as_ref().expect("reloaded stop target");
1643            assert!(Arc::ptr_eq(&gate, &meta.activation_gate), "reload must retain the drained run's gate");
1644            assert!(meta.cancel.as_ref().is_some_and(|token| token.is_cancelled()));
1645        }
1646        store
1647            .stop_rebalance_for_id(Some(id))
1648            .await
1649            .expect("finish the stop after the worker's terminal event");
1650        store
1651            .load_rebalance_meta()
1652            .await
1653            .expect("reload the acknowledged durable stop");
1654        let meta = store.rebalance_meta.read().await;
1655        let meta = meta.as_ref().expect("durable stopped metadata");
1656        assert!(meta.stopped_at.is_some(), "a successful stop must retain its durable timestamp");
1657        assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped);
1658    }
1659
1660    #[tokio::test]
1661    #[serial_test::serial]
1662    async fn equal_free_ratio_admin_no_participant_rebalance_succeeds_and_persists_terminal_generation_after_restart() {
1663        let (_temp_dirs, store, restarted) =
1664            crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
1665        let movement_floor = OffsetDateTime::from_unix_timestamp(4_100_000_000).expect("future test timestamp should be valid");
1666        *store.rebalance_meta.write().await = Some(RebalanceMeta {
1667            id: "previous-terminal-rebalance".to_string(),
1668            stopped_at: Some(movement_floor),
1669            ..Default::default()
1670        });
1671        set_rebalance_disk_stats_override_for_test(
1672            store.id,
1673            vec![
1674                DiskStat {
1675                    total_space: 100,
1676                    available_space: 50,
1677                },
1678                DiskStat {
1679                    total_space: 100,
1680                    available_space: 50,
1681                },
1682            ],
1683        );
1684
1685        let rebalance_id = store
1686            .init_and_start_rebalance(vec!["equal-ratio-no-op".to_string()])
1687            .await
1688            .expect("equal free ratio admin rebalance should succeed as a terminal no-op");
1689        let stopped_at = {
1690            let local = store.rebalance_meta.read().await;
1691            let local = local.as_ref().expect("no-op rebalance metadata should remain available");
1692            assert_eq!(local.id, rebalance_id);
1693            assert!(local.pool_stats.iter().all(|pool_stat| !pool_stat.participating));
1694            let stopped_at = local.stopped_at.expect("no-op rebalance must persist a terminal timestamp");
1695            assert_eq!(stopped_at, movement_floor + time::Duration::nanoseconds(1));
1696            stopped_at
1697        };
1698
1699        let stopped_generation =
1700            u64::try_from(stopped_at.unix_timestamp_nanos()).expect("terminal timestamp should map to scanner generation");
1701        let live_status = store.scanner_data_movement_pause_status().await;
1702        assert!(!live_status.paused);
1703        assert_eq!(live_status.movement_generation, stopped_generation);
1704
1705        restarted
1706            .load_rebalance_meta()
1707            .await
1708            .expect("restarted store should load the persisted no-op rebalance metadata");
1709        let status = restarted.scanner_data_movement_pause_status().await;
1710
1711        assert!(!status.paused);
1712        assert_eq!(status.movement_generation, stopped_generation);
1713        assert_eq!(restarted.scanner_data_movement_generation(), stopped_generation);
1714    }
1715
1716    #[tokio::test]
1717    #[serial_test::serial]
1718    async fn rebalance_activation_rejects_initialized_cluster_with_all_pool_meta_missing() {
1719        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
1720        persist_initialized_identity_then_remove_pool_meta(&store).await;
1721        set_rebalance_disk_stats_override_for_test(
1722            store.id,
1723            vec![
1724                DiskStat {
1725                    total_space: 100,
1726                    available_space: 0,
1727                },
1728                DiskStat {
1729                    total_space: 100,
1730                    available_space: 100,
1731                },
1732            ],
1733        );
1734
1735        let err = store
1736            .init_rebalance_start(vec!["missing-pool-meta".to_string()])
1737            .await
1738            .expect_err("rebalance activation must fail closed when every pool.bin is missing");
1739        assert!(err.to_string().contains("initialized cluster identity exists"));
1740        store
1741            .ensure_pool_meta_side_effects_safe("rebalance activation after missing pool metadata")
1742            .await
1743            .expect_err("the missing metadata observation must latch the shared runtime gate");
1744
1745        let mut persisted = RebalanceMeta::new();
1746        assert!(
1747            matches!(persisted.load(store.pools[0].clone()).await, Err(Error::ConfigNotFound)),
1748            "rejected activation must not create rebalance metadata"
1749        );
1750        assert_activation_locks_released(&store).await;
1751    }
1752
1753    #[tokio::test]
1754    #[serial_test::serial]
1755    async fn rebalance_worker_rejects_initialized_cluster_with_all_pool_meta_missing() {
1756        let rebalance_id = "missing-pool-meta-worker";
1757        let active = RebalanceMeta {
1758            id: rebalance_id.to_string(),
1759            percent_free_goal: 0.5,
1760            pool_stats: vec![
1761                RebalanceStats {
1762                    participating: true,
1763                    init_capacity: 100,
1764                    info: RebalanceInfo {
1765                        status: RebalStatus::Started,
1766                        ..Default::default()
1767                    },
1768                    ..Default::default()
1769                },
1770                RebalanceStats {
1771                    participating: true,
1772                    init_capacity: 100,
1773                    info: RebalanceInfo {
1774                        status: RebalStatus::Started,
1775                        ..Default::default()
1776                    },
1777                    ..Default::default()
1778                },
1779            ],
1780            ..Default::default()
1781        };
1782        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(active)).await;
1783        persist_initialized_identity_then_remove_pool_meta(&store).await;
1784
1785        let err = match store
1786            .fence_rebalance_worker_activation(store.pools[0].clone(), rebalance_id)
1787            .await
1788        {
1789            Ok(_) => panic!("worker activation must not return a fence when every pool.bin is missing"),
1790            Err(err) => err,
1791        };
1792        assert!(err.to_string().contains("initialized cluster identity exists"));
1793        store
1794            .ensure_pool_meta_side_effects_safe("rebalance worker after missing pool metadata")
1795            .await
1796            .expect_err("worker validation must latch the shared runtime gate");
1797        assert_activation_locks_released(&store).await;
1798    }
1799
1800    #[tokio::test]
1801    #[serial_test::serial]
1802    async fn rebalance_worker_skips_terminal_metadata_without_fleet_proof() {
1803        let rebalance_id = "terminal-metadata-without-proof";
1804        let completed = RebalanceMeta {
1805            id: rebalance_id.to_string(),
1806            pool_stats: vec![RebalanceStats {
1807                participating: true,
1808                info: RebalanceInfo {
1809                    status: RebalStatus::Completed,
1810                    ..Default::default()
1811                },
1812                ..Default::default()
1813            }],
1814            ..Default::default()
1815        };
1816        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(completed)).await;
1817        let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
1818
1819        let activation = store
1820            .fence_rebalance_worker_activation(store.pools[0].clone(), rebalance_id)
1821            .await
1822            .expect("terminal metadata should not require a fleet proof");
1823        assert!(matches!(activation, RebalanceWorkerActivationFence::NotStartedTerminal));
1824    }
1825
1826    #[tokio::test]
1827    #[serial_test::serial]
1828    async fn rebalance_worker_still_requires_fleet_proof_for_active_metadata() {
1829        let rebalance_id = "active-metadata-without-proof";
1830        let active = RebalanceMeta {
1831            id: rebalance_id.to_string(),
1832            pool_stats: vec![RebalanceStats {
1833                participating: true,
1834                info: RebalanceInfo {
1835                    status: RebalStatus::Started,
1836                    ..Default::default()
1837                },
1838                ..Default::default()
1839            }],
1840            ..Default::default()
1841        };
1842        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(active)).await;
1843        let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
1844
1845        let err = match store
1846            .fence_rebalance_worker_activation(store.pools[0].clone(), rebalance_id)
1847            .await
1848        {
1849            Ok(_) => panic!("active metadata must not be admitted without a fleet proof"),
1850            Err(err) => err,
1851        };
1852        assert!(
1853            err.to_string()
1854                .contains("pool activation requires a live fleet capability proof")
1855        );
1856    }
1857
1858    #[tokio::test]
1859    #[serial_test::serial]
1860    async fn rebalance_activation_adopts_commit_after_post_save_fence_loss() {
1861        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
1862        set_rebalance_disk_stats_override_for_test(
1863            store.id,
1864            vec![
1865                DiskStat {
1866                    total_space: 100,
1867                    available_space: 0,
1868                },
1869                DiskStat {
1870                    total_space: 100,
1871                    available_space: 100,
1872                },
1873            ],
1874        );
1875        let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
1876        let start_store = Arc::clone(&store);
1877        let start_task = tokio::spawn(async move {
1878            start_store
1879                .init_rebalance_start(vec!["post-commit-fence-loss".to_string()])
1880                .await
1881        });
1882
1883        barrier.wait_until_paused().await;
1884        barrier.release_after_fence_loss();
1885        let rebalance_id = tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
1886            .await
1887            .expect("rebalance activation should finish after its durable commit")
1888            .expect("rebalance activation task should not panic")
1889            .expect("post-commit fence loss must not report the committed activation as failed");
1890
1891        let local_meta = store.rebalance_meta.read().await;
1892        let local = local_meta
1893            .as_ref()
1894            .expect("the committed rebalance metadata should be installed locally");
1895        assert_eq!(local.id, rebalance_id);
1896        assert!(is_rebalance_conflicting_with_decommission(local));
1897        drop(local_meta);
1898
1899        let mut persisted = RebalanceMeta::new();
1900        persisted
1901            .load(store.pools[0].clone())
1902            .await
1903            .expect("the committed rebalance metadata should remain readable");
1904        assert_eq!(persisted.id, rebalance_id);
1905        assert!(is_rebalance_conflicting_with_decommission(&persisted));
1906    }
1907
1908    #[tokio::test]
1909    #[serial_test::serial]
1910    async fn rebalance_worker_admission_installs_committed_candidate_after_post_save_fence_loss() {
1911        let rebalance_id = "post-save-worker-admission";
1912        let active = RebalanceMeta {
1913            id: rebalance_id.to_string(),
1914            percent_free_goal: 0.5,
1915            pool_stats: vec![
1916                RebalanceStats {
1917                    participating: true,
1918                    init_free_space: 50,
1919                    init_capacity: 100,
1920                    buckets: vec!["completed-pool-bucket".to_string()],
1921                    info: RebalanceInfo {
1922                        status: RebalStatus::Started,
1923                        ..Default::default()
1924                    },
1925                    ..Default::default()
1926                },
1927                RebalanceStats {
1928                    participating: true,
1929                    init_capacity: 100,
1930                    buckets: vec!["active-pool-bucket".to_string()],
1931                    info: RebalanceInfo {
1932                        status: RebalStatus::Started,
1933                        ..Default::default()
1934                    },
1935                    ..Default::default()
1936                },
1937            ],
1938            ..Default::default()
1939        };
1940        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(active)).await;
1941        let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
1942        let start_store = Arc::clone(&store);
1943        let start_task = tokio::spawn(async move { start_store.start_rebalance().await });
1944
1945        barrier.wait_until_paused().await;
1946        barrier.release_after_fence_loss();
1947        tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
1948            .await
1949            .expect("rebalance worker admission should finish after its durable commit")
1950            .expect("rebalance worker admission task should not panic")
1951            .expect("post-commit fence loss must not reject the committed worker candidate");
1952
1953        let local_meta = store.rebalance_meta.read().await;
1954        let local = local_meta
1955            .as_ref()
1956            .expect("the committed worker candidate should remain installed locally");
1957        assert_eq!(local.id, rebalance_id);
1958        assert_eq!(local.pool_stats[0].info.status, RebalStatus::Completed);
1959        let admitted_cancel = local
1960            .cancel
1961            .clone()
1962            .expect("the committed worker candidate should install its cancellation token");
1963        assert!(!admitted_cancel.is_cancelled());
1964        drop(local_meta);
1965
1966        admitted_cancel.cancel();
1967        assert!(admitted_cancel.is_cancelled());
1968
1969        let mut persisted = RebalanceMeta::new();
1970        persisted
1971            .load(store.pools[0].clone())
1972            .await
1973            .expect("the committed worker candidate should remain readable");
1974        assert_eq!(persisted.id, rebalance_id);
1975        assert_eq!(persisted.pool_stats[0].info.status, RebalStatus::Completed);
1976    }
1977
1978    async fn assert_real_activation_start_race(paused_kind: PoolActivationStartKind) {
1979        let (_temp_dirs, rebalance_store, decommission_store) =
1980            crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
1981        crate::services::rebalance::promote_test_pool_meta_to_v2(&rebalance_store).await;
1982        crate::services::rebalance::promote_test_pool_meta_to_v2(&decommission_store).await;
1983        let disk_stats = vec![
1984            DiskStat {
1985                total_space: 100,
1986                available_space: 0,
1987            },
1988            DiskStat {
1989                total_space: 100,
1990                available_space: 100,
1991            },
1992        ];
1993        set_rebalance_disk_stats_override_for_test(rebalance_store.id, disk_stats.clone());
1994        set_rebalance_disk_stats_override_for_test(decommission_store.id, disk_stats);
1995        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
1996        let capacity_snapshot = vec![
1997            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
1998            DecommissionPoolCapacityInfo::for_test(1, layout, 200, 200, 0),
1999        ];
2000        // Decommission start samples capacity before and inside its durable activation fence.
2001        set_decommission_capacity_info_overrides_for_test(
2002            decommission_store.id,
2003            vec![capacity_snapshot.clone(), capacity_snapshot],
2004        );
2005        let (first_object, competing_object, competing_kind) = match paused_kind {
2006            PoolActivationStartKind::Rebalance => {
2007                (REBAL_META_NAME, crate::core::pools::POOL_META_NAME, PoolActivationStartKind::Decommission)
2008            }
2009            PoolActivationStartKind::Decommission => {
2010                (crate::core::pools::POOL_META_NAME, REBAL_META_NAME, PoolActivationStartKind::Rebalance)
2011            }
2012        };
2013        let first_barrier = PutObjectCommitBarrier::install(
2014            crate::disk::RUSTFS_META_BUCKET,
2015            first_object,
2016            PutObjectCommitPause::BeforeQuotaRename,
2017        );
2018        let competing_barrier = PutObjectCommitBarrier::install(
2019            crate::disk::RUSTFS_META_BUCKET,
2020            competing_object,
2021            PutObjectCommitPause::BeforeQuotaRename,
2022        );
2023
2024        let mut rebalance_task;
2025        let mut decommission_task;
2026        let mut observed_rebalance_result = None;
2027        let mut observed_decommission_result = None;
2028        let reached_activation = if paused_kind == PoolActivationStartKind::Rebalance {
2029            let store = Arc::clone(&rebalance_store);
2030            rebalance_task =
2031                tokio::spawn(async move { store.init_rebalance_start(vec!["bucket".to_string()]).await.map(|_| ()) });
2032            tokio::select! {
2033                _ = first_barrier.wait_until_paused() => {}
2034                result = &mut rebalance_task => {
2035                    panic!("real rebalance start finished before its activation commit: {:?}", result.expect("rebalance activation task should not panic"));
2036                }
2037            }
2038            let probe = PoolActivationStartProbe::install(competing_kind);
2039            let store = Arc::clone(&decommission_store);
2040            decommission_task = tokio::spawn(async move { store.start_decommission(vec![0]).await });
2041            tokio::time::timeout(std::time::Duration::from_secs(15), async {
2042                tokio::select! {
2043                    _ = probe.wait_until_attempted() => true,
2044                    result = &mut decommission_task => {
2045                        observed_decommission_result = Some(result.expect("decommission activation task should not panic"));
2046                        false
2047                    }
2048                }
2049            })
2050            .await
2051            .unwrap_or(false)
2052        } else {
2053            let store = Arc::clone(&decommission_store);
2054            decommission_task = tokio::spawn(async move { store.start_decommission(vec![0]).await });
2055            tokio::select! {
2056                _ = first_barrier.wait_until_paused() => {}
2057                result = &mut decommission_task => {
2058                    panic!("real decommission start finished before its activation commit: {:?}", result.expect("decommission activation task should not panic"));
2059                }
2060            }
2061            let probe = PoolActivationStartProbe::install(competing_kind);
2062            let store = Arc::clone(&rebalance_store);
2063            rebalance_task =
2064                tokio::spawn(async move { store.init_rebalance_start(vec!["bucket".to_string()]).await.map(|_| ()) });
2065            tokio::time::timeout(std::time::Duration::from_secs(15), async {
2066                tokio::select! {
2067                    _ = probe.wait_until_attempted() => true,
2068                    result = &mut rebalance_task => {
2069                        observed_rebalance_result = Some(result.expect("rebalance activation task should not panic"));
2070                        false
2071                    }
2072                }
2073            })
2074            .await
2075            .unwrap_or(false)
2076        };
2077
2078        // A competing start may be fenced while refreshing persisted metadata,
2079        // before it reaches the activation save, or at the activation commit.
2080        let competing_reached_commit = if reached_activation {
2081            tokio::time::timeout(std::time::Duration::from_secs(15), async {
2082                if paused_kind == PoolActivationStartKind::Rebalance {
2083                    tokio::select! {
2084                        _ = competing_barrier.wait_until_paused() => true,
2085                        result = &mut decommission_task => {
2086                            observed_decommission_result = Some(result.expect("decommission activation task should not panic"));
2087                            false
2088                        }
2089                    }
2090                } else {
2091                    tokio::select! {
2092                        _ = competing_barrier.wait_until_paused() => true,
2093                        result = &mut rebalance_task => {
2094                            observed_rebalance_result = Some(result.expect("rebalance activation task should not panic"));
2095                            false
2096                        }
2097                    }
2098                }
2099            })
2100            .await
2101            .expect("competing activation should either finish or reach its commit")
2102        } else {
2103            false
2104        };
2105        if competing_reached_commit {
2106            competing_barrier.release();
2107        }
2108        drop(competing_barrier);
2109        first_barrier.release();
2110        drop(first_barrier);
2111
2112        let rebalance_result = match observed_rebalance_result {
2113            Some(result) => result,
2114            None => tokio::time::timeout(std::time::Duration::from_secs(15), &mut rebalance_task)
2115                .await
2116                .expect("real rebalance activation should finish")
2117                .expect("rebalance activation task should not panic"),
2118        };
2119        let decommission_result = match observed_decommission_result {
2120            Some(result) => result,
2121            None => tokio::time::timeout(std::time::Duration::from_secs(15), &mut decommission_task)
2122                .await
2123                .expect("real decommission activation should finish")
2124                .expect("decommission activation task should not panic"),
2125        };
2126        assert!(
2127            !competing_reached_commit,
2128            "the competing real start reached its commit while the other activation lock was held"
2129        );
2130        assert_ne!(
2131            rebalance_result.is_ok(),
2132            decommission_result.is_ok(),
2133            "exactly one real activation entry may commit"
2134        );
2135
2136        let mut persisted_rebalance = RebalanceMeta::new();
2137        let rebalance_committed = persisted_rebalance
2138            .load(rebalance_store.pools[0].clone())
2139            .await
2140            .is_ok_and(|_| is_rebalance_conflicting_with_decommission(&persisted_rebalance));
2141        let mut persisted_pool = PoolMeta::default();
2142        persisted_pool
2143            .load_no_lock_from_replicas(rebalance_store.pools.clone())
2144            .await
2145            .expect("persisted pool metadata should remain readable");
2146        let decommission_committed = pool_meta_has_active_decommission(&persisted_pool);
2147        assert_eq!(
2148            usize::from(rebalance_committed) + usize::from(decommission_committed),
2149            1,
2150            "at most one persisted activation state may be active"
2151        );
2152        assert_eq!(rebalance_committed, paused_kind == PoolActivationStartKind::Rebalance);
2153        assert_eq!(decommission_committed, paused_kind == PoolActivationStartKind::Decommission);
2154    }
2155
2156    #[tokio::test]
2157    #[serial_test::serial]
2158    async fn real_rebalance_and_decommission_starts_commit_at_most_one_side() {
2159        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
2160            assert_real_activation_start_race(PoolActivationStartKind::Rebalance).await;
2161            assert_real_activation_start_race(PoolActivationStartKind::Decommission).await;
2162        })
2163        .await;
2164    }
2165
2166    #[test]
2167    fn rebalance_activation_rejects_persisted_decommission_despite_idle_local_snapshot() {
2168        let persisted = PoolMeta {
2169            pools: vec![crate::core::pools::PoolStatus {
2170                id: 0,
2171                cmd_line: "pool-0".to_string(),
2172                last_update: OffsetDateTime::UNIX_EPOCH,
2173                decommission: Some(crate::core::pools::PoolDecommissionInfo {
2174                    start_time: Some(OffsetDateTime::UNIX_EPOCH),
2175                    ..Default::default()
2176                }),
2177            }],
2178            ..Default::default()
2179        };
2180
2181        let err = ensure_rebalance_activation_pool_meta_allowed(&persisted)
2182            .expect_err("persisted decommission must block a stale rebalance admission");
2183        assert!(matches!(err, Error::DecommissionAlreadyRunning));
2184    }
2185
2186    #[test]
2187    fn rebalance_activation_allows_persisted_idle_pool_meta() {
2188        assert!(ensure_rebalance_activation_pool_meta_allowed(&PoolMeta::default()).is_ok());
2189    }
2190
2191    #[tokio::test]
2192    async fn rebalance_merge_save_does_not_commit_after_namespace_fence_loss() {
2193        let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks_isolated(4).await;
2194
2195        let persisted = RebalanceMeta {
2196            id: "rebalance-a".to_string(),
2197            pool_stats: vec![RebalanceStats {
2198                participating: true,
2199                bytes: 1,
2200                info: RebalanceInfo {
2201                    status: RebalStatus::Started,
2202                    ..Default::default()
2203                },
2204                ..Default::default()
2205            }],
2206            ..Default::default()
2207        };
2208        persisted
2209            .save(set_disks.clone())
2210            .await
2211            .expect("baseline rebalance metadata should be saved");
2212
2213        let mut update = persisted.clone();
2214        update.pool_stats[0].bytes = 999;
2215        let err = merge_and_save_rebalance_meta_no_lock(
2216            set_disks.clone(),
2217            &update,
2218            "lost namespace fence test",
2219            ObjectOptions {
2220                no_lock: true,
2221                namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()),
2222                ..Default::default()
2223            },
2224            None,
2225            Some(persisted.id.as_str()),
2226        )
2227        .await
2228        .expect_err("lost namespace fence must reject the metadata commit");
2229        assert!(matches!(err, Error::NamespaceLockQuorumUnavailable { .. }));
2230
2231        let mut after = RebalanceMeta::new();
2232        after
2233            .load(set_disks)
2234            .await
2235            .expect("baseline rebalance metadata should remain readable");
2236        assert_eq!(after.id, persisted.id);
2237        assert_eq!(after.pool_stats[0].bytes, 1);
2238    }
2239
2240    #[tokio::test]
2241    async fn persisted_rebalance_run_fence_rejects_stale_active_node_after_remote_stop() {
2242        let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks_isolated(4).await;
2243        let active = RebalanceMeta {
2244            id: "rebalance-a".to_string(),
2245            cancel: Some(tokio_util::sync::CancellationToken::new()),
2246            pool_stats: vec![RebalanceStats {
2247                participating: true,
2248                info: RebalanceInfo {
2249                    status: RebalStatus::Started,
2250                    ..Default::default()
2251                },
2252                ..Default::default()
2253            }],
2254            ..Default::default()
2255        };
2256        active
2257            .save(set_disks.clone())
2258            .await
2259            .expect("active rebalance metadata should be saved");
2260        ensure_rebalance_worker_active(Some(&active), active.id.as_str(), "stale node precondition")
2261            .expect("the stale node snapshot should still look active locally");
2262
2263        let mut stopped = active.clone();
2264        stopped.stopped_at = Some(OffsetDateTime::now_utc());
2265        stopped.pool_stats[0].info.stopping = true;
2266        stopped
2267            .save(set_disks.clone())
2268            .await
2269            .expect("remote stop should replace persisted metadata");
2270
2271        let err = acquire_persisted_rebalance_run_guard(set_disks, active.id.as_str(), "cross-node stale snapshot")
2272            .await
2273            .expect_err("persisted stop must fence a node that missed stop propagation");
2274        assert!(
2275            matches!(err, Error::OperationCanceled),
2276            "a durable remote stop cancels the same run: {err}"
2277        );
2278    }
2279
2280    #[test]
2281    fn pool_rebalance_status_ignores_non_participating_pool_state() {
2282        let meta = RebalanceMeta {
2283            pool_stats: vec![
2284                RebalanceStats {
2285                    participating: false,
2286                    info: RebalanceInfo {
2287                        status: RebalStatus::Started,
2288                        stopping: true,
2289                        ..Default::default()
2290                    },
2291                    ..Default::default()
2292                },
2293                RebalanceStats {
2294                    participating: true,
2295                    info: RebalanceInfo {
2296                        status: RebalStatus::Started,
2297                        ..Default::default()
2298                    },
2299                    ..Default::default()
2300                },
2301            ],
2302            ..Default::default()
2303        };
2304
2305        assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 0), (RebalStatus::None, false));
2306        assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 1), (RebalStatus::Started, false));
2307        assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 2), (RebalStatus::None, false));
2308    }
2309
2310    #[test]
2311    fn rebalance_status_refresh_applies_persisted_terminal_state() {
2312        let rebalance_id = "rebalance-id".to_string();
2313        let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
2314        let mut current = Some(RebalanceMeta {
2315            id: rebalance_id.clone(),
2316            pool_stats: vec![RebalanceStats {
2317                participating: true,
2318                info: RebalanceInfo {
2319                    status: RebalStatus::Started,
2320                    start_time: Some(now),
2321                    ..Default::default()
2322                },
2323                ..Default::default()
2324            }],
2325            ..Default::default()
2326        });
2327        let persisted = RebalanceMeta {
2328            id: rebalance_id,
2329            pool_stats: vec![RebalanceStats {
2330                participating: true,
2331                info: RebalanceInfo {
2332                    status: RebalStatus::Completed,
2333                    start_time: Some(now),
2334                    end_time: Some(now),
2335                    ..Default::default()
2336                },
2337                ..Default::default()
2338            }],
2339            ..Default::default()
2340        };
2341
2342        assert!(merge_rebalance_status_refresh(&mut current, persisted));
2343
2344        let refreshed = current.as_ref().expect("refresh should keep rebalance metadata");
2345        assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
2346        assert_eq!(refreshed.pool_stats[0].info.end_time, Some(now));
2347    }
2348
2349    #[test]
2350    fn rebalance_status_refresh_preserves_runtime_cancel_token() {
2351        let rebalance_id = "rebalance-id".to_string();
2352        let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
2353        let mut current = Some(RebalanceMeta {
2354            id: rebalance_id.clone(),
2355            cancel: Some(tokio_util::sync::CancellationToken::new()),
2356            pool_stats: vec![RebalanceStats {
2357                participating: true,
2358                info: RebalanceInfo {
2359                    status: RebalStatus::Started,
2360                    start_time: Some(now),
2361                    ..Default::default()
2362                },
2363                ..Default::default()
2364            }],
2365            ..Default::default()
2366        });
2367        let persisted = RebalanceMeta {
2368            id: rebalance_id,
2369            pool_stats: vec![RebalanceStats {
2370                participating: true,
2371                info: RebalanceInfo {
2372                    status: RebalStatus::Started,
2373                    start_time: Some(now),
2374                    ..Default::default()
2375                },
2376                ..Default::default()
2377            }],
2378            ..Default::default()
2379        };
2380
2381        assert!(!merge_rebalance_status_refresh(&mut current, persisted));
2382
2383        assert!(
2384            current.as_ref().and_then(|meta| meta.cancel.as_ref()).is_some(),
2385            "status refresh must not drop the runtime cancellation token"
2386        );
2387    }
2388
2389    #[test]
2390    fn rebalance_status_refresh_preserves_local_active_different_id_conflict() {
2391        let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
2392        let mut current = Some(RebalanceMeta {
2393            id: "old-active-id".to_string(),
2394            cancel: Some(tokio_util::sync::CancellationToken::new()),
2395            pool_stats: vec![RebalanceStats {
2396                participating: true,
2397                info: RebalanceInfo {
2398                    status: RebalStatus::Started,
2399                    start_time: Some(now),
2400                    ..Default::default()
2401                },
2402                ..Default::default()
2403            }],
2404            ..Default::default()
2405        });
2406        let persisted = RebalanceMeta {
2407            id: "new-terminal-id".to_string(),
2408            pool_stats: vec![RebalanceStats {
2409                participating: true,
2410                info: RebalanceInfo {
2411                    status: RebalStatus::Completed,
2412                    start_time: Some(now),
2413                    end_time: Some(now),
2414                    ..Default::default()
2415                },
2416                ..Default::default()
2417            }],
2418            ..Default::default()
2419        };
2420
2421        merge_rebalance_status_refresh(&mut current, persisted);
2422
2423        let refreshed = current.as_ref().expect("local active metadata should remain visible");
2424        assert_eq!(refreshed.id, "old-active-id");
2425        assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Started);
2426        assert!(
2427            refreshed.cancel.is_some(),
2428            "status refresh must not drop a live runtime cancellation token"
2429        );
2430    }
2431
2432    #[test]
2433    fn rebalance_status_refresh_replaces_stale_memory_without_runtime_token() {
2434        let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
2435        let mut current = Some(RebalanceMeta {
2436            id: "old-stale-id".to_string(),
2437            pool_stats: vec![RebalanceStats {
2438                participating: true,
2439                info: RebalanceInfo {
2440                    status: RebalStatus::Started,
2441                    start_time: Some(now),
2442                    ..Default::default()
2443                },
2444                ..Default::default()
2445            }],
2446            ..Default::default()
2447        });
2448        let persisted = RebalanceMeta {
2449            id: "new-terminal-id".to_string(),
2450            pool_stats: vec![RebalanceStats {
2451                participating: true,
2452                info: RebalanceInfo {
2453                    status: RebalStatus::Completed,
2454                    start_time: Some(now),
2455                    end_time: Some(now),
2456                    ..Default::default()
2457                },
2458                ..Default::default()
2459            }],
2460            ..Default::default()
2461        };
2462
2463        merge_rebalance_status_refresh(&mut current, persisted);
2464
2465        let refreshed = current.as_ref().expect("persisted metadata should replace stale memory");
2466        assert_eq!(refreshed.id, "new-terminal-id");
2467        assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
2468        assert!(refreshed.cancel.is_none());
2469    }
2470}