Skip to main content

rustfs_ecstore/services/rebalance/
control.rs

1use super::meta::{
2    RebalanceMetaMergeOutcome, clone_first_arc, clone_rebalance_pool_stats, defer_bucket_in_rebalance_queue,
3    ensure_valid_rebalance_pool_index, invalid_rebalance_pool_index_error, is_rebalance_actively_running,
4    is_rebalance_conflicting_with_decommission, mark_rebalance_bucket_done, merge_rebalance_meta, percent_free_ratio,
5    rebalance_metadata_not_initialized_error, record_rebalance_cleanup_warning_in_meta,
6    record_rebalance_stop_propagation_snapshot, resolve_next_rebalance_bucket, rollback_rebalance_start_meta_snapshot_for_id,
7    should_accept_rebalance_stats_update, should_pool_participate, stop_rebalance_meta_snapshot_for_id,
8    validate_init_rebalance_state,
9};
10use super::worker::{
11    rebalance_max_attempts, rebalance_meta_lock_error, resolve_load_rebalance_stats_update_result,
12    resolve_rebalance_deferred_last_error, resolve_rebalance_meta_load_result, resolve_rebalance_meta_save_result,
13    retry_rebalance_metadata_access,
14};
15use super::{
16    DiskStat, EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE, REBAL_META_NAME,
17    RebalStatus, RebalanceDeferKind, RebalanceInfo, RebalanceMeta, RebalanceStats, RebalanceStopPropagationRecord,
18    encode_rebalance_stop_propagation_record,
19};
20use crate::core::pools::{
21    PoolMeta, PoolRebalanceActivationFence, acquire_pool_activation_fleet_proof, acquire_pool_rebalance_activation_locks,
22    pool_meta_has_active_decommission,
23};
24use crate::error::{Error, Result};
25use crate::object_api::ObjectOptions;
26use crate::set_disk::get_lock_acquire_timeout;
27use crate::storage_api_contracts::{
28    admin::StorageAdminApi,
29    namespace::NamespaceLocking as StorageNamespaceLocking,
30    object::{EcstoreObjectIO, EcstoreObjectOperations},
31};
32use crate::store::ECStore;
33use rustfs_filemeta::FileInfo;
34use std::sync::Arc;
35use time::OffsetDateTime;
36use tracing::{debug, info};
37use uuid::Uuid;
38
39#[cfg(test)]
40static FAIL_NEXT_REBALANCE_ACTIVATION_SAVE: std::sync::Mutex<Option<String>> = std::sync::Mutex::new(None);
41
42#[cfg(test)]
43static REBALANCE_DISK_STATS_OVERRIDES: std::sync::OnceLock<std::sync::Mutex<std::collections::HashMap<Uuid, Vec<DiskStat>>>> =
44    std::sync::OnceLock::new();
45
46#[cfg(test)]
47pub(super) fn fail_next_rebalance_activation_save_for_test(rebalance_id: &str) {
48    *FAIL_NEXT_REBALANCE_ACTIVATION_SAVE
49        .lock()
50        .expect("rebalance activation save failure hook should not be poisoned") = Some(rebalance_id.to_string());
51}
52
53#[cfg(test)]
54fn set_rebalance_disk_stats_override_for_test(store_id: Uuid, disk_stats: Vec<DiskStat>) {
55    REBALANCE_DISK_STATS_OVERRIDES
56        .get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
57        .lock()
58        .expect("rebalance disk stats override should not be poisoned")
59        .insert(store_id, disk_stats);
60}
61
62#[cfg(test)]
63fn take_rebalance_disk_stats_override_for_test(store_id: Uuid) -> Option<Vec<DiskStat>> {
64    REBALANCE_DISK_STATS_OVERRIDES
65        .get_or_init(|| std::sync::Mutex::new(std::collections::HashMap::new()))
66        .lock()
67        .expect("rebalance disk stats override should not be poisoned")
68        .remove(&store_id)
69}
70
71fn ensure_rebalance_activation_pool_meta_allowed(meta: &PoolMeta) -> Result<()> {
72    if pool_meta_has_active_decommission(meta) {
73        return Err(Error::DecommissionAlreadyRunning);
74    }
75
76    Ok(())
77}
78
79pub(super) enum RebalanceWorkerActivationFence {
80    Ready(Box<PoolRebalanceActivationFence>),
81    NotStartedTerminal,
82}
83
84pub(super) struct RebalanceRunGuard {
85    _local_guard: tokio::sync::OwnedRwLockReadGuard<()>,
86    persisted_guard: rustfs_lock::NamespaceLockGuard,
87}
88
89#[cfg(any(test, feature = "test-util"))]
90struct RebalanceStopWaitProbeState {
91    expected_id: String,
92    attempted: std::sync::atomic::AtomicBool,
93    notify: tokio::sync::Notify,
94}
95
96#[cfg(any(test, feature = "test-util"))]
97static REBALANCE_STOP_WAIT_PROBES: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<RebalanceStopWaitProbeState>>>> =
98    std::sync::OnceLock::new();
99
100#[cfg(any(test, feature = "test-util"))]
101pub(super) struct RebalanceStopWaitProbe {
102    state: Arc<RebalanceStopWaitProbeState>,
103}
104
105#[cfg(any(test, feature = "test-util"))]
106impl RebalanceStopWaitProbe {
107    pub(super) fn install(expected_id: &str) -> Self {
108        let state = Arc::new(RebalanceStopWaitProbeState {
109            expected_id: expected_id.to_string(),
110            attempted: std::sync::atomic::AtomicBool::new(false),
111            notify: tokio::sync::Notify::new(),
112        });
113        REBALANCE_STOP_WAIT_PROBES
114            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
115            .lock()
116            .expect("rebalance stop wait probe should not be poisoned")
117            .push(Arc::clone(&state));
118        Self { state }
119    }
120
121    pub(super) async fn wait_until_attempted(&self) {
122        tokio::time::timeout(std::time::Duration::from_secs(30), async {
123            while !self.state.attempted.load(std::sync::atomic::Ordering::Acquire) {
124                self.state.notify.notified().await;
125            }
126        })
127        .await
128        .expect("rebalance stop should reach the deterministic activation wait probe");
129    }
130}
131
132#[cfg(any(test, feature = "test-util"))]
133impl Drop for RebalanceStopWaitProbe {
134    fn drop(&mut self) {
135        let mut probes = REBALANCE_STOP_WAIT_PROBES
136            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
137            .lock()
138            .expect("rebalance stop wait probe should not be poisoned");
139        probes.retain(|state| !Arc::ptr_eq(state, &self.state));
140    }
141}
142
143#[cfg(any(test, feature = "test-util"))]
144fn observe_rebalance_stop_wait_attempt(expected_id: Option<&str>) {
145    let probes = REBALANCE_STOP_WAIT_PROBES
146        .get_or_init(|| std::sync::Mutex::new(Vec::new()))
147        .lock()
148        .expect("rebalance stop wait probe should not be poisoned")
149        .clone();
150    for state in probes {
151        if expected_id == Some(state.expected_id.as_str()) {
152            state.attempted.store(true, std::sync::atomic::Ordering::Release);
153            state.notify.notify_one();
154        }
155    }
156}
157
158impl RebalanceRunGuard {
159    pub(super) fn ensure_held(&self, stage: &str) -> Result<()> {
160        #[cfg(test)]
161        let forced_lost = self
162            .persisted_guard
163            .lock_lost_signal()
164            .is_some_and(|signal| crate::object_api::namespace_lock_signal_test_fence_is_lost(&signal));
165        #[cfg(not(test))]
166        let forced_lost = false;
167        if self.persisted_guard.is_lock_lost() || forced_lost {
168            return Err(Error::other(format!("rebalance distributed run fence lost during {stage}")));
169        }
170        Ok(())
171    }
172
173    pub(super) fn lock_lost_signal(&self) -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
174        self.persisted_guard.lock_lost_signal()
175    }
176}
177
178async fn acquire_persisted_rebalance_run_guard<S>(
179    pool: Arc<S>,
180    expected_id: &str,
181    stage: &str,
182) -> Result<rustfs_lock::NamespaceLockGuard>
183where
184    S: EcstoreObjectIO
185        + EcstoreObjectOperations
186        + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
187{
188    let ns_lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
189    let guard = ns_lock
190        .get_read_lock(get_lock_acquire_timeout())
191        .await
192        .map_err(|err| rebalance_meta_lock_error(err, "read"))?;
193    let mut opts = ObjectOptions {
194        no_lock: true,
195        ..Default::default()
196    };
197    opts.add_namespace_lock_guard(&guard);
198    let object_info = pool
199        .get_object_info(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME, &opts)
200        .await;
201    let persisted_run_id = match object_info {
202        Ok(object_info) => {
203            let metadata = object_info.user_defined.as_ref();
204            let run_id = rustfs_utils::http::metadata_compat::get_consistent_str(
205                metadata,
206                rustfs_utils::http::metadata_compat::SUFFIX_REBALANCE_RUN_ID,
207            );
208            if run_id.is_none()
209                && rustfs_utils::http::metadata_compat::contains_key_str(
210                    metadata,
211                    rustfs_utils::http::metadata_compat::SUFFIX_REBALANCE_RUN_ID,
212                )
213            {
214                return Err(Error::other(format!("rebalance run marker is inconsistent during {stage}")));
215            }
216            run_id.map(str::to_string)
217        }
218        Err(err) if crate::error::is_err_object_not_found(&err) => None,
219        Err(err) => return Err(err),
220    };
221    if let Some(persisted_run_id) = persisted_run_id {
222        if persisted_run_id != expected_id {
223            return Err(Error::other(format!(
224                "stale rebalance run rejected during {stage}: expected {expected_id}, found {persisted_run_id}"
225            )));
226        }
227    } else {
228        // Metadata written before the fixed-size run marker was introduced
229        // remains readable during rolling upgrades.
230        let mut persisted = RebalanceMeta::new();
231        persisted.load_with_opts(pool, opts).await?;
232        ensure_rebalance_worker_active(Some(&persisted), expected_id, stage)?;
233    }
234    if guard.is_lock_lost() {
235        return Err(Error::other(format!("rebalance distributed run fence lost during {stage}")));
236    }
237    Ok(guard)
238}
239
240pub(super) async fn merge_and_save_rebalance_meta_no_lock<S>(
241    pool: Arc<S>,
242    local_snapshot: &RebalanceMeta,
243    stage: &str,
244    mut opts: ObjectOptions,
245    activation_fence: Option<&PoolRebalanceActivationFence>,
246    expected_id: Option<&str>,
247) -> Result<()>
248where
249    S: EcstoreObjectIO,
250{
251    let mut merged = RebalanceMeta::new();
252    match merged.load_with_opts(pool.clone(), opts.clone()).await {
253        Ok(()) => {
254            if let Some(expected_id) = expected_id {
255                ensure_rebalance_run_id(Some(&merged), expected_id, stage)?;
256            }
257            if merge_rebalance_meta(&mut merged, local_snapshot) == RebalanceMetaMergeOutcome::RejectedActiveConflict {
258                return Err(Error::RebalanceAlreadyRunning);
259            }
260        }
261        Err(Error::ConfigNotFound) => {
262            if expected_id.is_some() {
263                return Err(rebalance_metadata_not_initialized_error(stage));
264            }
265            merged = local_snapshot.clone();
266        }
267        Err(err) => return Err(Error::other(format!("rebalance meta load before save failed during {stage}: {err}"))),
268    }
269
270    if let Some(fence) = activation_fence {
271        fence.add_namespace_lock_fence(&mut opts);
272        fence.ensure_held()?;
273    }
274    #[cfg(test)]
275    let barrier_pool = pool.clone();
276    merged.save_with_opts(pool, opts).await?;
277    #[cfg(test)]
278    if let Some(fence) = activation_fence {
279        crate::core::pools::pause_pool_activation_after_durable_save(&barrier_pool, fence).await;
280    }
281    // With an activation fence, a successful save is the commit point. Lease
282    // loss after this point cannot make the durable activation uncommitted.
283    Ok(())
284}
285
286pub(super) fn ensure_rebalance_run_id(meta: Option<&RebalanceMeta>, expected_id: &str, stage: &str) -> Result<()> {
287    let Some(meta) = meta else {
288        return Err(rebalance_metadata_not_initialized_error(stage));
289    };
290    if meta.id != expected_id {
291        return Err(Error::other(format!(
292            "stale rebalance worker rejected during {stage}: expected {expected_id}, found {}",
293            meta.id
294        )));
295    }
296    Ok(())
297}
298
299pub(super) fn ensure_rebalance_worker_active(meta: Option<&RebalanceMeta>, expected_id: &str, stage: &str) -> Result<()> {
300    ensure_rebalance_run_id(meta, expected_id, stage)?;
301    let Some(meta) = meta else {
302        return Err(rebalance_metadata_not_initialized_error(stage));
303    };
304    if meta.stopped_at.is_some() || meta.stop_requested {
305        return Err(Error::OperationCanceled);
306    }
307    if meta
308        .cancel
309        .as_ref()
310        .is_some_and(tokio_util::sync::CancellationToken::is_cancelled)
311        || !is_rebalance_conflicting_with_decommission(meta)
312    {
313        return Err(Error::other(format!("inactive rebalance worker rejected during {stage}: {expected_id}")));
314    }
315    Ok(())
316}
317
318pub(super) fn validate_rebalance_disk_stats_coverage(disk_stats: &[DiskStat]) -> Result<()> {
319    for (idx, disk_stat) in disk_stats.iter().enumerate() {
320        if disk_stat.total_space == 0 {
321            return Err(Error::other(format!(
322                "rebalance storage info is incomplete: pool {idx} has no reported capacity"
323            )));
324        }
325    }
326
327    Ok(())
328}
329
330fn pool_rebalance_status_from_meta(meta: Option<&RebalanceMeta>, pool_index: usize) -> (RebalStatus, bool) {
331    meta.and_then(|meta| meta.pool_stats.get(pool_index))
332        .filter(|pool_stat| pool_stat.participating)
333        .map(|pool_stat| (pool_stat.info.status, pool_stat.info.stopping))
334        .unwrap_or_default()
335}
336
337fn merge_rebalance_status_refresh(current: &mut Option<RebalanceMeta>, persisted: RebalanceMeta) -> bool {
338    if persisted.id.is_empty() && persisted.pool_stats.is_empty() {
339        return clear_rebalance_status_refresh(current);
340    }
341
342    let before = current.clone();
343    match current.as_mut() {
344        Some(current_meta) => {
345            if merge_rebalance_meta(current_meta, &persisted) == RebalanceMetaMergeOutcome::RejectedActiveConflict
346                && !is_rebalance_actively_running(current_meta)
347            {
348                *current = Some(persisted);
349            }
350        }
351        None => {
352            *current = Some(persisted);
353        }
354    }
355
356    match (before.as_ref(), current.as_ref()) {
357        (None, None) => false,
358        (None, Some(_)) | (Some(_), None) => true,
359        (Some(before), Some(after)) => rebalance_movement_snapshot_changed(Some(before), after),
360    }
361}
362
363fn clear_rebalance_status_refresh(current: &mut Option<RebalanceMeta>) -> bool {
364    if current.as_ref().is_none_or(|meta| !is_rebalance_actively_running(meta)) {
365        current.take().is_some()
366    } else {
367        false
368    }
369}
370
371fn rebalance_movement_snapshot_changed(current: Option<&RebalanceMeta>, persisted: &RebalanceMeta) -> bool {
372    let Some(current) = current else {
373        return true;
374    };
375
376    current.id != persisted.id
377        || current.stopped_at != persisted.stopped_at
378        || current.pool_stats.len() != persisted.pool_stats.len()
379        || current
380            .pool_stats
381            .iter()
382            .zip(persisted.pool_stats.iter())
383            .any(|(current, persisted)| {
384                current.participating != persisted.participating
385                    || current.info.status != persisted.info.status
386                    || current.info.stopping != persisted.info.stopping
387            })
388}
389
390impl ECStore {
391    // Transition order is start_gate -> activation_gate -> movement gate -> rebalance_meta;
392    // the probe read below is released before the activation gate.
393    pub(super) async fn rebalance_activation_write_guard(
394        &self,
395        expected_id: Option<&str>,
396        stage: &str,
397    ) -> Result<Option<tokio::sync::OwnedRwLockWriteGuard<()>>> {
398        let activation_gate = {
399            let meta = self.rebalance_meta.read().await;
400            if let Some(expected_id) = expected_id {
401                ensure_rebalance_run_id(meta.as_ref(), expected_id, stage)?;
402            }
403            meta.as_ref().map(|meta| Arc::clone(&meta.activation_gate))
404        };
405        Ok(match activation_gate {
406            Some(gate) => Some(gate.write_owned().await),
407            None => None,
408        })
409    }
410
411    pub(super) async fn rebalance_run_guard(&self, expected_id: &str, stage: &str) -> Result<RebalanceRunGuard> {
412        // Runtime fence order is activation_gate -> rebalance.bin.
413        let activation_gate = {
414            let meta = self.rebalance_meta.read().await;
415            ensure_rebalance_worker_active(meta.as_ref(), expected_id, stage)?;
416            Arc::clone(
417                &meta
418                    .as_ref()
419                    .ok_or_else(|| rebalance_metadata_not_initialized_error(stage))?
420                    .activation_gate,
421            )
422        };
423        let guard = Arc::clone(&activation_gate).read_owned().await;
424        let meta = self.rebalance_meta.read().await;
425        ensure_rebalance_worker_active(meta.as_ref(), expected_id, stage)?;
426        let current_gate = &meta
427            .as_ref()
428            .ok_or_else(|| rebalance_metadata_not_initialized_error(stage))?
429            .activation_gate;
430        if !Arc::ptr_eq(&activation_gate, current_gate) {
431            return Err(Error::other(format!(
432                "stale rebalance activation gate rejected during {stage}: {expected_id}"
433            )));
434        }
435        drop(meta);
436        let pool = clone_first_arc(self.pools.as_slice(), "rebalance run fence: no pools available")?;
437        let persisted_guard = acquire_persisted_rebalance_run_guard(pool, expected_id, stage).await?;
438        Ok(RebalanceRunGuard {
439            _local_guard: guard,
440            persisted_guard,
441        })
442    }
443
444    pub(super) async fn save_rebalance_meta_with_merge<S>(
445        &self,
446        pool: Arc<S>,
447        local_snapshot: &RebalanceMeta,
448        stage: &str,
449    ) -> Result<()>
450    where
451        S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
452    {
453        self.save_rebalance_meta_with_merge_for_id(pool, local_snapshot, stage, None)
454            .await
455    }
456
457    pub(super) async fn save_rebalance_meta_for_id_with_merge<S>(
458        &self,
459        pool: Arc<S>,
460        local_snapshot: &RebalanceMeta,
461        stage: &str,
462        expected_id: &str,
463    ) -> Result<()>
464    where
465        S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
466    {
467        self.save_rebalance_meta_with_merge_for_id(pool, local_snapshot, stage, Some(expected_id))
468            .await
469    }
470
471    pub(super) async fn save_rebalance_meta_under_activation_fence<S>(
472        &self,
473        pool: Arc<S>,
474        local_snapshot: &RebalanceMeta,
475        stage: &str,
476        activation_fence: &PoolRebalanceActivationFence,
477        expected_id: &str,
478    ) -> Result<()>
479    where
480        S: EcstoreObjectIO,
481    {
482        #[cfg(test)]
483        {
484            let mut fail_id = FAIL_NEXT_REBALANCE_ACTIVATION_SAVE
485                .lock()
486                .expect("rebalance activation save failure hook should not be poisoned");
487            if fail_id.as_deref() == Some(local_snapshot.id.as_str()) {
488                fail_id.take();
489                return Err(Error::other("injected rebalance activation save failure"));
490            }
491        }
492        merge_and_save_rebalance_meta_no_lock(
493            pool,
494            local_snapshot,
495            stage,
496            ObjectOptions {
497                no_lock: true,
498                ..Default::default()
499            },
500            Some(activation_fence),
501            Some(expected_id),
502        )
503        .await
504    }
505
506    async fn save_rebalance_meta_with_merge_for_id<S>(
507        &self,
508        pool: Arc<S>,
509        local_snapshot: &RebalanceMeta,
510        stage: &str,
511        expected_id: Option<&str>,
512    ) -> Result<()>
513    where
514        S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
515    {
516        let ns_lock = pool.new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
517        let guard = retry_rebalance_metadata_access(None, rebalance_max_attempts(), || async {
518            ns_lock
519                .get_write_lock(get_lock_acquire_timeout())
520                .await
521                .map_err(|err| rebalance_meta_lock_error(err, "write"))
522        })
523        .await?;
524        let mut opts = ObjectOptions {
525            no_lock: true,
526            ..Default::default()
527        };
528        opts.add_namespace_lock_guard(&guard);
529
530        merge_and_save_rebalance_meta_no_lock(pool, local_snapshot, stage, opts, None, expected_id).await?;
531        if guard.is_lock_lost() {
532            return Err(Error::other("rebalance metadata lock lost during metadata commit"));
533        }
534        Ok(())
535    }
536
537    async fn save_rebalance_activation_meta_with_merge<S>(
538        &self,
539        pool: Arc<S>,
540        local_snapshot: &RebalanceMeta,
541        stage: &str,
542    ) -> Result<()>
543    where
544        S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
545    {
546        #[cfg(test)]
547        crate::core::pools::observe_pool_activation_start_attempt(crate::core::pools::PoolActivationStartKind::Rebalance);
548        let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
549        let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
550        pool_meta_guard.ensure_write_safe(stage)?;
551        let activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), fleet_proof).await?;
552        let pool_meta = self
553            .load_runtime_pool_meta_under_activation_fence(&mut pool_meta_guard, &activation_fence, stage)
554            .await?;
555        ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?;
556
557        merge_and_save_rebalance_meta_no_lock(
558            pool,
559            local_snapshot,
560            stage,
561            ObjectOptions {
562                no_lock: true,
563                ..Default::default()
564            },
565            Some(&activation_fence),
566            None,
567        )
568        .await
569    }
570
571    pub(super) async fn fence_rebalance_worker_activation<S>(
572        &self,
573        pool: Arc<S>,
574        expected_id: &str,
575    ) -> Result<RebalanceWorkerActivationFence>
576    where
577        S: EcstoreObjectIO + StorageNamespaceLocking<Error = Error, NamespaceLock = rustfs_lock::NamespaceLockWrapper>,
578    {
579        // Lock order: pool_meta_save_gate -> rebalance.bin -> pool.bin.
580        let mut pool_meta_guard = self.pool_meta_save_gate.lock().await;
581        pool_meta_guard.ensure_write_safe("rebalance worker activation")?;
582        // Classify the durable rebalance record while holding both namespace
583        // fences. A terminal record is a no-op and must not depend on the
584        // notification subsystem having published a fleet proof yet.
585        let mut activation_fence = acquire_pool_rebalance_activation_locks(pool.clone(), None).await?;
586        let pool_meta = self
587            .load_runtime_pool_meta_under_activation_fence(&mut pool_meta_guard, &activation_fence, "rebalance worker activation")
588            .await?;
589        ensure_rebalance_activation_pool_meta_allowed(&pool_meta)?;
590
591        let mut persisted = RebalanceMeta::new();
592        persisted
593            .load_with_opts(
594                pool,
595                ObjectOptions {
596                    no_lock: true,
597                    ..Default::default()
598                },
599            )
600            .await?;
601        if persisted.id != expected_id {
602            return Err(Error::other(format!(
603                "rebalance metadata changed before worker activation: expected {expected_id}, found {}",
604                persisted.id
605            )));
606        }
607
608        activation_fence.ensure_held()?;
609        if !crate::services::rebalance::rebalance_requires_worker_activation(&persisted) {
610            return Ok(RebalanceWorkerActivationFence::NotStartedTerminal);
611        }
612
613        // Active worker admission still requires the fail-closed fleet proof.
614        // Attach it immediately before the final fence validation so expiry or
615        // topology changes are checked again at every later commit boundary.
616        let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
617        activation_fence.set_fleet_proof(fleet_proof);
618        activation_fence.ensure_held()?;
619
620        Ok(RebalanceWorkerActivationFence::Ready(Box::new(activation_fence)))
621    }
622
623    #[tracing::instrument(skip_all)]
624    pub async fn load_rebalance_meta(&self) -> Result<()> {
625        let _start_guard = self.start_gate.lock().await;
626        self.load_rebalance_meta_under_start_gate().await
627    }
628
629    /// Cancels local admission before refreshing the persisted stop target under the start gate.
630    pub async fn prepare_rebalance_stop(&self) -> Result<Option<String>> {
631        let _start_guard = self.start_gate.lock().await;
632
633        let (local_stop_id, activation_gate) = {
634            let mut rebalance_meta = self.rebalance_meta.write().await;
635            if let Some(meta) = rebalance_meta.as_mut()
636                && is_rebalance_conflicting_with_decommission(meta)
637            {
638                meta.stop_requested = true;
639                meta.cancel
640                    .get_or_insert_with(tokio_util::sync::CancellationToken::new)
641                    .cancel();
642                #[cfg(any(test, feature = "test-util"))]
643                observe_rebalance_stop_wait_attempt(Some(meta.id.as_str()));
644                (Some(meta.id.clone()), Some(Arc::clone(&meta.activation_gate)))
645            } else {
646                (None, None)
647            }
648        };
649
650        // An in-flight entry already owns the activation read gate. Admission
651        // has been cancelled above, so do not wait on that same gate merely to
652        // refresh a stop target: the eventual stop operation will take the
653        // write gate after the entry drains. If the gate is currently free,
654        // retain the historical reload behaviour (which observes a worker that
655        // reached a terminal state just before this request).
656        if let Some(gate) = activation_gate
657            && gate.try_write_owned().is_err()
658        {
659            return Ok(local_stop_id);
660        }
661
662        self.load_rebalance_meta_under_start_gate().await?;
663
664        let mut rebalance_meta = self.rebalance_meta.write().await;
665        let Some(meta) = rebalance_meta.as_mut() else {
666            return Ok(None);
667        };
668        if meta.stopped_at.is_some() || (!is_rebalance_conflicting_with_decommission(meta) && !meta.stop_requested) {
669            return Ok(None);
670        }
671        if meta.id.is_empty() {
672            return Err(Error::other("active rebalance metadata has no activation id"));
673        }
674        meta.stop_requested = true;
675        meta.cancel
676            .get_or_insert_with(tokio_util::sync::CancellationToken::new)
677            .cancel();
678        Ok(Some(meta.id.clone()))
679    }
680
681    pub(crate) async fn load_rebalance_meta_under_start_gate(&self) -> Result<()> {
682        let mut meta = RebalanceMeta::new();
683        debug!(
684            event = EVENT_REBALANCE_STATE,
685            component = LOG_COMPONENT_ECSTORE,
686            subsystem = LOG_SUBSYSTEM_REBALANCE,
687            state = "metadata_loading",
688            "Loading rebalance metadata"
689        );
690        let pool = clone_first_arc(&self.pools, "rebalanceMeta: no pools available")?;
691        let _activation_guard = self.rebalance_activation_write_guard(None, "load rebalance metadata").await?;
692        let movement_gate = self.ctx.data_movement_operation_gate();
693        let _movement_guard = movement_gate.write().await;
694        let loaded = resolve_rebalance_meta_load_result(meta.load(pool).await)?;
695        if loaded {
696            let movement_changed = rebalance_movement_snapshot_changed(self.rebalance_meta.read().await.as_ref(), &meta);
697            {
698                let mut rebalance_meta = self.rebalance_meta.write().await;
699                if let Some(current) = rebalance_meta.as_ref()
700                    && current.id == meta.id
701                {
702                    meta.cancel = current.cancel.clone();
703                    meta.activation_gate = Arc::clone(&current.activation_gate);
704                    meta.stop_requested = current.stop_requested;
705                }
706                *rebalance_meta = Some(meta);
707
708                drop(rebalance_meta);
709            }
710
711            if movement_changed {
712                self.ctx.advance_data_movement_operation_epoch();
713            }
714            drop(_movement_guard);
715            resolve_load_rebalance_stats_update_result(self.update_rebalance_stats().await)?;
716            debug!(
717                event = EVENT_REBALANCE_STATE,
718                component = LOG_COMPONENT_ECSTORE,
719                subsystem = LOG_SUBSYSTEM_REBALANCE,
720                state = "metadata_loaded",
721                "Loaded rebalance metadata"
722            );
723        } else {
724            let movement_changed = self.rebalance_meta.read().await.is_some();
725            {
726                let mut rebalance_meta = self.rebalance_meta.write().await;
727                *rebalance_meta = None;
728            }
729            if movement_changed {
730                self.ctx.advance_data_movement_operation_epoch();
731            }
732            drop(_movement_guard);
733            debug!(
734                event = EVENT_REBALANCE_STATE,
735                component = LOG_COMPONENT_ECSTORE,
736                subsystem = LOG_SUBSYSTEM_REBALANCE,
737                state = "metadata_missing",
738                reason = "rebalance_not_started",
739                "Rebalance metadata not found"
740            );
741        }
742
743        Ok(())
744    }
745
746    #[tracing::instrument(skip_all)]
747    pub async fn refresh_rebalance_status_meta(&self) -> Result<()> {
748        let _start_guard = self.start_gate.lock().await;
749        let pool = clone_first_arc(&self.pools, "refresh_rebalance_status_meta: no pools available")?;
750        let mut persisted = RebalanceMeta::new();
751        let _activation_guard = self
752            .rebalance_activation_write_guard(None, "refresh rebalance metadata")
753            .await?;
754        let movement_gate = self.ctx.data_movement_operation_gate();
755        let _movement_guard = movement_gate.write().await;
756        let loaded = persisted.load(pool).await;
757        match loaded {
758            Ok(()) => {
759                let mut rebalance_meta = self.rebalance_meta.write().await;
760                if merge_rebalance_status_refresh(&mut rebalance_meta, persisted) {
761                    self.ctx.advance_data_movement_operation_epoch();
762                }
763            }
764            Err(Error::ConfigNotFound) => {
765                let mut rebalance_meta = self.rebalance_meta.write().await;
766                if clear_rebalance_status_refresh(&mut rebalance_meta) {
767                    self.ctx.advance_data_movement_operation_epoch();
768                }
769            }
770            Err(err) => {
771                return Err(Error::other(format!("rebalance metadata refresh failed during pool status: {err}")));
772            }
773        }
774
775        Ok(())
776    }
777
778    #[tracing::instrument(skip_all)]
779    pub async fn update_rebalance_stats(&self) -> Result<()> {
780        let mut ok = false;
781
782        let pool_stats = {
783            let rebalance_meta = self.rebalance_meta.read().await;
784            clone_rebalance_pool_stats(rebalance_meta.as_ref())?
785        };
786
787        debug!(
788            event = EVENT_REBALANCE_STATE,
789            component = LOG_COMPONENT_ECSTORE,
790            subsystem = LOG_SUBSYSTEM_REBALANCE,
791            pool_count = pool_stats.len(),
792            "Refreshing rebalance stats snapshot"
793        );
794
795        for i in 0..self.pools.len() {
796            if pool_stats.get(i).is_none() {
797                let mut rebalance_meta = self.rebalance_meta.write().await;
798                debug!(
799                    event = EVENT_REBALANCE_STATE,
800                    component = LOG_COMPONENT_ECSTORE,
801                    subsystem = LOG_SUBSYSTEM_REBALANCE,
802                    pool_index = i,
803                    state = "pool_stat_missing",
804                    "Adding missing rebalance pool stats entry"
805                );
806                if let Some(meta) = rebalance_meta.as_mut() {
807                    meta.pool_stats.push(RebalanceStats::default());
808                }
809                ok = true;
810                drop(rebalance_meta);
811            }
812        }
813
814        if ok {
815            debug!(
816                event = EVENT_REBALANCE_STATE,
817                component = LOG_COMPONENT_ECSTORE,
818                subsystem = LOG_SUBSYSTEM_REBALANCE,
819                state = "metadata_saving",
820                "Saving rebalance metadata after stats refresh"
821            );
822
823            let rebalance_meta = self.rebalance_meta.read().await;
824            if let Some(meta) = rebalance_meta.as_ref() {
825                let pool = clone_first_arc(&self.pools, "update_rebalance_stats: no pools available")?;
826                resolve_rebalance_meta_save_result(
827                    self.save_rebalance_meta_for_id_with_merge(pool, meta, "update_rebalance_stats", meta.id.as_str())
828                        .await,
829                    "update_rebalance_stats",
830                )?;
831            }
832        }
833
834        Ok(())
835    }
836
837    #[tracing::instrument(skip(self))]
838    pub async fn init_rebalance_meta(&self, bucktes: Vec<String>) -> Result<String> {
839        info!(
840            event = EVENT_REBALANCE_STATE,
841            component = LOG_COMPONENT_ECSTORE,
842            subsystem = LOG_SUBSYSTEM_REBALANCE,
843            state = "initializing",
844            bucket_count = bucktes.len(),
845            "Initializing rebalance metadata"
846        );
847        let si = StorageAdminApi::storage_info(self).await;
848
849        let mut disk_stats = vec![DiskStat::default(); self.pools.len()];
850
851        let mut total_cap = 0;
852        let mut total_free = 0;
853        for disk in si.disks.iter() {
854            if disk.pool_index < 0 || disk_stats.len() <= disk.pool_index as usize {
855                continue;
856            }
857
858            total_cap += disk.total_space;
859            total_free += disk.available_space;
860
861            disk_stats[disk.pool_index as usize].total_space += disk.total_space;
862            disk_stats[disk.pool_index as usize].available_space += disk.available_space;
863        }
864
865        #[cfg(test)]
866        if let Some(overridden) = take_rebalance_disk_stats_override_for_test(self.id) {
867            disk_stats = overridden;
868            total_cap = disk_stats.iter().map(|stat| stat.total_space).sum();
869            total_free = disk_stats.iter().map(|stat| stat.available_space).sum();
870        }
871
872        let percent_free_goal = percent_free_ratio(total_free, total_cap);
873        validate_rebalance_disk_stats_coverage(&disk_stats)?;
874
875        let mut pool_stats = Vec::with_capacity(self.pools.len());
876
877        let now = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
878
879        for disk_stat in disk_stats.iter() {
880            let mut pool_stat = RebalanceStats {
881                init_free_space: disk_stat.available_space,
882                init_capacity: disk_stat.total_space,
883                buckets: bucktes.clone(),
884                rebalanced_buckets: Vec::with_capacity(bucktes.len()),
885                ..Default::default()
886            };
887
888            if should_pool_participate(disk_stat.available_space, disk_stat.total_space, percent_free_goal) {
889                pool_stat.participating = true;
890                pool_stat.info = RebalanceInfo {
891                    start_time: Some(now),
892                    status: RebalStatus::Started,
893                    ..Default::default()
894                };
895            }
896
897            pool_stats.push(pool_stat);
898        }
899
900        let has_participating_pool = pool_stats.iter().any(|pool_stat| pool_stat.participating);
901        let meta = RebalanceMeta {
902            id: Uuid::new_v4().to_string(),
903            stopped_at: (!has_participating_pool).then_some(now),
904            percent_free_goal,
905            pool_stats,
906            ..Default::default()
907        };
908
909        let pool = clone_first_arc(&self.pools, "init_rebalance_meta: no pools available")?;
910        resolve_rebalance_meta_save_result(
911            self.save_rebalance_activation_meta_with_merge(pool, &meta, "init_rebalance_meta")
912                .await,
913            "init_rebalance_meta",
914        )?;
915
916        info!(
917            event = EVENT_REBALANCE_STATE,
918            component = LOG_COMPONENT_ECSTORE,
919            subsystem = LOG_SUBSYSTEM_REBALANCE,
920            state = "metadata_initialized",
921            bucket_count = bucktes.len(),
922            "Rebalance metadata initialized"
923        );
924
925        let id = meta.id.clone();
926
927        {
928            let mut rebalance_meta = self.rebalance_meta.write().await;
929            *rebalance_meta = Some(meta);
930            drop(rebalance_meta);
931        }
932
933        Ok(id)
934    }
935
936    #[tracing::instrument(skip(self, bucktes))]
937    pub async fn init_and_start_rebalance(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
938        let id = self.init_rebalance_start(bucktes).await?;
939        if let Err(start_err) = self.start_rebalance_for_id(&id).await {
940            if let Err(rollback_err) = self
941                .rollback_rebalance_start_without_worker_for_id(Some(&id), start_err.to_string())
942                .await
943            {
944                return Err(Error::other(format!(
945                    "failed to start rebalance after metadata initialized for {id}: {start_err}; rollback failed: {rollback_err}"
946                )));
947            }
948
949            return Err(Error::other(format!(
950                "failed to start rebalance after metadata initialized for {id}; local metadata was finalized as failed: {start_err}"
951            )));
952        }
953
954        Ok(id)
955    }
956
957    #[tracing::instrument(skip(self, bucktes))]
958    pub async fn init_rebalance_start(self: &Arc<Self>, bucktes: Vec<String>) -> Result<String> {
959        let _start_guard = self.start_gate.lock().await;
960        {
961            let decommission_running = self.is_decommission_running().await;
962            let rebalance_meta = self.rebalance_meta.read().await;
963            validate_init_rebalance_state(decommission_running, rebalance_meta.as_ref())?;
964        }
965        let _activation_guard = self
966            .rebalance_activation_write_guard(None, "initialize replacement rebalance")
967            .await?;
968        let movement_gate = self.ctx.data_movement_operation_gate();
969        let _movement_guard = movement_gate.write().await;
970
971        let id = self.init_rebalance_meta(bucktes).await?;
972        self.ctx.advance_data_movement_operation_epoch();
973        Ok(id)
974    }
975
976    #[tracing::instrument(skip(self))]
977    pub async fn start_rebalance_for_id(self: &Arc<Self>, expected_id: &str) -> Result<()> {
978        let _start_guard = self.start_gate.lock().await;
979        let _activation_guard = self
980            .rebalance_activation_write_guard(Some(expected_id), "start rebalance")
981            .await?;
982        let movement_gate = self.ctx.data_movement_operation_gate();
983        let _movement_guard = movement_gate.write().await;
984
985        {
986            let rebalance_meta = self.rebalance_meta.read().await;
987            let Some(meta) = rebalance_meta.as_ref() else {
988                return Err(Error::ConfigNotFound);
989            };
990            if meta.id != expected_id {
991                return Err(Error::other(format!(
992                    "rebalance metadata changed before start: expected {expected_id}, found {}",
993                    meta.id
994                )));
995            }
996            if meta.stopped_at.is_some() {
997                if !is_rebalance_conflicting_with_decommission(meta) {
998                    debug!(
999                        event = EVENT_REBALANCE_STATE,
1000                        component = LOG_COMPONENT_ECSTORE,
1001                        subsystem = LOG_SUBSYSTEM_REBALANCE,
1002                        state = "start_skipped",
1003                        reason = "not_started_terminal",
1004                        rebalance_id = %expected_id,
1005                        "Skipped rebalance start because metadata is already terminal"
1006                    );
1007                    return Ok(());
1008                }
1009                return Err(Error::other(format!("rebalance {expected_id} was stopped before start")));
1010            }
1011        }
1012
1013        if self.start_rebalance_under_gate().await? {
1014            self.ctx.advance_data_movement_operation_epoch();
1015        }
1016        Ok(())
1017    }
1018
1019    pub async fn rollback_rebalance_start_for_id(self: &Arc<Self>, expected_id: Option<&str>, start_error: String) -> Result<()> {
1020        self.rollback_rebalance_start_without_worker_for_id(expected_id, start_error)
1021            .await
1022    }
1023
1024    #[tracing::instrument(skip(self, fi))]
1025    pub async fn update_pool_stats(&self, pool_index: usize, bucket: String, fi: &FileInfo) -> Result<()> {
1026        self.update_pool_stats_batch(pool_index, bucket, &[fi]).await
1027    }
1028
1029    #[tracing::instrument(skip(self, versions))]
1030    pub async fn update_pool_stats_batch(&self, pool_index: usize, bucket: String, versions: &[&FileInfo]) -> Result<()> {
1031        self.update_pool_stats_batch_for_id(pool_index, bucket, versions, None).await
1032    }
1033
1034    pub(super) async fn update_pool_stats_batch_for_rebalance(
1035        &self,
1036        pool_index: usize,
1037        bucket: String,
1038        versions: &[&FileInfo],
1039        expected_id: &str,
1040    ) -> Result<()> {
1041        self.update_pool_stats_batch_for_id(pool_index, bucket, versions, Some(expected_id))
1042            .await
1043    }
1044
1045    async fn update_pool_stats_batch_for_id(
1046        &self,
1047        pool_index: usize,
1048        bucket: String,
1049        versions: &[&FileInfo],
1050        expected_id: Option<&str>,
1051    ) -> Result<()> {
1052        if versions.is_empty() {
1053            return Ok(());
1054        }
1055
1056        let mut rebalance_meta = self.rebalance_meta.write().await;
1057        if let Some(expected_id) = expected_id {
1058            ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "update rebalance pool stats")?;
1059        }
1060        if let Some(meta) = rebalance_meta.as_mut() {
1061            if !should_accept_rebalance_stats_update(meta, pool_index) {
1062                return Ok(());
1063            }
1064
1065            if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
1066                pool_stat.update_batch(bucket, versions);
1067            }
1068        }
1069
1070        Ok(())
1071    }
1072
1073    #[tracing::instrument(skip(self))]
1074    pub async fn next_rebal_bucket(&self, pool_index: usize, expected_id: &str) -> Result<Option<String>> {
1075        let rebalance_meta = self.rebalance_meta.read().await;
1076        ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "next rebalance bucket")?;
1077        debug!(
1078            event = EVENT_REBALANCE_BUCKET,
1079            component = LOG_COMPONENT_ECSTORE,
1080            subsystem = LOG_SUBSYSTEM_REBALANCE,
1081            pool_index,
1082            has_meta = rebalance_meta.is_some(),
1083            state = "next_bucket_lookup",
1084            "Rebalance next bucket lookup"
1085        );
1086        resolve_next_rebalance_bucket(rebalance_meta.as_ref(), pool_index)
1087    }
1088
1089    #[tracing::instrument(skip(self))]
1090    pub async fn bucket_rebalance_done(&self, pool_index: usize, bucket: String, expected_id: &str) -> Result<()> {
1091        let mut rebalance_meta = self.rebalance_meta.write().await;
1092        ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "mark rebalance bucket done")?;
1093        mark_rebalance_bucket_done(rebalance_meta.as_mut(), pool_index, &bucket)
1094    }
1095
1096    pub(super) async fn record_rebalance_cleanup_warning(
1097        &self,
1098        pool_index: usize,
1099        bucket: &str,
1100        object: &str,
1101        message: String,
1102        expected_id: &str,
1103    ) -> Result<()> {
1104        let mut rebalance_meta = self.rebalance_meta.write().await;
1105        ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "record rebalance cleanup warning")?;
1106        record_rebalance_cleanup_warning_in_meta(
1107            rebalance_meta.as_mut(),
1108            pool_index,
1109            bucket,
1110            object,
1111            message,
1112            OffsetDateTime::now_utc(),
1113        )
1114    }
1115
1116    pub(super) async fn defer_rebalance_bucket(
1117        &self,
1118        pool_index: usize,
1119        bucket: String,
1120        last_error: String,
1121        expected_id: &str,
1122        kind: RebalanceDeferKind,
1123    ) -> Result<()> {
1124        let mut rebalance_meta = self.rebalance_meta.write().await;
1125        ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "defer rebalance bucket")?;
1126        let Some(meta) = rebalance_meta.as_mut() else {
1127            return Err(rebalance_metadata_not_initialized_error("defer rebalance bucket"));
1128        };
1129        let pool_count = meta.pool_stats.len();
1130        ensure_valid_rebalance_pool_index(pool_count, pool_index)?;
1131        let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) else {
1132            return Err(invalid_rebalance_pool_index_error(pool_index, pool_count));
1133        };
1134
1135        defer_bucket_in_rebalance_queue(pool_stat, &bucket)?;
1136        let pending_entry_defer = pool_stat.info.last_error.clone();
1137        pool_stat.info.last_error = resolve_rebalance_deferred_last_error(kind, pending_entry_defer.as_deref(), &last_error);
1138        meta.last_refreshed_at = Some(OffsetDateTime::now_utc());
1139        Ok(())
1140    }
1141
1142    pub async fn is_rebalance_started(&self) -> bool {
1143        let rebalance_meta = self.rebalance_meta.read().await;
1144        if let Some(meta) = rebalance_meta.as_ref() {
1145            meta.pool_stats.iter().enumerate().for_each(|(i, v)| {
1146                debug!(
1147                    event = EVENT_REBALANCE_STATE,
1148                    component = LOG_COMPONENT_ECSTORE,
1149                    subsystem = LOG_SUBSYSTEM_REBALANCE,
1150                    pool_index = i,
1151                    participating = v.participating,
1152                    status = ?v.info.status,
1153                    state = "status_inspected",
1154                    "Rebalance status inspected"
1155                );
1156            });
1157
1158            let started = is_rebalance_conflicting_with_decommission(meta);
1159            if started {
1160                debug!(
1161                    event = EVENT_REBALANCE_STATE,
1162                    component = LOG_COMPONENT_ECSTORE,
1163                    subsystem = LOG_SUBSYSTEM_REBALANCE,
1164                    state = "running",
1165                    "Rebalance is running"
1166                );
1167                return true;
1168            }
1169        }
1170
1171        debug!(
1172            event = EVENT_REBALANCE_STATE,
1173            component = LOG_COMPONENT_ECSTORE,
1174            subsystem = LOG_SUBSYSTEM_REBALANCE,
1175            state = "not_running",
1176            "Rebalance is not running"
1177        );
1178        false
1179    }
1180
1181    pub async fn is_rebalance_conflicting_with_decommission(&self) -> bool {
1182        let rebalance_meta = self.rebalance_meta.read().await;
1183        rebalance_meta
1184            .as_ref()
1185            .is_some_and(is_rebalance_conflicting_with_decommission)
1186    }
1187
1188    pub async fn is_pool_rebalancing(&self, pool_index: usize) -> bool {
1189        let rebalance_meta = self.rebalance_meta.read().await;
1190        if let Some(ref meta) = *rebalance_meta {
1191            if meta.stopped_at.is_some() {
1192                return false;
1193            }
1194
1195            if let Some(pool_stat) = meta.pool_stats.get(pool_index) {
1196                return pool_stat.participating && pool_stat.info.status == RebalStatus::Started;
1197            }
1198        }
1199
1200        false
1201    }
1202
1203    pub async fn pool_rebalance_status(&self, pool_index: usize) -> (RebalStatus, bool) {
1204        let rebalance_meta = self.rebalance_meta.read().await;
1205        pool_rebalance_status_from_meta(rebalance_meta.as_ref(), pool_index)
1206    }
1207
1208    pub async fn current_rebalance_id(&self) -> Option<String> {
1209        let rebalance_meta = self.rebalance_meta.read().await;
1210        rebalance_meta
1211            .as_ref()
1212            .and_then(|meta| (!meta.id.is_empty()).then(|| meta.id.clone()))
1213    }
1214
1215    pub async fn cancel_rebalance_admission_for_id(&self, expected_id: &str) -> Result<()> {
1216        let _start_guard = self.start_gate.lock().await;
1217        let mut rebalance_meta = self.rebalance_meta.write().await;
1218        ensure_rebalance_run_id(rebalance_meta.as_ref(), expected_id, "cancel rebalance admission")?;
1219        let meta = rebalance_meta
1220            .as_mut()
1221            .ok_or_else(|| rebalance_metadata_not_initialized_error("cancel rebalance admission"))?;
1222        if meta.stopped_at.is_some() || (!is_rebalance_conflicting_with_decommission(meta) && !meta.stop_requested) {
1223            return Err(Error::other(format!(
1224                "inactive rebalance rejected while cancelling admission: {expected_id}"
1225            )));
1226        }
1227        meta.stop_requested = true;
1228        meta.cancel
1229            .get_or_insert_with(tokio_util::sync::CancellationToken::new)
1230            .cancel();
1231        Ok(())
1232    }
1233
1234    #[tracing::instrument(skip(self))]
1235    pub async fn stop_rebalance(self: &Arc<Self>) -> Result<()> {
1236        self.stop_rebalance_for_id(None).await
1237    }
1238
1239    #[tracing::instrument(skip(self))]
1240    pub async fn stop_rebalance_for_id(self: &Arc<Self>, expected_id: Option<&str>) -> Result<()> {
1241        let _start_guard = self.start_gate.lock().await;
1242        let activation_gate = {
1243            let mut rebalance_meta = self.rebalance_meta.write().await;
1244            if let Some(expected_id) = expected_id {
1245                ensure_rebalance_run_id(rebalance_meta.as_ref(), expected_id, "stop rebalance")?;
1246            }
1247            rebalance_meta.as_mut().map(|meta| {
1248                meta.stop_requested |= is_rebalance_conflicting_with_decommission(meta);
1249                let cancel = meta.cancel.get_or_insert_with(tokio_util::sync::CancellationToken::new);
1250                cancel.cancel();
1251                Arc::clone(&meta.activation_gate)
1252            })
1253        };
1254        let _activation_guard = match activation_gate {
1255            Some(gate) => {
1256                #[cfg(any(test, feature = "test-util"))]
1257                observe_rebalance_stop_wait_attempt(expected_id);
1258                Some(gate.write_owned().await)
1259            }
1260            None => None,
1261        };
1262        let movement_gate = self.ctx.data_movement_operation_gate();
1263        let _movement_guard = movement_gate.write().await;
1264        let stopped_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
1265        let (previous_meta, meta_to_save) = {
1266            let mut rebalance_meta = self.rebalance_meta.write().await;
1267            let previous_meta = rebalance_meta.clone();
1268            let meta_to_save = stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), stopped_at, expected_id)?;
1269            (previous_meta, meta_to_save)
1270        };
1271
1272        if let Some(meta_to_save) = meta_to_save {
1273            let pool = clone_first_arc(self.pools.as_slice(), "stop_rebalance: no pools available")?;
1274            let save_result = resolve_rebalance_meta_save_result(
1275                self.save_rebalance_meta_for_id_with_merge(pool, &meta_to_save, "stop_rebalance", meta_to_save.id.as_str())
1276                    .await,
1277                "stop_rebalance",
1278            );
1279            if let Err(err) = save_result {
1280                *self.rebalance_meta.write().await = previous_meta;
1281                return Err(err);
1282            }
1283            self.ctx.advance_data_movement_operation_epoch();
1284        }
1285
1286        Ok(())
1287    }
1288
1289    async fn rollback_rebalance_start_without_worker_for_id(
1290        self: &Arc<Self>,
1291        expected_id: Option<&str>,
1292        start_error: String,
1293    ) -> Result<()> {
1294        let _start_guard = self.start_gate.lock().await;
1295        let _activation_guard = self
1296            .rebalance_activation_write_guard(expected_id, "rollback rebalance start")
1297            .await?;
1298        let movement_gate = self.ctx.data_movement_operation_gate();
1299        let _movement_guard = movement_gate.write().await;
1300        let failed_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
1301        let meta_to_save = {
1302            let mut rebalance_meta = self.rebalance_meta.write().await;
1303            rollback_rebalance_start_meta_snapshot_for_id(rebalance_meta.as_mut(), failed_at, expected_id, start_error)
1304        };
1305
1306        if let Some(meta_to_save) = meta_to_save {
1307            let pool = clone_first_arc(self.pools.as_slice(), "rollback_rebalance_start: no pools available")?;
1308            resolve_rebalance_meta_save_result(
1309                self.save_rebalance_meta_for_id_with_merge(
1310                    pool,
1311                    &meta_to_save,
1312                    "rollback_rebalance_start",
1313                    meta_to_save.id.as_str(),
1314                )
1315                .await,
1316                "rollback_rebalance_start",
1317            )?;
1318            self.ctx.advance_data_movement_operation_epoch();
1319        }
1320
1321        Ok(())
1322    }
1323
1324    pub async fn record_rebalance_stop_propagation(
1325        self: &Arc<Self>,
1326        expected_id: &str,
1327        record: RebalanceStopPropagationRecord,
1328    ) -> Result<()> {
1329        if !record.has_failures() {
1330            return Ok(());
1331        }
1332
1333        let _start_guard = self.start_gate.lock().await;
1334        let _activation_guard = self
1335            .rebalance_activation_write_guard(Some(expected_id), "record rebalance stop propagation")
1336            .await?;
1337        let movement_gate = self.ctx.data_movement_operation_gate();
1338        let _movement_guard = movement_gate.write().await;
1339        let encoded_error = encode_rebalance_stop_propagation_record(&record);
1340        let meta_to_save = {
1341            let mut rebalance_meta = self.rebalance_meta.write().await;
1342            ensure_rebalance_run_id(rebalance_meta.as_ref(), expected_id, "record rebalance stop propagation")?;
1343            record_rebalance_stop_propagation_snapshot(rebalance_meta.as_mut(), encoded_error, OffsetDateTime::now_utc())
1344        };
1345
1346        if let Some(meta_to_save) = meta_to_save {
1347            let pool = clone_first_arc(self.pools.as_slice(), "record_rebalance_stop_propagation: no pools available")?;
1348            resolve_rebalance_meta_save_result(
1349                self.save_rebalance_meta_for_id_with_merge(pool, &meta_to_save, "record_rebalance_stop_propagation", expected_id)
1350                    .await,
1351                "record_rebalance_stop_propagation",
1352            )?;
1353            self.ctx.advance_data_movement_operation_epoch();
1354        }
1355
1356        Ok(())
1357    }
1358}
1359
1360#[cfg(test)]
1361mod tests {
1362    use super::*;
1363    use crate::config::com::delete_config;
1364    use crate::core::pools::{
1365        DecommissionErasureLayout, DecommissionPoolCapacityInfo, POOL_META_NAME, PoolActivationDurableSaveBarrier,
1366        PoolActivationStartKind, PoolActivationStartProbe, PoolMetaWriteState, persist_pool_meta_identity_for_startup,
1367        set_decommission_capacity_info_overrides_for_test,
1368    };
1369    use crate::object_api::NamespaceLockFence;
1370    use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause, hermetic_set_disks_isolated};
1371
1372    #[tokio::test]
1373    async fn rebalance_status_read_recovers_after_metadata_lock_timeout() {
1374        let id = "rebalance-status-lock-retry";
1375        let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1376            id: id.to_string(),
1377            pool_stats: vec![RebalanceStats::default()],
1378            ..Default::default()
1379        })
1380        .await;
1381        let ns_lock = store.pools[0]
1382            .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME)
1383            .await
1384            .expect("metadata lock should be available");
1385        let writer = ns_lock
1386            .get_write_lock(get_lock_acquire_timeout())
1387            .await
1388            .expect("hold metadata writer");
1389        let retried = Arc::new(tokio::sync::Notify::new());
1390        let refresh = super::super::worker::REBALANCE_METADATA_RETRY_PROBE
1391            .scope(Arc::clone(&retried), store.refresh_rebalance_status_meta());
1392        tokio::pin!(refresh);
1393        tokio::time::timeout(std::time::Duration::from_secs(30), async {
1394            tokio::select! {
1395                _ = retried.notified() => {},
1396                result = &mut refresh => panic!("status read must retry its contended metadata lock: {result:?}"),
1397            }
1398        })
1399        .await
1400        .expect("status read should encounter a real lock timeout");
1401        drop(writer);
1402        tokio::time::timeout(std::time::Duration::from_secs(30), refresh)
1403            .await
1404            .expect("status refresh should finish after the writer releases")
1405            .expect("transient metadata contention must not fail status refresh");
1406        assert_eq!(
1407            store
1408                .rebalance_meta
1409                .read()
1410                .await
1411                .as_ref()
1412                .expect("metadata must remain present")
1413                .id,
1414            id
1415        );
1416    }
1417
1418    #[tokio::test]
1419    async fn rebalance_stats_save_recovers_after_metadata_lock_timeout() {
1420        let id = "rebalance-stats-lock-retry";
1421        let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1422            id: id.to_string(),
1423            pool_stats: vec![RebalanceStats::default()],
1424            ..Default::default()
1425        })
1426        .await;
1427        store
1428            .rebalance_meta
1429            .write()
1430            .await
1431            .as_mut()
1432            .expect("local metadata must exist")
1433            .pool_stats[0]
1434            .num_objects = 7;
1435        let ns_lock = store.pools[0]
1436            .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME)
1437            .await
1438            .expect("metadata lock should be available");
1439        let reader = ns_lock
1440            .get_read_lock(get_lock_acquire_timeout())
1441            .await
1442            .expect("hold a migration read fence");
1443        let retried = Arc::new(tokio::sync::Notify::new());
1444        let save = super::super::worker::REBALANCE_METADATA_RETRY_PROBE.scope(
1445            Arc::clone(&retried),
1446            store.save_rebalance_stats_for_id(0, super::super::RebalSaveOpt::Stats, id),
1447        );
1448        tokio::pin!(save);
1449        tokio::time::timeout(std::time::Duration::from_secs(30), async {
1450            tokio::select! {
1451                _ = retried.notified() => {},
1452                result = &mut save => panic!("stats save must retry its contended metadata lock: {result:?}"),
1453            }
1454        })
1455        .await
1456        .expect("stats save should encounter a real lock timeout");
1457        drop(reader);
1458        tokio::time::timeout(std::time::Duration::from_secs(30), save)
1459            .await
1460            .expect("stats save should finish after the migration read fence releases")
1461            .expect("transient metadata contention must not lose stats");
1462        let mut persisted = RebalanceMeta::new();
1463        persisted
1464            .load(Arc::clone(&store.pools[0]))
1465            .await
1466            .expect("persisted stats must be readable");
1467        assert_eq!(persisted.id, id);
1468        assert_eq!(persisted.pool_stats[0].num_objects, 7);
1469    }
1470
1471    async fn persist_initialized_identity_then_remove_pool_meta(store: &Arc<ECStore>) {
1472        let deployment_id = store
1473            .ctx
1474            .deployment_id()
1475            .expect("test store should have a deployment identity");
1476        let mut write_state = PoolMetaWriteState::for_startup(deployment_id, false);
1477        persist_pool_meta_identity_for_startup(store.pools.clone(), &mut write_state, true)
1478            .await
1479            .expect("initialized pool metadata identity should persist");
1480        *store.pool_meta_save_gate.lock().await = write_state;
1481        for pool in &store.pools {
1482            delete_config(pool.clone(), POOL_META_NAME)
1483                .await
1484                .expect("every pool metadata replica should be removed");
1485        }
1486    }
1487
1488    async fn assert_activation_locks_released(store: &Arc<ECStore>) {
1489        let fleet_proof = acquire_pool_activation_fleet_proof(&store.ctx)
1490            .await
1491            .expect("fleet proof should remain available");
1492        tokio::time::timeout(
1493            std::time::Duration::from_secs(5),
1494            acquire_pool_rebalance_activation_locks(store.pools[0].clone(), fleet_proof),
1495        )
1496        .await
1497        .expect("a rejected activation must release namespace fences promptly")
1498        .expect("a rejected activation must release both namespace fences");
1499    }
1500
1501    #[tokio::test]
1502    async fn rebalance_stop_wait_probe_matches_run_id() {
1503        let probe = RebalanceStopWaitProbe::install("rebalance-stop-current");
1504
1505        observe_rebalance_stop_wait_attempt(Some("rebalance-stop-stale"));
1506        assert!(!probe.state.attempted.load(std::sync::atomic::Ordering::Acquire));
1507
1508        observe_rebalance_stop_wait_attempt(Some("rebalance-stop-current"));
1509        probe.wait_until_attempted().await;
1510    }
1511
1512    #[test]
1513    fn rebalance_stop_classification_checks_identity_and_explicit_intent() {
1514        let mut meta = RebalanceMeta {
1515            id: "current".to_string(),
1516            cancel: Some(tokio_util::sync::CancellationToken::new()),
1517            pool_stats: vec![RebalanceStats {
1518                participating: true,
1519                info: RebalanceInfo {
1520                    status: RebalStatus::Started,
1521                    ..Default::default()
1522                },
1523                ..Default::default()
1524            }],
1525            ..Default::default()
1526        };
1527        ensure_rebalance_worker_active(Some(&meta), "current", "test").expect("active worker");
1528        meta.cancel.as_ref().unwrap().cancel();
1529        assert!(
1530            !matches!(
1531                ensure_rebalance_worker_active(Some(&meta), "current", "test"),
1532                Err(Error::OperationCanceled)
1533            ),
1534            "a sibling failure is not an operator stop"
1535        );
1536        meta.stop_requested = true;
1537        assert!(matches!(
1538            ensure_rebalance_worker_active(Some(&meta), "current", "test"),
1539            Err(Error::OperationCanceled)
1540        ));
1541        assert!(
1542            !matches!(ensure_rebalance_worker_active(Some(&meta), "old", "test"), Err(Error::OperationCanceled)),
1543            "stale identity remains a failure even during stop"
1544        );
1545        assert!(!matches!(
1546            ensure_rebalance_worker_active(None, "current", "test"),
1547            Err(Error::OperationCanceled)
1548        ));
1549    }
1550
1551    #[tokio::test]
1552    async fn rebalance_stop_intent_does_not_survive_replacement_run_reload() {
1553        let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1554            id: "replacement".to_string(),
1555            pool_stats: vec![RebalanceStats {
1556                participating: true,
1557                info: RebalanceInfo {
1558                    status: RebalStatus::Completed,
1559                    ..Default::default()
1560                },
1561                ..Default::default()
1562            }],
1563            ..Default::default()
1564        })
1565        .await;
1566        let previous_gate = {
1567            let mut meta = store.rebalance_meta.write().await;
1568            let meta = meta.as_mut().unwrap();
1569            meta.id = "previous".to_string();
1570            meta.stop_requested = true;
1571            let cancel = tokio_util::sync::CancellationToken::new();
1572            cancel.cancel();
1573            meta.cancel = Some(cancel);
1574            Arc::clone(&meta.activation_gate)
1575        };
1576        store.load_rebalance_meta().await.expect("reload replacement run");
1577        let meta = store.rebalance_meta.read().await;
1578        let meta = meta.as_ref().unwrap();
1579        assert_eq!(meta.id, "replacement");
1580        assert!(!meta.stop_requested);
1581        assert!(meta.cancel.is_none());
1582        assert!(!Arc::ptr_eq(&previous_gate, &meta.activation_gate));
1583    }
1584
1585    #[tokio::test]
1586    async fn cancel_rebalance_admission_is_id_checked_and_idempotent() {
1587        let rebalance_id = "rebalance-admission-current";
1588        let cancel = tokio_util::sync::CancellationToken::new();
1589        let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1590            id: rebalance_id.to_string(),
1591            percent_free_goal: 1.0,
1592            cancel: Some(cancel.clone()),
1593            pool_stats: vec![RebalanceStats {
1594                participating: true,
1595                init_capacity: 100,
1596                info: RebalanceInfo {
1597                    start_time: Some(OffsetDateTime::now_utc()),
1598                    status: RebalStatus::Started,
1599                    ..Default::default()
1600                },
1601                ..Default::default()
1602            }],
1603            ..Default::default()
1604        })
1605        .await;
1606
1607        let error = store
1608            .cancel_rebalance_admission_for_id("rebalance-admission-replacement")
1609            .await
1610            .expect_err("a stale admin stop must not cancel the current run");
1611        assert!(error.to_string().contains("expected rebalance-admission-replacement"));
1612        assert!(!cancel.is_cancelled());
1613
1614        store
1615            .cancel_rebalance_admission_for_id(rebalance_id)
1616            .await
1617            .expect("the current run admission should close");
1618        store
1619            .cancel_rebalance_admission_for_id(rebalance_id)
1620            .await
1621            .expect("retrying admission cancellation should be idempotent");
1622        assert!(cancel.is_cancelled());
1623        let err = store
1624            .update_pool_stats_batch_for_rebalance(0, "bucket".to_string(), &[&FileInfo::default()], rebalance_id)
1625            .await
1626            .expect_err("stop racing with a final stats update must cancel that update");
1627        assert!(matches!(err, Error::OperationCanceled), "operator stop lost its cancellation type: {err}");
1628    }
1629
1630    #[tokio::test]
1631    async fn prepare_rebalance_stop_preserves_intent_when_worker_stops_before_reload() {
1632        let id = "stop-worker-before-reload";
1633        let cancel = tokio_util::sync::CancellationToken::new();
1634        let (_temp_dirs, store) = crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta {
1635            id: id.to_string(),
1636            cancel: Some(cancel.clone()),
1637            pool_stats: vec![RebalanceStats {
1638                participating: true,
1639                info: RebalanceInfo {
1640                    status: RebalStatus::Stopped,
1641                    ..Default::default()
1642                },
1643                ..Default::default()
1644            }],
1645            ..Default::default()
1646        })
1647        .await;
1648        let gate = {
1649            let mut meta = store.rebalance_meta.write().await;
1650            let meta = meta.as_mut().expect("local rebalance metadata");
1651            meta.pool_stats[0].info.status = RebalStatus::Started;
1652            Arc::clone(&meta.activation_gate)
1653        };
1654        assert_eq!(
1655            store.prepare_rebalance_stop().await.expect("prepare the same run stop"),
1656            Some(id.to_string())
1657        );
1658        {
1659            let meta = store.rebalance_meta.read().await;
1660            let meta = meta.as_ref().expect("reloaded stop target");
1661            assert!(Arc::ptr_eq(&gate, &meta.activation_gate), "reload must retain the drained run's gate");
1662            assert!(meta.cancel.as_ref().is_some_and(|token| token.is_cancelled()));
1663        }
1664        store
1665            .stop_rebalance_for_id(Some(id))
1666            .await
1667            .expect("finish the stop after the worker's terminal event");
1668        store
1669            .load_rebalance_meta()
1670            .await
1671            .expect("reload the acknowledged durable stop");
1672        let meta = store.rebalance_meta.read().await;
1673        let meta = meta.as_ref().expect("durable stopped metadata");
1674        assert!(meta.stopped_at.is_some(), "a successful stop must retain its durable timestamp");
1675        assert_eq!(meta.pool_stats[0].info.status, RebalStatus::Stopped);
1676    }
1677
1678    #[tokio::test]
1679    #[serial_test::serial]
1680    async fn equal_free_ratio_admin_no_participant_rebalance_succeeds_and_persists_terminal_generation_after_restart() {
1681        let (_temp_dirs, store, restarted) =
1682            crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
1683        let movement_floor = OffsetDateTime::from_unix_timestamp(4_100_000_000).expect("future test timestamp should be valid");
1684        *store.rebalance_meta.write().await = Some(RebalanceMeta {
1685            id: "previous-terminal-rebalance".to_string(),
1686            stopped_at: Some(movement_floor),
1687            ..Default::default()
1688        });
1689        set_rebalance_disk_stats_override_for_test(
1690            store.id,
1691            vec![
1692                DiskStat {
1693                    total_space: 100,
1694                    available_space: 50,
1695                },
1696                DiskStat {
1697                    total_space: 100,
1698                    available_space: 50,
1699                },
1700            ],
1701        );
1702
1703        let rebalance_id = store
1704            .init_and_start_rebalance(vec!["equal-ratio-no-op".to_string()])
1705            .await
1706            .expect("equal free ratio admin rebalance should succeed as a terminal no-op");
1707        let stopped_at = {
1708            let local = store.rebalance_meta.read().await;
1709            let local = local.as_ref().expect("no-op rebalance metadata should remain available");
1710            assert_eq!(local.id, rebalance_id);
1711            assert!(local.pool_stats.iter().all(|pool_stat| !pool_stat.participating));
1712            let stopped_at = local.stopped_at.expect("no-op rebalance must persist a terminal timestamp");
1713            assert_eq!(stopped_at, movement_floor + time::Duration::nanoseconds(1));
1714            stopped_at
1715        };
1716
1717        let stopped_generation =
1718            u64::try_from(stopped_at.unix_timestamp_nanos()).expect("terminal timestamp should map to scanner generation");
1719        let live_status = store.scanner_data_movement_pause_status().await;
1720        assert!(!live_status.paused);
1721        assert_eq!(live_status.movement_generation, stopped_generation);
1722
1723        restarted
1724            .load_rebalance_meta()
1725            .await
1726            .expect("restarted store should load the persisted no-op rebalance metadata");
1727        let status = restarted.scanner_data_movement_pause_status().await;
1728
1729        assert!(!status.paused);
1730        assert_eq!(status.movement_generation, stopped_generation);
1731        assert_eq!(restarted.scanner_data_movement_generation(), stopped_generation);
1732    }
1733
1734    #[tokio::test]
1735    #[serial_test::serial]
1736    async fn rebalance_activation_rejects_initialized_cluster_with_all_pool_meta_missing() {
1737        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
1738        persist_initialized_identity_then_remove_pool_meta(&store).await;
1739        set_rebalance_disk_stats_override_for_test(
1740            store.id,
1741            vec![
1742                DiskStat {
1743                    total_space: 100,
1744                    available_space: 0,
1745                },
1746                DiskStat {
1747                    total_space: 100,
1748                    available_space: 100,
1749                },
1750            ],
1751        );
1752
1753        let err = store
1754            .init_rebalance_start(vec!["missing-pool-meta".to_string()])
1755            .await
1756            .expect_err("rebalance activation must fail closed when every pool.bin is missing");
1757        assert!(err.to_string().contains("initialized cluster identity exists"));
1758        store
1759            .ensure_pool_meta_side_effects_safe("rebalance activation after missing pool metadata")
1760            .await
1761            .expect_err("the missing metadata observation must latch the shared runtime gate");
1762
1763        let mut persisted = RebalanceMeta::new();
1764        assert!(
1765            matches!(persisted.load(store.pools[0].clone()).await, Err(Error::ConfigNotFound)),
1766            "rejected activation must not create rebalance metadata"
1767        );
1768        assert_activation_locks_released(&store).await;
1769    }
1770
1771    #[tokio::test]
1772    #[serial_test::serial]
1773    async fn rebalance_worker_rejects_initialized_cluster_with_all_pool_meta_missing() {
1774        let rebalance_id = "missing-pool-meta-worker";
1775        let active = RebalanceMeta {
1776            id: rebalance_id.to_string(),
1777            percent_free_goal: 0.5,
1778            pool_stats: vec![
1779                RebalanceStats {
1780                    participating: true,
1781                    init_capacity: 100,
1782                    info: RebalanceInfo {
1783                        status: RebalStatus::Started,
1784                        ..Default::default()
1785                    },
1786                    ..Default::default()
1787                },
1788                RebalanceStats {
1789                    participating: true,
1790                    init_capacity: 100,
1791                    info: RebalanceInfo {
1792                        status: RebalStatus::Started,
1793                        ..Default::default()
1794                    },
1795                    ..Default::default()
1796                },
1797            ],
1798            ..Default::default()
1799        };
1800        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(active)).await;
1801        persist_initialized_identity_then_remove_pool_meta(&store).await;
1802
1803        let err = match store
1804            .fence_rebalance_worker_activation(store.pools[0].clone(), rebalance_id)
1805            .await
1806        {
1807            Ok(_) => panic!("worker activation must not return a fence when every pool.bin is missing"),
1808            Err(err) => err,
1809        };
1810        assert!(err.to_string().contains("initialized cluster identity exists"));
1811        store
1812            .ensure_pool_meta_side_effects_safe("rebalance worker after missing pool metadata")
1813            .await
1814            .expect_err("worker validation must latch the shared runtime gate");
1815        assert_activation_locks_released(&store).await;
1816    }
1817
1818    #[tokio::test]
1819    #[serial_test::serial]
1820    async fn rebalance_worker_skips_terminal_metadata_without_fleet_proof() {
1821        let rebalance_id = "terminal-metadata-without-proof";
1822        let completed = RebalanceMeta {
1823            id: rebalance_id.to_string(),
1824            pool_stats: vec![RebalanceStats {
1825                participating: true,
1826                info: RebalanceInfo {
1827                    status: RebalStatus::Completed,
1828                    ..Default::default()
1829                },
1830                ..Default::default()
1831            }],
1832            ..Default::default()
1833        };
1834        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(completed)).await;
1835        let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
1836
1837        let activation = store
1838            .fence_rebalance_worker_activation(store.pools[0].clone(), rebalance_id)
1839            .await
1840            .expect("terminal metadata should not require a fleet proof");
1841        assert!(matches!(activation, RebalanceWorkerActivationFence::NotStartedTerminal));
1842    }
1843
1844    #[tokio::test]
1845    #[serial_test::serial]
1846    async fn rebalance_worker_still_requires_fleet_proof_for_active_metadata() {
1847        let rebalance_id = "active-metadata-without-proof";
1848        let active = RebalanceMeta {
1849            id: rebalance_id.to_string(),
1850            pool_stats: vec![RebalanceStats {
1851                participating: true,
1852                info: RebalanceInfo {
1853                    status: RebalStatus::Started,
1854                    ..Default::default()
1855                },
1856                ..Default::default()
1857            }],
1858            ..Default::default()
1859        };
1860        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(active)).await;
1861        let _proof_guard = crate::services::notification_sys::without_cross_pool_fence_fleet_proof_for_test();
1862
1863        let err = match store
1864            .fence_rebalance_worker_activation(store.pools[0].clone(), rebalance_id)
1865            .await
1866        {
1867            Ok(_) => panic!("active metadata must not be admitted without a fleet proof"),
1868            Err(err) => err,
1869        };
1870        assert!(
1871            err.to_string()
1872                .contains("pool activation requires a live fleet capability proof")
1873        );
1874    }
1875
1876    #[tokio::test]
1877    #[serial_test::serial]
1878    async fn rebalance_activation_adopts_commit_after_post_save_fence_loss() {
1879        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
1880        set_rebalance_disk_stats_override_for_test(
1881            store.id,
1882            vec![
1883                DiskStat {
1884                    total_space: 100,
1885                    available_space: 0,
1886                },
1887                DiskStat {
1888                    total_space: 100,
1889                    available_space: 100,
1890                },
1891            ],
1892        );
1893        let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
1894        let start_store = Arc::clone(&store);
1895        let start_task = tokio::spawn(async move {
1896            start_store
1897                .init_rebalance_start(vec!["post-commit-fence-loss".to_string()])
1898                .await
1899        });
1900
1901        barrier.wait_until_paused().await;
1902        barrier.release_after_fence_loss();
1903        let rebalance_id = tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
1904            .await
1905            .expect("rebalance activation should finish after its durable commit")
1906            .expect("rebalance activation task should not panic")
1907            .expect("post-commit fence loss must not report the committed activation as failed");
1908
1909        let local_meta = store.rebalance_meta.read().await;
1910        let local = local_meta
1911            .as_ref()
1912            .expect("the committed rebalance metadata should be installed locally");
1913        assert_eq!(local.id, rebalance_id);
1914        assert!(is_rebalance_conflicting_with_decommission(local));
1915        drop(local_meta);
1916
1917        let mut persisted = RebalanceMeta::new();
1918        persisted
1919            .load(store.pools[0].clone())
1920            .await
1921            .expect("the committed rebalance metadata should remain readable");
1922        assert_eq!(persisted.id, rebalance_id);
1923        assert!(is_rebalance_conflicting_with_decommission(&persisted));
1924    }
1925
1926    #[tokio::test]
1927    #[serial_test::serial]
1928    async fn rebalance_worker_admission_installs_committed_candidate_after_post_save_fence_loss() {
1929        let rebalance_id = "post-save-worker-admission";
1930        let active = RebalanceMeta {
1931            id: rebalance_id.to_string(),
1932            percent_free_goal: 0.5,
1933            pool_stats: vec![
1934                RebalanceStats {
1935                    participating: true,
1936                    init_free_space: 50,
1937                    init_capacity: 100,
1938                    buckets: vec!["completed-pool-bucket".to_string()],
1939                    info: RebalanceInfo {
1940                        status: RebalStatus::Started,
1941                        ..Default::default()
1942                    },
1943                    ..Default::default()
1944                },
1945                RebalanceStats {
1946                    participating: true,
1947                    init_capacity: 100,
1948                    buckets: vec!["active-pool-bucket".to_string()],
1949                    info: RebalanceInfo {
1950                        status: RebalStatus::Started,
1951                        ..Default::default()
1952                    },
1953                    ..Default::default()
1954                },
1955            ],
1956            ..Default::default()
1957        };
1958        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(Some(active)).await;
1959        let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
1960        let start_store = Arc::clone(&store);
1961        let start_task = tokio::spawn(async move { start_store.start_rebalance().await });
1962
1963        barrier.wait_until_paused().await;
1964        barrier.release_after_fence_loss();
1965        tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
1966            .await
1967            .expect("rebalance worker admission should finish after its durable commit")
1968            .expect("rebalance worker admission task should not panic")
1969            .expect("post-commit fence loss must not reject the committed worker candidate");
1970
1971        let local_meta = store.rebalance_meta.read().await;
1972        let local = local_meta
1973            .as_ref()
1974            .expect("the committed worker candidate should remain installed locally");
1975        assert_eq!(local.id, rebalance_id);
1976        assert_eq!(local.pool_stats[0].info.status, RebalStatus::Completed);
1977        let admitted_cancel = local
1978            .cancel
1979            .clone()
1980            .expect("the committed worker candidate should install its cancellation token");
1981        assert!(!admitted_cancel.is_cancelled());
1982        drop(local_meta);
1983
1984        admitted_cancel.cancel();
1985        assert!(admitted_cancel.is_cancelled());
1986
1987        let mut persisted = RebalanceMeta::new();
1988        persisted
1989            .load(store.pools[0].clone())
1990            .await
1991            .expect("the committed worker candidate should remain readable");
1992        assert_eq!(persisted.id, rebalance_id);
1993        assert_eq!(persisted.pool_stats[0].info.status, RebalStatus::Completed);
1994    }
1995
1996    async fn assert_real_activation_start_race(paused_kind: PoolActivationStartKind) {
1997        let (_temp_dirs, rebalance_store, decommission_store) =
1998            crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
1999        crate::services::rebalance::promote_test_pool_meta_to_v2(&rebalance_store).await;
2000        crate::services::rebalance::promote_test_pool_meta_to_v2(&decommission_store).await;
2001        let disk_stats = vec![
2002            DiskStat {
2003                total_space: 100,
2004                available_space: 0,
2005            },
2006            DiskStat {
2007                total_space: 100,
2008                available_space: 100,
2009            },
2010        ];
2011        set_rebalance_disk_stats_override_for_test(rebalance_store.id, disk_stats.clone());
2012        set_rebalance_disk_stats_override_for_test(decommission_store.id, disk_stats);
2013        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
2014        let capacity_snapshot = vec![
2015            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
2016            DecommissionPoolCapacityInfo::for_test(1, layout, 200, 200, 0),
2017        ];
2018        // Decommission start samples capacity before and inside its durable activation fence.
2019        set_decommission_capacity_info_overrides_for_test(
2020            decommission_store.id,
2021            vec![capacity_snapshot.clone(), capacity_snapshot],
2022        );
2023        let (first_object, competing_object, competing_kind) = match paused_kind {
2024            PoolActivationStartKind::Rebalance => {
2025                (REBAL_META_NAME, crate::core::pools::POOL_META_NAME, PoolActivationStartKind::Decommission)
2026            }
2027            PoolActivationStartKind::Decommission => {
2028                (crate::core::pools::POOL_META_NAME, REBAL_META_NAME, PoolActivationStartKind::Rebalance)
2029            }
2030        };
2031        let first_barrier = PutObjectCommitBarrier::install(
2032            crate::disk::RUSTFS_META_BUCKET,
2033            first_object,
2034            PutObjectCommitPause::BeforeQuotaRename,
2035        );
2036        let competing_barrier = PutObjectCommitBarrier::install(
2037            crate::disk::RUSTFS_META_BUCKET,
2038            competing_object,
2039            PutObjectCommitPause::BeforeQuotaRename,
2040        );
2041
2042        let mut rebalance_task;
2043        let mut decommission_task;
2044        let mut observed_rebalance_result = None;
2045        let mut observed_decommission_result = None;
2046        let reached_activation = if paused_kind == PoolActivationStartKind::Rebalance {
2047            let store = Arc::clone(&rebalance_store);
2048            rebalance_task =
2049                tokio::spawn(async move { store.init_rebalance_start(vec!["bucket".to_string()]).await.map(|_| ()) });
2050            tokio::select! {
2051                _ = first_barrier.wait_until_paused() => {}
2052                result = &mut rebalance_task => {
2053                    panic!("real rebalance start finished before its activation commit: {:?}", result.expect("rebalance activation task should not panic"));
2054                }
2055            }
2056            let probe = PoolActivationStartProbe::install(competing_kind);
2057            let store = Arc::clone(&decommission_store);
2058            decommission_task = tokio::spawn(async move { store.start_decommission(vec![0]).await });
2059            tokio::time::timeout(std::time::Duration::from_secs(15), async {
2060                tokio::select! {
2061                    _ = probe.wait_until_attempted() => true,
2062                    result = &mut decommission_task => {
2063                        observed_decommission_result = Some(result.expect("decommission activation task should not panic"));
2064                        false
2065                    }
2066                }
2067            })
2068            .await
2069            .unwrap_or(false)
2070        } else {
2071            let store = Arc::clone(&decommission_store);
2072            decommission_task = tokio::spawn(async move { store.start_decommission(vec![0]).await });
2073            tokio::select! {
2074                _ = first_barrier.wait_until_paused() => {}
2075                result = &mut decommission_task => {
2076                    panic!("real decommission start finished before its activation commit: {:?}", result.expect("decommission activation task should not panic"));
2077                }
2078            }
2079            let probe = PoolActivationStartProbe::install(competing_kind);
2080            let store = Arc::clone(&rebalance_store);
2081            rebalance_task =
2082                tokio::spawn(async move { store.init_rebalance_start(vec!["bucket".to_string()]).await.map(|_| ()) });
2083            tokio::time::timeout(std::time::Duration::from_secs(15), async {
2084                tokio::select! {
2085                    _ = probe.wait_until_attempted() => true,
2086                    result = &mut rebalance_task => {
2087                        observed_rebalance_result = Some(result.expect("rebalance activation task should not panic"));
2088                        false
2089                    }
2090                }
2091            })
2092            .await
2093            .unwrap_or(false)
2094        };
2095
2096        // A competing start may be fenced while refreshing persisted metadata,
2097        // before it reaches the activation save, or at the activation commit.
2098        let competing_reached_commit = if reached_activation {
2099            tokio::time::timeout(std::time::Duration::from_secs(15), async {
2100                if paused_kind == PoolActivationStartKind::Rebalance {
2101                    tokio::select! {
2102                        _ = competing_barrier.wait_until_paused() => true,
2103                        result = &mut decommission_task => {
2104                            observed_decommission_result = Some(result.expect("decommission activation task should not panic"));
2105                            false
2106                        }
2107                    }
2108                } else {
2109                    tokio::select! {
2110                        _ = competing_barrier.wait_until_paused() => true,
2111                        result = &mut rebalance_task => {
2112                            observed_rebalance_result = Some(result.expect("rebalance activation task should not panic"));
2113                            false
2114                        }
2115                    }
2116                }
2117            })
2118            .await
2119            .expect("competing activation should either finish or reach its commit")
2120        } else {
2121            false
2122        };
2123        if competing_reached_commit {
2124            competing_barrier.release();
2125        }
2126        drop(competing_barrier);
2127        first_barrier.release();
2128        drop(first_barrier);
2129
2130        let rebalance_result = match observed_rebalance_result {
2131            Some(result) => result,
2132            None => tokio::time::timeout(std::time::Duration::from_secs(15), &mut rebalance_task)
2133                .await
2134                .expect("real rebalance activation should finish")
2135                .expect("rebalance activation task should not panic"),
2136        };
2137        let decommission_result = match observed_decommission_result {
2138            Some(result) => result,
2139            None => tokio::time::timeout(std::time::Duration::from_secs(15), &mut decommission_task)
2140                .await
2141                .expect("real decommission activation should finish")
2142                .expect("decommission activation task should not panic"),
2143        };
2144        assert!(
2145            !competing_reached_commit,
2146            "the competing real start reached its commit while the other activation lock was held"
2147        );
2148        assert_ne!(
2149            rebalance_result.is_ok(),
2150            decommission_result.is_ok(),
2151            "exactly one real activation entry may commit"
2152        );
2153
2154        let mut persisted_rebalance = RebalanceMeta::new();
2155        let rebalance_committed = persisted_rebalance
2156            .load(rebalance_store.pools[0].clone())
2157            .await
2158            .is_ok_and(|_| is_rebalance_conflicting_with_decommission(&persisted_rebalance));
2159        let mut persisted_pool = PoolMeta::default();
2160        persisted_pool
2161            .load_no_lock_from_replicas(rebalance_store.pools.clone())
2162            .await
2163            .expect("persisted pool metadata should remain readable");
2164        let decommission_committed = pool_meta_has_active_decommission(&persisted_pool);
2165        assert_eq!(
2166            usize::from(rebalance_committed) + usize::from(decommission_committed),
2167            1,
2168            "at most one persisted activation state may be active"
2169        );
2170        assert_eq!(rebalance_committed, paused_kind == PoolActivationStartKind::Rebalance);
2171        assert_eq!(decommission_committed, paused_kind == PoolActivationStartKind::Decommission);
2172    }
2173
2174    #[tokio::test]
2175    #[serial_test::serial]
2176    async fn real_rebalance_and_decommission_starts_commit_at_most_one_side() {
2177        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
2178            assert_real_activation_start_race(PoolActivationStartKind::Rebalance).await;
2179            assert_real_activation_start_race(PoolActivationStartKind::Decommission).await;
2180        })
2181        .await;
2182    }
2183
2184    #[test]
2185    fn rebalance_activation_rejects_persisted_decommission_despite_idle_local_snapshot() {
2186        let persisted = PoolMeta {
2187            pools: vec![crate::core::pools::PoolStatus {
2188                id: 0,
2189                cmd_line: "pool-0".to_string(),
2190                last_update: OffsetDateTime::UNIX_EPOCH,
2191                decommission: Some(crate::core::pools::PoolDecommissionInfo {
2192                    start_time: Some(OffsetDateTime::UNIX_EPOCH),
2193                    ..Default::default()
2194                }),
2195            }],
2196            ..Default::default()
2197        };
2198
2199        let err = ensure_rebalance_activation_pool_meta_allowed(&persisted)
2200            .expect_err("persisted decommission must block a stale rebalance admission");
2201        assert!(matches!(err, Error::DecommissionAlreadyRunning));
2202    }
2203
2204    #[test]
2205    fn rebalance_activation_allows_persisted_idle_pool_meta() {
2206        assert!(ensure_rebalance_activation_pool_meta_allowed(&PoolMeta::default()).is_ok());
2207    }
2208
2209    #[tokio::test]
2210    async fn rebalance_merge_save_does_not_commit_after_namespace_fence_loss() {
2211        let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks_isolated(4).await;
2212
2213        let persisted = RebalanceMeta {
2214            id: "rebalance-a".to_string(),
2215            pool_stats: vec![RebalanceStats {
2216                participating: true,
2217                bytes: 1,
2218                info: RebalanceInfo {
2219                    status: RebalStatus::Started,
2220                    ..Default::default()
2221                },
2222                ..Default::default()
2223            }],
2224            ..Default::default()
2225        };
2226        persisted
2227            .save(set_disks.clone())
2228            .await
2229            .expect("baseline rebalance metadata should be saved");
2230
2231        let mut update = persisted.clone();
2232        update.pool_stats[0].bytes = 999;
2233        let err = merge_and_save_rebalance_meta_no_lock(
2234            set_disks.clone(),
2235            &update,
2236            "lost namespace fence test",
2237            ObjectOptions {
2238                no_lock: true,
2239                namespace_lock_fence: Some(NamespaceLockFence::lost_for_test()),
2240                ..Default::default()
2241            },
2242            None,
2243            Some(persisted.id.as_str()),
2244        )
2245        .await
2246        .expect_err("lost namespace fence must reject the metadata commit");
2247        assert!(matches!(err, Error::NamespaceLockQuorumUnavailable { .. }));
2248
2249        let mut after = RebalanceMeta::new();
2250        after
2251            .load(set_disks)
2252            .await
2253            .expect("baseline rebalance metadata should remain readable");
2254        assert_eq!(after.id, persisted.id);
2255        assert_eq!(after.pool_stats[0].bytes, 1);
2256    }
2257
2258    #[tokio::test]
2259    async fn persisted_rebalance_run_fence_rejects_stale_active_node_after_remote_stop() {
2260        let (_temp_dirs, _disk_stores, set_disks) = hermetic_set_disks_isolated(4).await;
2261        let active = RebalanceMeta {
2262            id: "rebalance-a".to_string(),
2263            cancel: Some(tokio_util::sync::CancellationToken::new()),
2264            pool_stats: vec![RebalanceStats {
2265                participating: true,
2266                info: RebalanceInfo {
2267                    status: RebalStatus::Started,
2268                    ..Default::default()
2269                },
2270                ..Default::default()
2271            }],
2272            ..Default::default()
2273        };
2274        active
2275            .save(set_disks.clone())
2276            .await
2277            .expect("active rebalance metadata should be saved");
2278        ensure_rebalance_worker_active(Some(&active), active.id.as_str(), "stale node precondition")
2279            .expect("the stale node snapshot should still look active locally");
2280
2281        let mut stopped = active.clone();
2282        stopped.stopped_at = Some(OffsetDateTime::now_utc());
2283        stopped.pool_stats[0].info.stopping = true;
2284        stopped
2285            .save(set_disks.clone())
2286            .await
2287            .expect("remote stop should replace persisted metadata");
2288
2289        let err = acquire_persisted_rebalance_run_guard(set_disks, active.id.as_str(), "cross-node stale snapshot")
2290            .await
2291            .expect_err("persisted stop must fence a node that missed stop propagation");
2292        assert!(
2293            matches!(err, Error::OperationCanceled),
2294            "a durable remote stop cancels the same run: {err}"
2295        );
2296    }
2297
2298    #[test]
2299    fn pool_rebalance_status_ignores_non_participating_pool_state() {
2300        let meta = RebalanceMeta {
2301            pool_stats: vec![
2302                RebalanceStats {
2303                    participating: false,
2304                    info: RebalanceInfo {
2305                        status: RebalStatus::Started,
2306                        stopping: true,
2307                        ..Default::default()
2308                    },
2309                    ..Default::default()
2310                },
2311                RebalanceStats {
2312                    participating: true,
2313                    info: RebalanceInfo {
2314                        status: RebalStatus::Started,
2315                        ..Default::default()
2316                    },
2317                    ..Default::default()
2318                },
2319            ],
2320            ..Default::default()
2321        };
2322
2323        assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 0), (RebalStatus::None, false));
2324        assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 1), (RebalStatus::Started, false));
2325        assert_eq!(pool_rebalance_status_from_meta(Some(&meta), 2), (RebalStatus::None, false));
2326    }
2327
2328    #[test]
2329    fn rebalance_status_refresh_applies_persisted_terminal_state() {
2330        let rebalance_id = "rebalance-id".to_string();
2331        let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
2332        let mut current = Some(RebalanceMeta {
2333            id: rebalance_id.clone(),
2334            pool_stats: vec![RebalanceStats {
2335                participating: true,
2336                info: RebalanceInfo {
2337                    status: RebalStatus::Started,
2338                    start_time: Some(now),
2339                    ..Default::default()
2340                },
2341                ..Default::default()
2342            }],
2343            ..Default::default()
2344        });
2345        let persisted = RebalanceMeta {
2346            id: rebalance_id,
2347            pool_stats: vec![RebalanceStats {
2348                participating: true,
2349                info: RebalanceInfo {
2350                    status: RebalStatus::Completed,
2351                    start_time: Some(now),
2352                    end_time: Some(now),
2353                    ..Default::default()
2354                },
2355                ..Default::default()
2356            }],
2357            ..Default::default()
2358        };
2359
2360        assert!(merge_rebalance_status_refresh(&mut current, persisted));
2361
2362        let refreshed = current.as_ref().expect("refresh should keep rebalance metadata");
2363        assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
2364        assert_eq!(refreshed.pool_stats[0].info.end_time, Some(now));
2365    }
2366
2367    #[test]
2368    fn rebalance_status_refresh_preserves_runtime_cancel_token() {
2369        let rebalance_id = "rebalance-id".to_string();
2370        let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
2371        let mut current = Some(RebalanceMeta {
2372            id: rebalance_id.clone(),
2373            cancel: Some(tokio_util::sync::CancellationToken::new()),
2374            pool_stats: vec![RebalanceStats {
2375                participating: true,
2376                info: RebalanceInfo {
2377                    status: RebalStatus::Started,
2378                    start_time: Some(now),
2379                    ..Default::default()
2380                },
2381                ..Default::default()
2382            }],
2383            ..Default::default()
2384        });
2385        let persisted = RebalanceMeta {
2386            id: rebalance_id,
2387            pool_stats: vec![RebalanceStats {
2388                participating: true,
2389                info: RebalanceInfo {
2390                    status: RebalStatus::Started,
2391                    start_time: Some(now),
2392                    ..Default::default()
2393                },
2394                ..Default::default()
2395            }],
2396            ..Default::default()
2397        };
2398
2399        assert!(!merge_rebalance_status_refresh(&mut current, persisted));
2400
2401        assert!(
2402            current.as_ref().and_then(|meta| meta.cancel.as_ref()).is_some(),
2403            "status refresh must not drop the runtime cancellation token"
2404        );
2405    }
2406
2407    #[test]
2408    fn rebalance_status_refresh_preserves_local_active_different_id_conflict() {
2409        let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
2410        let mut current = Some(RebalanceMeta {
2411            id: "old-active-id".to_string(),
2412            cancel: Some(tokio_util::sync::CancellationToken::new()),
2413            pool_stats: vec![RebalanceStats {
2414                participating: true,
2415                info: RebalanceInfo {
2416                    status: RebalStatus::Started,
2417                    start_time: Some(now),
2418                    ..Default::default()
2419                },
2420                ..Default::default()
2421            }],
2422            ..Default::default()
2423        });
2424        let persisted = RebalanceMeta {
2425            id: "new-terminal-id".to_string(),
2426            pool_stats: vec![RebalanceStats {
2427                participating: true,
2428                info: RebalanceInfo {
2429                    status: RebalStatus::Completed,
2430                    start_time: Some(now),
2431                    end_time: Some(now),
2432                    ..Default::default()
2433                },
2434                ..Default::default()
2435            }],
2436            ..Default::default()
2437        };
2438
2439        merge_rebalance_status_refresh(&mut current, persisted);
2440
2441        let refreshed = current.as_ref().expect("local active metadata should remain visible");
2442        assert_eq!(refreshed.id, "old-active-id");
2443        assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Started);
2444        assert!(
2445            refreshed.cancel.is_some(),
2446            "status refresh must not drop a live runtime cancellation token"
2447        );
2448    }
2449
2450    #[test]
2451    fn rebalance_status_refresh_replaces_stale_memory_without_runtime_token() {
2452        let now = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
2453        let mut current = Some(RebalanceMeta {
2454            id: "old-stale-id".to_string(),
2455            pool_stats: vec![RebalanceStats {
2456                participating: true,
2457                info: RebalanceInfo {
2458                    status: RebalStatus::Started,
2459                    start_time: Some(now),
2460                    ..Default::default()
2461                },
2462                ..Default::default()
2463            }],
2464            ..Default::default()
2465        });
2466        let persisted = RebalanceMeta {
2467            id: "new-terminal-id".to_string(),
2468            pool_stats: vec![RebalanceStats {
2469                participating: true,
2470                info: RebalanceInfo {
2471                    status: RebalStatus::Completed,
2472                    start_time: Some(now),
2473                    end_time: Some(now),
2474                    ..Default::default()
2475                },
2476                ..Default::default()
2477            }],
2478            ..Default::default()
2479        };
2480
2481        merge_rebalance_status_refresh(&mut current, persisted);
2482
2483        let refreshed = current.as_ref().expect("persisted metadata should replace stale memory");
2484        assert_eq!(refreshed.id, "new-terminal-id");
2485        assert_eq!(refreshed.pool_stats[0].info.status, RebalStatus::Completed);
2486        assert!(refreshed.cancel.is_none());
2487    }
2488}