Skip to main content

rustfs_ecstore/services/rebalance/
runtime.rs

1use super::control::RebalanceWorkerActivationFence;
2use super::meta::{
3    apply_rebalance_save_option, apply_rebalance_terminal_event, classify_rebalance_terminal_event, clone_first_arc,
4    complete_rebalance_pools_at_goal, complete_rebalance_pools_with_empty_queue, ensure_valid_rebalance_pool_index,
5    has_deferred_rebalance_error, has_rebalance_cleanup_warnings, is_rebalance_in_progress, rebalance_goal_reached,
6    resolve_rebalance_participants, should_preserve_rebalance_stopped_state, should_skip_start_rebalance,
7    validate_start_rebalance_state,
8};
9use super::worker::{
10    rebalance_max_attempts, resolve_rebalance_bucket_result, resolve_rebalance_meta_save_result,
11    resolve_rebalance_save_task_result, resolve_rebalance_terminal_error, retry_rebalance_metadata_access,
12    send_rebalance_done_signal,
13};
14use super::{
15    EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
16    REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_MAX_DEFERS,
17    RebalSaveOpt, RebalStatus, RebalanceBucketOutcome, RebalanceDeferKind,
18};
19use crate::error::{Error, Result};
20use crate::runtime::sources as runtime_sources;
21use crate::store::ECStore;
22use std::collections::HashMap;
23use std::collections::HashSet;
24use std::sync::Arc;
25use time::OffsetDateTime;
26use tokio::time::{Duration, Instant};
27use tokio_util::sync::CancellationToken;
28use tracing::{debug, error, info, warn};
29
30pub(super) fn should_fail_repeated_rebalance_bucket_defer(
31    deferred_buckets: &mut HashSet<String>,
32    bucket: &str,
33    source_cleanup_deferred: bool,
34) -> bool {
35    !source_cleanup_deferred && !deferred_buckets.insert(bucket.to_string())
36}
37
38pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap<String, usize>, bucket: &str) -> usize {
39    let attempts = deferred_attempts.entry(bucket.to_string()).or_default();
40    *attempts = attempts.saturating_add(1);
41    *attempts
42}
43
44/// Retryable source cleanup conflicts are bounded per run, so a source replica that stays
45/// unreclaimable fails the bucket explicitly instead of deferring forever.
46pub(super) fn reached_rebalance_source_cleanup_defer_limit(attempt: usize) -> bool {
47    attempt >= REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
48}
49
50#[derive(Clone, Copy, Debug, PartialEq, Eq)]
51pub(super) enum RebalanceLocalActivationOutcome {
52    Started,
53    NotStartedTerminal,
54}
55
56pub(super) fn commit_local_rebalance_worker_activation(
57    meta: &mut super::RebalanceMeta,
58    expected_id: &str,
59    cancel: CancellationToken,
60) -> Result<RebalanceLocalActivationOutcome> {
61    if meta.id != expected_id {
62        return Err(Error::other(format!(
63            "rebalance metadata changed before local worker activation: expected {expected_id}, found {}",
64            meta.id
65        )));
66    }
67    if meta.stopped_at.is_some() || meta.stop_requested || !is_rebalance_in_progress(meta) {
68        return Ok(RebalanceLocalActivationOutcome::NotStartedTerminal);
69    }
70    meta.cancel = Some(cancel);
71    Ok(RebalanceLocalActivationOutcome::Started)
72}
73
74pub(super) fn stage_local_rebalance_worker_activation(
75    meta: &super::RebalanceMeta,
76    expected_id: &str,
77    cancel: CancellationToken,
78    now: OffsetDateTime,
79) -> Result<(super::RebalanceMeta, RebalanceLocalActivationOutcome, bool)> {
80    let mut candidate = meta.clone();
81    let completed_at_goal = complete_rebalance_pools_at_goal(&mut candidate, now);
82    let completed_empty_queue = complete_rebalance_pools_with_empty_queue(&mut candidate, now);
83    let outcome = commit_local_rebalance_worker_activation(&mut candidate, expected_id, cancel)?;
84    let must_persist =
85        completed_at_goal || completed_empty_queue || outcome == RebalanceLocalActivationOutcome::NotStartedTerminal;
86    Ok((candidate, outcome, must_persist))
87}
88
89pub(super) fn commit_local_rebalance_worker_activation_candidate(
90    current: &mut super::RebalanceMeta,
91    expected_id: &str,
92    expected_cancel: Option<&CancellationToken>,
93    candidate: super::RebalanceMeta,
94) -> Result<()> {
95    if current.id != expected_id || candidate.id != expected_id {
96        return Err(Error::other(format!(
97            "rebalance metadata changed before local worker activation commit: expected {expected_id}, found {}",
98            current.id
99        )));
100    }
101    if !Arc::ptr_eq(&current.activation_gate, &candidate.activation_gate) {
102        return Err(Error::other(format!(
103            "rebalance activation gate changed before local worker activation commit: {expected_id}"
104        )));
105    }
106    if current.cancel.as_ref() != expected_cancel {
107        return Err(Error::other(format!(
108            "rebalance worker token changed before local worker activation commit: {expected_id}"
109        )));
110    }
111    *current = candidate;
112    Ok(())
113}
114
115pub(super) fn rollback_local_rebalance_worker_activation(
116    meta: Option<&mut super::RebalanceMeta>,
117    expected_id: &str,
118    activation_token: &CancellationToken,
119) -> bool {
120    let Some(meta) = meta else {
121        return false;
122    };
123    if meta.id != expected_id || meta.cancel.as_ref() != Some(activation_token) {
124        return false;
125    }
126    if let Some(cancel) = meta.cancel.take() {
127        cancel.cancel();
128        return true;
129    }
130    false
131}
132
133impl ECStore {
134    #[tracing::instrument(skip_all)]
135    pub async fn start_rebalance(self: &Arc<Self>) -> Result<()> {
136        let _start_guard = self.start_gate.lock().await;
137        let _activation_guard = self.rebalance_activation_write_guard(None, "start rebalance").await?;
138        let movement_gate = self.ctx.data_movement_operation_gate();
139        let _movement_guard = movement_gate.write().await;
140        if self.start_rebalance_under_gate().await? {
141            self.ctx.advance_data_movement_operation_epoch();
142        }
143        Ok(())
144    }
145
146    pub(super) async fn start_rebalance_under_gate(self: &Arc<Self>) -> Result<bool> {
147        info!(
148            event = EVENT_REBALANCE_STATE,
149            component = LOG_COMPONENT_ECSTORE,
150            subsystem = LOG_SUBSYSTEM_REBALANCE,
151            state = "starting",
152            "Starting rebalance"
153        );
154        let expected_id: Arc<str> = {
155            let rebalance_meta = self.rebalance_meta.read().await;
156            Arc::from(rebalance_meta.as_ref().ok_or(Error::ConfigNotFound)?.id.as_str())
157        };
158        let pool = clone_first_arc(self.pools.as_slice(), "start_rebalance: no pools available")?;
159        let activation_fence = match self
160            .fence_rebalance_worker_activation(pool.clone(), expected_id.as_ref())
161            .await?
162        {
163            RebalanceWorkerActivationFence::Ready(fence) => fence,
164            RebalanceWorkerActivationFence::NotStartedTerminal => return Ok(false),
165        };
166
167        let decommission_running = self.is_decommission_running().await;
168
169        let cancel_tx = CancellationToken::new();
170        let rx = cancel_tx.clone();
171        let activation_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
172        let activation_outcome;
173        let candidate;
174        let expected_cancel;
175        let must_persist;
176
177        {
178            let mut rebalance_meta = self.rebalance_meta.write().await;
179            validate_start_rebalance_state(decommission_running, rebalance_meta.is_some())?;
180
181            let Some(meta) = rebalance_meta.as_mut() else {
182                return Err(Error::ConfigNotFound);
183            };
184            if should_skip_start_rebalance(meta.cancel.is_some(), is_rebalance_in_progress(meta)) {
185                debug!(
186                    event = EVENT_REBALANCE_STATE,
187                    component = LOG_COMPONENT_ECSTORE,
188                    subsystem = LOG_SUBSYSTEM_REBALANCE,
189                    state = "start_skipped",
190                    reason = "already_in_progress",
191                    "Skipped duplicate rebalance start"
192                );
193                return Ok(false);
194            }
195            expected_cancel = meta.cancel.clone();
196            (candidate, activation_outcome, must_persist) =
197                stage_local_rebalance_worker_activation(meta, expected_id.as_ref(), cancel_tx.clone(), activation_at)?;
198            if let Err(err) = activation_fence.ensure_held() {
199                cancel_tx.cancel();
200                return Err(err);
201            }
202            if !must_persist
203                && let Err(err) = commit_local_rebalance_worker_activation_candidate(
204                    meta,
205                    expected_id.as_ref(),
206                    expected_cancel.as_ref(),
207                    candidate.clone(),
208                )
209            {
210                cancel_tx.cancel();
211                return Err(err);
212            }
213        }
214
215        if must_persist {
216            let save_result = resolve_rebalance_meta_save_result(
217                self.save_rebalance_meta_under_activation_fence(
218                    pool,
219                    &candidate,
220                    "start_rebalance persist activation candidate",
221                    activation_fence.as_ref(),
222                    expected_id.as_ref(),
223                )
224                .await,
225                "start_rebalance persist activation candidate",
226            );
227            if let Err(err) = save_result {
228                cancel_tx.cancel();
229                return Err(err);
230            }
231            let mut rebalance_meta = self.rebalance_meta.write().await;
232            let Some(meta) = rebalance_meta.as_mut() else {
233                cancel_tx.cancel();
234                return Err(Error::ConfigNotFound);
235            };
236            if let Err(err) = commit_local_rebalance_worker_activation_candidate(
237                meta,
238                expected_id.as_ref(),
239                expected_cancel.as_ref(),
240                candidate,
241            ) {
242                cancel_tx.cancel();
243                return Err(err);
244            }
245        }
246        if !must_persist && let Err(err) = activation_fence.ensure_held() {
247            let mut rebalance_meta = self.rebalance_meta.write().await;
248            rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
249            return Err(err);
250        }
251        drop(activation_fence);
252
253        if activation_outcome != RebalanceLocalActivationOutcome::Started {
254            return Ok(must_persist);
255        }
256
257        let participants = if let Some(ref meta) = *self.rebalance_meta.read().await {
258            resolve_rebalance_participants(meta.pool_stats.as_slice(), self.pools.len())
259        } else {
260            debug!(
261                event = EVENT_REBALANCE_STATE,
262                component = LOG_COMPONENT_ECSTORE,
263                subsystem = LOG_SUBSYSTEM_REBALANCE,
264                state = "start_skipped",
265                reason = "metadata_missing",
266                "Skipped rebalance start because metadata is unavailable"
267            );
268            Vec::new()
269        };
270
271        if !participants.iter().any(|participating| *participating) {
272            let mut rebalance_meta = self.rebalance_meta.write().await;
273            rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
274            debug!(
275                event = EVENT_REBALANCE_STATE,
276                component = LOG_COMPONENT_ECSTORE,
277                subsystem = LOG_SUBSYSTEM_REBALANCE,
278                state = "start_skipped",
279                reason = "no_participants",
280                "Skipped rebalance start because no pools are participating"
281            );
282            return Ok(must_persist);
283        }
284
285        #[cfg(test)]
286        let endpoints = self.instance_endpoints().unwrap_or_else(|| self.endpoints());
287        #[cfg(not(test))]
288        let endpoints = self.endpoints();
289
290        let mut workers_started = 0usize;
291        for (idx, participating) in participants.iter().enumerate() {
292            if !*participating {
293                debug!(
294                    event = EVENT_REBALANCE_STATE,
295                    component = LOG_COMPONENT_ECSTORE,
296                    subsystem = LOG_SUBSYSTEM_REBALANCE,
297                    pool_index = idx,
298                    state = "pool_skipped",
299                    reason = "not_participating",
300                    "Skipped rebalance pool"
301                );
302                continue;
303            }
304
305            if !runtime_sources::endpoint_pool_is_local(&endpoints, idx) {
306                debug!(
307                    event = EVENT_REBALANCE_STATE,
308                    component = LOG_COMPONENT_ECSTORE,
309                    subsystem = LOG_SUBSYSTEM_REBALANCE,
310                    pool_index = idx,
311                    state = "pool_skipped",
312                    reason = "not_local",
313                    "Skipped non-local rebalance pool"
314                );
315                continue;
316            }
317
318            let pool_idx = idx;
319            let store = self.clone();
320            let rx_clone = rx.clone();
321            let worker_id = Arc::clone(&expected_id);
322            workers_started += 1;
323            tokio::spawn(async move {
324                if let Err(err) = store.rebalance_buckets(rx_clone, pool_idx, worker_id).await {
325                    error!(
326                        event = EVENT_REBALANCE_STATE,
327                        component = LOG_COMPONENT_ECSTORE,
328                        subsystem = LOG_SUBSYSTEM_REBALANCE,
329                        pool_index = pool_idx,
330                        state = "pool_failed",
331                        error = %err,
332                        "Rebalance pool failed"
333                    );
334                } else {
335                    debug!(
336                        event = EVENT_REBALANCE_STATE,
337                        component = LOG_COMPONENT_ECSTORE,
338                        subsystem = LOG_SUBSYSTEM_REBALANCE,
339                        pool_index = pool_idx,
340                        state = "completed",
341                        "Rebalance pool completed"
342                    );
343                }
344            });
345        }
346
347        if workers_started == 0 {
348            let mut rebalance_meta = self.rebalance_meta.write().await;
349            rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
350            debug!(
351                event = EVENT_REBALANCE_STATE,
352                component = LOG_COMPONENT_ECSTORE,
353                subsystem = LOG_SUBSYSTEM_REBALANCE,
354                state = "start_skipped",
355                reason = "no_local_participants",
356                "Skipped rebalance start because no local pools are participating"
357            );
358            return Ok(must_persist);
359        }
360
361        info!(
362            event = EVENT_REBALANCE_STATE,
363            component = LOG_COMPONENT_ECSTORE,
364            subsystem = LOG_SUBSYSTEM_REBALANCE,
365            state = "started",
366            worker_count = workers_started,
367            "Rebalance started"
368        );
369        Ok(true)
370    }
371
372    #[tracing::instrument(skip(self, rx))]
373    async fn rebalance_buckets(self: &Arc<Self>, rx: CancellationToken, pool_index: usize, rebalance_id: Arc<str>) -> Result<()> {
374        ensure_valid_rebalance_pool_index(self.pools.len(), pool_index)?;
375
376        let (done_tx, mut done_rx) = tokio::sync::mpsc::channel::<Result<()>>(1);
377
378        // Save rebalance metadata periodically
379        let store = self.clone();
380        let save_rebalance_id = Arc::clone(&rebalance_id);
381        let save_task = tokio::spawn(async move {
382            let mut timer = tokio::time::interval_at(Instant::now() + Duration::from_secs(30), Duration::from_secs(10));
383            let mut msg: String;
384            let mut quit = false;
385
386            loop {
387                let mut terminal_state_saved = false;
388                tokio::select! {
389                    result = done_rx.recv() => {
390                        quit = true;
391                        let movement_gate = store.ctx.data_movement_operation_gate();
392                        let movement_guard = movement_gate.write().await;
393                        let terminal_at = store.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
394                        let terminal_event = classify_rebalance_terminal_event(result, terminal_at);
395                        msg = terminal_event.message().to_string();
396                        let previous_meta = store.rebalance_meta.read().await.clone();
397                        let terminal_state_present = {
398                            let mut rebalance_meta = store.rebalance_meta.write().await;
399                            super::control::ensure_rebalance_run_id(
400                                rebalance_meta.as_ref(),
401                                save_rebalance_id.as_ref(),
402                                "apply rebalance terminal event",
403                            )?;
404                            if let Some(meta) = rebalance_meta.as_mut() {
405                                let meta_stopped = meta.stopped_at.is_some();
406                                if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
407                                    if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
408                                        && has_rebalance_cleanup_warnings(pool_stat)
409                                    {
410                                        pool_stat.info.stopping = false;
411                                        pool_stat.info.status = RebalStatus::Failed;
412                                        pool_stat.info.end_time = Some(terminal_at);
413                                        pool_stat.info.last_error = Some(
414                                            pool_stat
415                                                .cleanup_warnings
416                                                .last_message
417                                                .clone()
418                                                .unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
419                                        );
420                                    } else if should_preserve_rebalance_stopped_state(
421                                        meta_stopped,
422                                        pool_stat.info.status,
423                                        &terminal_event,
424                                    ) {
425                                        debug!(
426                                            event = EVENT_REBALANCE_STATE,
427                                            component = LOG_COMPONENT_ECSTORE,
428                                            subsystem = LOG_SUBSYSTEM_REBALANCE,
429                                            pool_index,
430                                            state = "stopped_preserved",
431                                            "Preserved stopped rebalance status"
432                                        );
433                                    } else {
434                                        pool_stat.info.stopping = false;
435                                        apply_rebalance_terminal_event(
436                                            &mut pool_stat.info.status,
437                                            &mut pool_stat.info.end_time,
438                                            &mut pool_stat.info.last_error,
439                                            terminal_event,
440                                            terminal_at,
441                                        );
442                                    }
443                                    true
444                                } else {
445                                    false
446                                }
447                            } else {
448                                false
449                            }
450                        };
451
452                        if terminal_state_present {
453                            if let Err(err) = retry_rebalance_metadata_access(None, rebalance_max_attempts(), || {
454                                store.save_rebalance_stats_inner(
455                                    pool_index,
456                                    RebalSaveOpt::Stats,
457                                    Some(save_rebalance_id.as_ref()),
458                                )
459                            })
460                            .await
461                            {
462                                let mut rebalance_meta = store.rebalance_meta.write().await;
463                                *rebalance_meta = previous_meta;
464                                drop(movement_guard);
465                                return Err(Error::other(format!(
466                                    "rebalance terminal state save failed for pool {pool_index}: {err}"
467                                )));
468                            }
469                            store.ctx.advance_data_movement_operation_epoch();
470                            terminal_state_saved = true;
471                        }
472                    }
473                    _ = timer.tick() => {
474                        let now = OffsetDateTime::now_utc();
475                        msg = format!("Saving rebalance metadata at {now:?}");
476                    }
477                }
478
479                if !terminal_state_saved
480                    && let Err(err) = retry_rebalance_metadata_access(None, rebalance_max_attempts(), || {
481                        store.save_rebalance_stats_for_id(pool_index, RebalSaveOpt::Stats, save_rebalance_id.as_ref())
482                    })
483                    .await
484                {
485                    let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}"));
486                    error!("{} err: {:?}", msg, wrapped);
487                    if quit {
488                        return Err(wrapped);
489                    }
490                } else {
491                    debug!(
492                        event = EVENT_REBALANCE_STATE,
493                        component = LOG_COMPONENT_ECSTORE,
494                        subsystem = LOG_SUBSYSTEM_REBALANCE,
495                        pool_index,
496                        state = "metadata_saved",
497                        message = %msg,
498                        "Saved rebalance metadata"
499                    );
500                }
501
502                if quit {
503                    debug!(
504                        event = EVENT_REBALANCE_STATE,
505                        component = LOG_COMPONENT_ECSTORE,
506                        subsystem = LOG_SUBSYSTEM_REBALANCE,
507                        pool_index,
508                        state = "save_task_exiting",
509                        message = %msg,
510                        "Exiting rebalance save task"
511                    );
512                    return Ok(());
513                }
514
515                timer.reset();
516            }
517        });
518
519        info!(
520            event = EVENT_REBALANCE_STATE,
521            component = LOG_COMPONENT_ECSTORE,
522            subsystem = LOG_SUBSYSTEM_REBALANCE,
523            pool_index,
524            state = "pool_started",
525            "Rebalance worker started"
526        );
527        let mut final_result: Result<()> = Ok(());
528        let mut deferred_buckets = HashSet::new();
529        let mut source_cleanup_deferred_attempts = HashMap::new();
530
531        loop {
532            if rx.is_cancelled() {
533                info!(
534                    event = EVENT_REBALANCE_STATE,
535                    component = LOG_COMPONENT_ECSTORE,
536                    subsystem = LOG_SUBSYSTEM_REBALANCE,
537                    pool_index,
538                    state = "pool_stopped",
539                    reason = "cancelled",
540                    "Stopped rebalance worker"
541                );
542                let err = Error::OperationCanceled;
543                final_result = Err(resolve_rebalance_terminal_error(
544                    err.clone(),
545                    send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
546                ));
547                break;
548            }
549
550            let next_bucket = match self.next_rebal_bucket(pool_index, rebalance_id.as_ref()).await {
551                Ok(bucket) => bucket,
552                Err(err) => {
553                    error!(
554                        event = EVENT_REBALANCE_BUCKET,
555                        component = LOG_COMPONENT_ECSTORE,
556                        subsystem = LOG_SUBSYSTEM_REBALANCE,
557                        pool_index,
558                        state = "next_bucket_failed",
559                        error = ?err,
560                        "Rebalance next bucket lookup failed"
561                    );
562                    final_result = Err(resolve_rebalance_terminal_error(
563                        err.clone(),
564                        send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
565                    ));
566                    break;
567                }
568            };
569
570            if let Some(bucket) = next_bucket {
571                debug!(
572                    event = EVENT_REBALANCE_BUCKET,
573                    component = LOG_COMPONENT_ECSTORE,
574                    subsystem = LOG_SUBSYSTEM_REBALANCE,
575                    pool_index,
576                    bucket = %bucket,
577                    state = "started",
578                    "Starting rebalance bucket"
579                );
580
581                let outcome = match resolve_rebalance_bucket_result(
582                    self.rebalance_bucket(rx.clone(), bucket.clone(), pool_index, Arc::clone(&rebalance_id))
583                        .await,
584                    pool_index,
585                    &bucket,
586                ) {
587                    Ok(outcome) => outcome,
588                    Err(err) => {
589                        error!(
590                            event = EVENT_REBALANCE_BUCKET,
591                            component = LOG_COMPONENT_ECSTORE,
592                            subsystem = LOG_SUBSYSTEM_REBALANCE,
593                            pool_index,
594                            bucket = %bucket,
595                            state = "bucket_failed",
596                            error = ?err,
597                            "Rebalance bucket failed"
598                        );
599                        final_result = Err(resolve_rebalance_terminal_error(
600                            err.clone(),
601                            send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
602                        ));
603                        break;
604                    }
605                };
606
607                if let RebalanceBucketOutcome::Deferred { last_error } = outcome {
608                    let source_cleanup_deferred = last_error.starts_with(REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX);
609                    if should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, &bucket, source_cleanup_deferred) {
610                        let err = Error::other(format!(
611                            "rebalance bucket {bucket} deferred repeatedly due to transient object failures: {last_error}"
612                        ));
613                        error!(
614                            event = EVENT_REBALANCE_BUCKET,
615                            component = LOG_COMPONENT_ECSTORE,
616                            subsystem = LOG_SUBSYSTEM_REBALANCE,
617                            pool_index,
618                            bucket = %bucket,
619                            state = "bucket_deferred_repeatedly",
620                            error = ?err,
621                            "Rebalance bucket failed after repeated deferral"
622                        );
623                        final_result = Err(resolve_rebalance_terminal_error(
624                            err.clone(),
625                            send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
626                        ));
627                        break;
628                    }
629
630                    let source_cleanup_attempt = if source_cleanup_deferred {
631                        source_cleanup_defer_attempt(&mut source_cleanup_deferred_attempts, &bucket)
632                    } else {
633                        0
634                    };
635                    let defer_kind = if source_cleanup_deferred {
636                        RebalanceDeferKind::SourceCleanup
637                    } else {
638                        RebalanceDeferKind::Entry
639                    };
640                    warn!(
641                        event = EVENT_REBALANCE_BUCKET,
642                        component = LOG_COMPONENT_ECSTORE,
643                        subsystem = LOG_SUBSYSTEM_REBALANCE,
644                        pool_index,
645                        bucket = %bucket,
646                        state = "deferred",
647                        error = %last_error,
648                        "Deferred rebalance bucket after transient object failures"
649                    );
650                    if let Err(err) = self
651                        .defer_rebalance_bucket(pool_index, bucket.clone(), last_error.clone(), rebalance_id.as_ref(), defer_kind)
652                        .await
653                    {
654                        error!(
655                            event = EVENT_REBALANCE_BUCKET,
656                            component = LOG_COMPONENT_ECSTORE,
657                            subsystem = LOG_SUBSYSTEM_REBALANCE,
658                            pool_index,
659                            bucket = %bucket,
660                            state = "defer_failed",
661                            error = ?err,
662                            "Rebalance bucket defer failed"
663                        );
664                        final_result = Err(resolve_rebalance_terminal_error(
665                            err.clone(),
666                            send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
667                        ));
668                        break;
669                    }
670                    if source_cleanup_deferred {
671                        if reached_rebalance_source_cleanup_defer_limit(source_cleanup_attempt) {
672                            let err = Error::other(format!(
673                                "rebalance bucket {bucket} source cleanup remained unstable after {} deferrals: {last_error}",
674                                REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
675                            ));
676                            warn!(
677                                event = EVENT_REBALANCE_BUCKET,
678                                component = LOG_COMPONENT_ECSTORE,
679                                subsystem = LOG_SUBSYSTEM_REBALANCE,
680                                pool_index,
681                                bucket = %bucket,
682                                state = "source_cleanup_defer_limit",
683                                error = ?err,
684                                "Rebalance bucket failed after repeated source cleanup conflicts"
685                            );
686                            final_result = Err(resolve_rebalance_terminal_error(
687                                err.clone(),
688                                send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
689                            ));
690                            break;
691                        }
692                        if let Err(err) =
693                            super::worker::wait_rebalance_listing_retry(&rx, REBALANCE_LISTING_RETRY_BASE_DELAY).await
694                        {
695                            final_result = Err(resolve_rebalance_terminal_error(
696                                err.clone(),
697                                send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
698                            ));
699                            break;
700                        }
701                    }
702                    continue;
703                }
704
705                debug!(
706                    event = EVENT_REBALANCE_BUCKET,
707                    component = LOG_COMPONENT_ECSTORE,
708                    subsystem = LOG_SUBSYSTEM_REBALANCE,
709                    pool_index,
710                    bucket = %bucket,
711                    state = "completed",
712                    "Completed rebalance bucket"
713                );
714                source_cleanup_deferred_attempts.remove(&bucket);
715                if let Err(err) = self.bucket_rebalance_done(pool_index, bucket, rebalance_id.as_ref()).await {
716                    error!(
717                        event = EVENT_REBALANCE_BUCKET,
718                        component = LOG_COMPONENT_ECSTORE,
719                        subsystem = LOG_SUBSYSTEM_REBALANCE,
720                        pool_index,
721                        state = "bucket_done_mark_failed",
722                        error = ?err,
723                        "Rebalance bucket completion mark failed"
724                    );
725                    final_result = Err(resolve_rebalance_terminal_error(
726                        err.clone(),
727                        send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
728                    ));
729                    break;
730                }
731            } else {
732                debug!(
733                    event = EVENT_REBALANCE_BUCKET,
734                    component = LOG_COMPONENT_ECSTORE,
735                    subsystem = LOG_SUBSYSTEM_REBALANCE,
736                    pool_index,
737                    state = "idle",
738                    reason = "no_bucket_to_rebalance",
739                    "No rebalance bucket available"
740                );
741                break;
742            }
743        }
744
745        info!(
746            event = EVENT_REBALANCE_STATE,
747            component = LOG_COMPONENT_ECSTORE,
748            subsystem = LOG_SUBSYSTEM_REBALANCE,
749            pool_index,
750            state = "pool_done",
751            "Rebalance worker finished"
752        );
753
754        if final_result.is_ok()
755            && let Err(err) = send_rebalance_done_signal(&done_tx, Ok(()), pool_index).await
756        {
757            final_result = Err(err);
758        }
759        drop(done_tx);
760        if let Err(err) = resolve_rebalance_save_task_result(pool_index, save_task.await)
761            && final_result.is_ok()
762        {
763            final_result = Err(err);
764        }
765        debug!(
766            event = EVENT_REBALANCE_STATE,
767            component = LOG_COMPONENT_ECSTORE,
768            subsystem = LOG_SUBSYSTEM_REBALANCE,
769            pool_index,
770            state = "pool_result_returned",
771            "Rebalance worker result returned"
772        );
773        final_result
774    }
775
776    pub(super) async fn check_if_rebalance_done(&self, pool_index: usize, expected_id: &str) -> Result<bool> {
777        let mut rebalance_meta = self.rebalance_meta.write().await;
778        super::control::ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "check rebalance completion")?;
779
780        if let Some(meta) = rebalance_meta.as_mut()
781            && let Some(pool_stat) = meta.pool_stats.get_mut(pool_index)
782        {
783            // Check if the pool's rebalance status is already completed
784            if pool_stat.info.status == RebalStatus::Completed {
785                debug!(
786                    event = EVENT_REBALANCE_STATE,
787                    component = LOG_COMPONENT_ECSTORE,
788                    subsystem = LOG_SUBSYSTEM_REBALANCE,
789                    pool_index,
790                    state = "already_completed",
791                    "Rebalance pool is already completed"
792                );
793                return Ok(true);
794            }
795
796            // Mark pool rebalance as done only after it reaches the PercentFreeGoal.
797            let pfi = if pool_stat.init_capacity == 0 {
798                0.0
799            } else {
800                (pool_stat.init_free_space + pool_stat.bytes) as f64 / pool_stat.init_capacity as f64
801            };
802
803            if !has_deferred_rebalance_error(pool_stat)
804                && !has_rebalance_cleanup_warnings(pool_stat)
805                && rebalance_goal_reached(
806                    pool_stat.init_free_space,
807                    pool_stat.init_capacity,
808                    pool_stat.bytes,
809                    meta.percent_free_goal,
810                )
811            {
812                info!(
813                    event = EVENT_REBALANCE_STATE,
814                    component = LOG_COMPONENT_ECSTORE,
815                    subsystem = LOG_SUBSYSTEM_REBALANCE,
816                    pool_index,
817                    state = "completion_ready",
818                    percent_free = pfi,
819                    "Rebalance pool reached completion goal"
820                );
821                return Ok(true);
822            }
823        }
824
825        Ok(false)
826    }
827}
828
829impl ECStore {
830    #[tracing::instrument(skip(self))]
831    pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
832        let movement_gate = self.ctx.data_movement_operation_gate();
833        let _movement_guard = movement_gate.write().await;
834        self.save_rebalance_stats_inner(pool_idx, opt, None).await
835    }
836
837    pub async fn save_rebalance_stats_for_id(&self, pool_idx: usize, opt: RebalSaveOpt, expected_id: &str) -> Result<()> {
838        let movement_gate = self.ctx.data_movement_operation_gate();
839        let _movement_guard = movement_gate.write().await;
840        self.save_rebalance_stats_inner(pool_idx, opt, Some(expected_id)).await
841    }
842
843    pub(super) async fn save_rebalance_stats_inner(
844        &self,
845        pool_idx: usize,
846        opt: RebalSaveOpt,
847        expected_id: Option<&str>,
848    ) -> Result<()> {
849        let now = match opt {
850            RebalSaveOpt::Stats => OffsetDateTime::now_utc(),
851            RebalSaveOpt::StoppedAt => self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await,
852        };
853        let meta_to_save = {
854            let mut rebalance_meta = self.rebalance_meta.write().await;
855            if let Some(expected_id) = expected_id {
856                super::control::ensure_rebalance_run_id(rebalance_meta.as_ref(), expected_id, "save rebalance stats")?;
857            }
858            let Some(meta) = rebalance_meta.as_mut() else {
859                return Ok(());
860            };
861
862            apply_rebalance_save_option(meta, pool_idx, opt, now);
863            meta.clone()
864        };
865
866        let pool = clone_first_arc(&self.pools, "save_rebalance_stats: no pools available")?;
867
868        debug!(
869            event = EVENT_REBALANCE_STATE,
870            component = LOG_COMPONENT_ECSTORE,
871            subsystem = LOG_SUBSYSTEM_REBALANCE,
872            pool_index = pool_idx,
873            save_opt = ?opt,
874            state = "metadata_save_requested",
875            "Rebalance metadata save requested"
876        );
877        let stage = format!("save_rebalance_stats for pool {pool_idx} opt {opt:?}");
878        let save_result = match expected_id {
879            Some(expected_id) => {
880                self.save_rebalance_meta_for_id_with_merge(pool, &meta_to_save, stage.as_str(), expected_id)
881                    .await
882            }
883            None => self.save_rebalance_meta_with_merge(pool, &meta_to_save, stage.as_str()).await,
884        };
885        resolve_rebalance_meta_save_result(save_result, stage.as_str())?;
886
887        Ok(())
888    }
889}