Skip to main content

rustfs_ecstore/services/rebalance/
runtime.rs

1use super::control::RebalanceWorkerActivationFence;
2use super::meta::{
3    apply_rebalance_save_option, apply_rebalance_terminal_event, classify_rebalance_terminal_event, clone_first_arc,
4    complete_rebalance_pools_at_goal, complete_rebalance_pools_with_empty_queue, ensure_valid_rebalance_pool_index,
5    has_deferred_rebalance_error, has_rebalance_cleanup_warnings, is_rebalance_in_progress, rebalance_goal_reached,
6    resolve_rebalance_participants, should_preserve_rebalance_stopped_state, should_skip_start_rebalance,
7    validate_start_rebalance_state,
8};
9use super::worker::{
10    resolve_rebalance_bucket_result, resolve_rebalance_meta_save_result, resolve_rebalance_save_task_result,
11    resolve_rebalance_terminal_error, send_rebalance_done_signal,
12};
13use super::{
14    EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
15    REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, RebalSaveOpt, RebalStatus,
16    RebalanceBucketOutcome,
17};
18use crate::error::{Error, Result};
19use crate::runtime::sources as runtime_sources;
20use crate::store::ECStore;
21use std::collections::HashMap;
22use std::collections::HashSet;
23use std::sync::Arc;
24use time::OffsetDateTime;
25use tokio::time::{Duration, Instant};
26use tokio_util::sync::CancellationToken;
27use tracing::{debug, error, info, warn};
28
29pub(super) fn should_fail_repeated_rebalance_bucket_defer(
30    deferred_buckets: &mut HashSet<String>,
31    bucket: &str,
32    source_cleanup_deferred: bool,
33) -> bool {
34    !source_cleanup_deferred && !deferred_buckets.insert(bucket.to_string())
35}
36
37pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap<String, usize>, bucket: &str) -> usize {
38    let attempts = deferred_attempts.entry(bucket.to_string()).or_default();
39    *attempts = attempts.saturating_add(1);
40    *attempts
41}
42
43#[derive(Clone, Copy, Debug, PartialEq, Eq)]
44pub(super) enum RebalanceLocalActivationOutcome {
45    Started,
46    NotStartedTerminal,
47}
48
49pub(super) fn commit_local_rebalance_worker_activation(
50    meta: &mut super::RebalanceMeta,
51    expected_id: &str,
52    cancel: CancellationToken,
53) -> Result<RebalanceLocalActivationOutcome> {
54    if meta.id != expected_id {
55        return Err(Error::other(format!(
56            "rebalance metadata changed before local worker activation: expected {expected_id}, found {}",
57            meta.id
58        )));
59    }
60    if meta.stopped_at.is_some() || meta.stop_requested || !is_rebalance_in_progress(meta) {
61        return Ok(RebalanceLocalActivationOutcome::NotStartedTerminal);
62    }
63    meta.cancel = Some(cancel);
64    Ok(RebalanceLocalActivationOutcome::Started)
65}
66
67pub(super) fn stage_local_rebalance_worker_activation(
68    meta: &super::RebalanceMeta,
69    expected_id: &str,
70    cancel: CancellationToken,
71    now: OffsetDateTime,
72) -> Result<(super::RebalanceMeta, RebalanceLocalActivationOutcome, bool)> {
73    let mut candidate = meta.clone();
74    let completed_at_goal = complete_rebalance_pools_at_goal(&mut candidate, now);
75    let completed_empty_queue = complete_rebalance_pools_with_empty_queue(&mut candidate, now);
76    let outcome = commit_local_rebalance_worker_activation(&mut candidate, expected_id, cancel)?;
77    let must_persist =
78        completed_at_goal || completed_empty_queue || outcome == RebalanceLocalActivationOutcome::NotStartedTerminal;
79    Ok((candidate, outcome, must_persist))
80}
81
82pub(super) fn commit_local_rebalance_worker_activation_candidate(
83    current: &mut super::RebalanceMeta,
84    expected_id: &str,
85    expected_cancel: Option<&CancellationToken>,
86    candidate: super::RebalanceMeta,
87) -> Result<()> {
88    if current.id != expected_id || candidate.id != expected_id {
89        return Err(Error::other(format!(
90            "rebalance metadata changed before local worker activation commit: expected {expected_id}, found {}",
91            current.id
92        )));
93    }
94    if !Arc::ptr_eq(&current.activation_gate, &candidate.activation_gate) {
95        return Err(Error::other(format!(
96            "rebalance activation gate changed before local worker activation commit: {expected_id}"
97        )));
98    }
99    if current.cancel.as_ref() != expected_cancel {
100        return Err(Error::other(format!(
101            "rebalance worker token changed before local worker activation commit: {expected_id}"
102        )));
103    }
104    *current = candidate;
105    Ok(())
106}
107
108pub(super) fn rollback_local_rebalance_worker_activation(
109    meta: Option<&mut super::RebalanceMeta>,
110    expected_id: &str,
111    activation_token: &CancellationToken,
112) -> bool {
113    let Some(meta) = meta else {
114        return false;
115    };
116    if meta.id != expected_id || meta.cancel.as_ref() != Some(activation_token) {
117        return false;
118    }
119    if let Some(cancel) = meta.cancel.take() {
120        cancel.cancel();
121        return true;
122    }
123    false
124}
125
126impl ECStore {
127    #[tracing::instrument(skip_all)]
128    pub async fn start_rebalance(self: &Arc<Self>) -> Result<()> {
129        let _start_guard = self.start_gate.lock().await;
130        let _activation_guard = self.rebalance_activation_write_guard(None, "start rebalance").await?;
131        let movement_gate = self.ctx.data_movement_operation_gate();
132        let _movement_guard = movement_gate.write().await;
133        if self.start_rebalance_under_gate().await? {
134            self.ctx.advance_data_movement_operation_epoch();
135        }
136        Ok(())
137    }
138
139    pub(super) async fn start_rebalance_under_gate(self: &Arc<Self>) -> Result<bool> {
140        info!(
141            event = EVENT_REBALANCE_STATE,
142            component = LOG_COMPONENT_ECSTORE,
143            subsystem = LOG_SUBSYSTEM_REBALANCE,
144            state = "starting",
145            "Starting rebalance"
146        );
147        let expected_id: Arc<str> = {
148            let rebalance_meta = self.rebalance_meta.read().await;
149            Arc::from(rebalance_meta.as_ref().ok_or(Error::ConfigNotFound)?.id.as_str())
150        };
151        let pool = clone_first_arc(self.pools.as_slice(), "start_rebalance: no pools available")?;
152        let activation_fence = match self
153            .fence_rebalance_worker_activation(pool.clone(), expected_id.as_ref())
154            .await?
155        {
156            RebalanceWorkerActivationFence::Ready(fence) => fence,
157            RebalanceWorkerActivationFence::NotStartedTerminal => return Ok(false),
158        };
159
160        let decommission_running = self.is_decommission_running().await;
161
162        let cancel_tx = CancellationToken::new();
163        let rx = cancel_tx.clone();
164        let activation_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
165        let activation_outcome;
166        let candidate;
167        let expected_cancel;
168        let must_persist;
169
170        {
171            let mut rebalance_meta = self.rebalance_meta.write().await;
172            validate_start_rebalance_state(decommission_running, rebalance_meta.is_some())?;
173
174            let Some(meta) = rebalance_meta.as_mut() else {
175                return Err(Error::ConfigNotFound);
176            };
177            if should_skip_start_rebalance(meta.cancel.is_some(), is_rebalance_in_progress(meta)) {
178                debug!(
179                    event = EVENT_REBALANCE_STATE,
180                    component = LOG_COMPONENT_ECSTORE,
181                    subsystem = LOG_SUBSYSTEM_REBALANCE,
182                    state = "start_skipped",
183                    reason = "already_in_progress",
184                    "Skipped duplicate rebalance start"
185                );
186                return Ok(false);
187            }
188            expected_cancel = meta.cancel.clone();
189            (candidate, activation_outcome, must_persist) =
190                stage_local_rebalance_worker_activation(meta, expected_id.as_ref(), cancel_tx.clone(), activation_at)?;
191            if let Err(err) = activation_fence.ensure_held() {
192                cancel_tx.cancel();
193                return Err(err);
194            }
195            if !must_persist
196                && let Err(err) = commit_local_rebalance_worker_activation_candidate(
197                    meta,
198                    expected_id.as_ref(),
199                    expected_cancel.as_ref(),
200                    candidate.clone(),
201                )
202            {
203                cancel_tx.cancel();
204                return Err(err);
205            }
206        }
207
208        if must_persist {
209            let save_result = resolve_rebalance_meta_save_result(
210                self.save_rebalance_meta_under_activation_fence(
211                    pool,
212                    &candidate,
213                    "start_rebalance persist activation candidate",
214                    activation_fence.as_ref(),
215                    expected_id.as_ref(),
216                )
217                .await,
218                "start_rebalance persist activation candidate",
219            );
220            if let Err(err) = save_result {
221                cancel_tx.cancel();
222                return Err(err);
223            }
224            let mut rebalance_meta = self.rebalance_meta.write().await;
225            let Some(meta) = rebalance_meta.as_mut() else {
226                cancel_tx.cancel();
227                return Err(Error::ConfigNotFound);
228            };
229            if let Err(err) = commit_local_rebalance_worker_activation_candidate(
230                meta,
231                expected_id.as_ref(),
232                expected_cancel.as_ref(),
233                candidate,
234            ) {
235                cancel_tx.cancel();
236                return Err(err);
237            }
238        }
239        if !must_persist && let Err(err) = activation_fence.ensure_held() {
240            let mut rebalance_meta = self.rebalance_meta.write().await;
241            rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
242            return Err(err);
243        }
244        drop(activation_fence);
245
246        if activation_outcome != RebalanceLocalActivationOutcome::Started {
247            return Ok(must_persist);
248        }
249
250        let participants = if let Some(ref meta) = *self.rebalance_meta.read().await {
251            resolve_rebalance_participants(meta.pool_stats.as_slice(), self.pools.len())
252        } else {
253            debug!(
254                event = EVENT_REBALANCE_STATE,
255                component = LOG_COMPONENT_ECSTORE,
256                subsystem = LOG_SUBSYSTEM_REBALANCE,
257                state = "start_skipped",
258                reason = "metadata_missing",
259                "Skipped rebalance start because metadata is unavailable"
260            );
261            Vec::new()
262        };
263
264        if !participants.iter().any(|participating| *participating) {
265            let mut rebalance_meta = self.rebalance_meta.write().await;
266            rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
267            debug!(
268                event = EVENT_REBALANCE_STATE,
269                component = LOG_COMPONENT_ECSTORE,
270                subsystem = LOG_SUBSYSTEM_REBALANCE,
271                state = "start_skipped",
272                reason = "no_participants",
273                "Skipped rebalance start because no pools are participating"
274            );
275            return Ok(must_persist);
276        }
277
278        #[cfg(test)]
279        let endpoints = self.instance_endpoints().unwrap_or_else(|| self.endpoints());
280        #[cfg(not(test))]
281        let endpoints = self.endpoints();
282
283        let mut workers_started = 0usize;
284        for (idx, participating) in participants.iter().enumerate() {
285            if !*participating {
286                debug!(
287                    event = EVENT_REBALANCE_STATE,
288                    component = LOG_COMPONENT_ECSTORE,
289                    subsystem = LOG_SUBSYSTEM_REBALANCE,
290                    pool_index = idx,
291                    state = "pool_skipped",
292                    reason = "not_participating",
293                    "Skipped rebalance pool"
294                );
295                continue;
296            }
297
298            if !runtime_sources::endpoint_pool_is_local(&endpoints, idx) {
299                debug!(
300                    event = EVENT_REBALANCE_STATE,
301                    component = LOG_COMPONENT_ECSTORE,
302                    subsystem = LOG_SUBSYSTEM_REBALANCE,
303                    pool_index = idx,
304                    state = "pool_skipped",
305                    reason = "not_local",
306                    "Skipped non-local rebalance pool"
307                );
308                continue;
309            }
310
311            let pool_idx = idx;
312            let store = self.clone();
313            let rx_clone = rx.clone();
314            let worker_id = Arc::clone(&expected_id);
315            workers_started += 1;
316            tokio::spawn(async move {
317                if let Err(err) = store.rebalance_buckets(rx_clone, pool_idx, worker_id).await {
318                    error!(
319                        event = EVENT_REBALANCE_STATE,
320                        component = LOG_COMPONENT_ECSTORE,
321                        subsystem = LOG_SUBSYSTEM_REBALANCE,
322                        pool_index = pool_idx,
323                        state = "pool_failed",
324                        error = %err,
325                        "Rebalance pool failed"
326                    );
327                } else {
328                    debug!(
329                        event = EVENT_REBALANCE_STATE,
330                        component = LOG_COMPONENT_ECSTORE,
331                        subsystem = LOG_SUBSYSTEM_REBALANCE,
332                        pool_index = pool_idx,
333                        state = "completed",
334                        "Rebalance pool completed"
335                    );
336                }
337            });
338        }
339
340        if workers_started == 0 {
341            let mut rebalance_meta = self.rebalance_meta.write().await;
342            rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
343            debug!(
344                event = EVENT_REBALANCE_STATE,
345                component = LOG_COMPONENT_ECSTORE,
346                subsystem = LOG_SUBSYSTEM_REBALANCE,
347                state = "start_skipped",
348                reason = "no_local_participants",
349                "Skipped rebalance start because no local pools are participating"
350            );
351            return Ok(must_persist);
352        }
353
354        info!(
355            event = EVENT_REBALANCE_STATE,
356            component = LOG_COMPONENT_ECSTORE,
357            subsystem = LOG_SUBSYSTEM_REBALANCE,
358            state = "started",
359            worker_count = workers_started,
360            "Rebalance started"
361        );
362        Ok(true)
363    }
364
365    #[tracing::instrument(skip(self, rx))]
366    async fn rebalance_buckets(self: &Arc<Self>, rx: CancellationToken, pool_index: usize, rebalance_id: Arc<str>) -> Result<()> {
367        ensure_valid_rebalance_pool_index(self.pools.len(), pool_index)?;
368
369        let (done_tx, mut done_rx) = tokio::sync::mpsc::channel::<Result<()>>(1);
370
371        // Save rebalance metadata periodically
372        let store = self.clone();
373        let save_rebalance_id = Arc::clone(&rebalance_id);
374        let save_task = tokio::spawn(async move {
375            let mut timer = tokio::time::interval_at(Instant::now() + Duration::from_secs(30), Duration::from_secs(10));
376            let mut msg: String;
377            let mut quit = false;
378
379            loop {
380                let mut terminal_state_saved = false;
381                tokio::select! {
382                    result = done_rx.recv() => {
383                        quit = true;
384                        let movement_gate = store.ctx.data_movement_operation_gate();
385                        let movement_guard = movement_gate.write().await;
386                        let terminal_at = store.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
387                        let terminal_event = classify_rebalance_terminal_event(result, terminal_at);
388                        msg = terminal_event.message().to_string();
389                        let previous_meta = store.rebalance_meta.read().await.clone();
390                        let terminal_state_present = {
391                            let mut rebalance_meta = store.rebalance_meta.write().await;
392                            super::control::ensure_rebalance_run_id(
393                                rebalance_meta.as_ref(),
394                                save_rebalance_id.as_ref(),
395                                "apply rebalance terminal event",
396                            )?;
397                            if let Some(meta) = rebalance_meta.as_mut() {
398                                let meta_stopped = meta.stopped_at.is_some();
399                                if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
400                                    if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
401                                        && has_rebalance_cleanup_warnings(pool_stat)
402                                    {
403                                        pool_stat.info.stopping = false;
404                                        pool_stat.info.status = RebalStatus::Failed;
405                                        pool_stat.info.end_time = Some(terminal_at);
406                                        pool_stat.info.last_error = Some(
407                                            pool_stat
408                                                .cleanup_warnings
409                                                .last_message
410                                                .clone()
411                                                .unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
412                                        );
413                                    } else if should_preserve_rebalance_stopped_state(
414                                        meta_stopped,
415                                        pool_stat.info.status,
416                                        &terminal_event,
417                                    ) {
418                                        debug!(
419                                            event = EVENT_REBALANCE_STATE,
420                                            component = LOG_COMPONENT_ECSTORE,
421                                            subsystem = LOG_SUBSYSTEM_REBALANCE,
422                                            pool_index,
423                                            state = "stopped_preserved",
424                                            "Preserved stopped rebalance status"
425                                        );
426                                    } else {
427                                        pool_stat.info.stopping = false;
428                                        apply_rebalance_terminal_event(
429                                            &mut pool_stat.info.status,
430                                            &mut pool_stat.info.end_time,
431                                            &mut pool_stat.info.last_error,
432                                            terminal_event,
433                                            terminal_at,
434                                        );
435                                    }
436                                    true
437                                } else {
438                                    false
439                                }
440                            } else {
441                                false
442                            }
443                        };
444
445                        if terminal_state_present {
446                            if let Err(err) = store
447                                .save_rebalance_stats_inner(
448                                    pool_index,
449                                    RebalSaveOpt::Stats,
450                                    Some(save_rebalance_id.as_ref()),
451                                )
452                                .await
453                            {
454                                let mut rebalance_meta = store.rebalance_meta.write().await;
455                                *rebalance_meta = previous_meta;
456                                drop(movement_guard);
457                                return Err(Error::other(format!(
458                                    "rebalance terminal state save failed for pool {pool_index}: {err}"
459                                )));
460                            }
461                            store.ctx.advance_data_movement_operation_epoch();
462                            terminal_state_saved = true;
463                        }
464                    }
465                    _ = timer.tick() => {
466                        let now = OffsetDateTime::now_utc();
467                        msg = format!("Saving rebalance metadata at {now:?}");
468                    }
469                }
470
471                if !terminal_state_saved
472                    && let Err(err) = store
473                        .save_rebalance_stats_for_id(pool_index, RebalSaveOpt::Stats, save_rebalance_id.as_ref())
474                        .await
475                {
476                    let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}"));
477                    error!("{} err: {:?}", msg, wrapped);
478                    if quit {
479                        return Err(wrapped);
480                    }
481                } else {
482                    debug!(
483                        event = EVENT_REBALANCE_STATE,
484                        component = LOG_COMPONENT_ECSTORE,
485                        subsystem = LOG_SUBSYSTEM_REBALANCE,
486                        pool_index,
487                        state = "metadata_saved",
488                        message = %msg,
489                        "Saved rebalance metadata"
490                    );
491                }
492
493                if quit {
494                    debug!(
495                        event = EVENT_REBALANCE_STATE,
496                        component = LOG_COMPONENT_ECSTORE,
497                        subsystem = LOG_SUBSYSTEM_REBALANCE,
498                        pool_index,
499                        state = "save_task_exiting",
500                        message = %msg,
501                        "Exiting rebalance save task"
502                    );
503                    return Ok(());
504                }
505
506                timer.reset();
507            }
508        });
509
510        info!(
511            event = EVENT_REBALANCE_STATE,
512            component = LOG_COMPONENT_ECSTORE,
513            subsystem = LOG_SUBSYSTEM_REBALANCE,
514            pool_index,
515            state = "pool_started",
516            "Rebalance worker started"
517        );
518        let mut final_result: Result<()> = Ok(());
519        let mut deferred_buckets = HashSet::new();
520        let mut source_cleanup_deferred_attempts = HashMap::new();
521
522        loop {
523            if rx.is_cancelled() {
524                info!(
525                    event = EVENT_REBALANCE_STATE,
526                    component = LOG_COMPONENT_ECSTORE,
527                    subsystem = LOG_SUBSYSTEM_REBALANCE,
528                    pool_index,
529                    state = "pool_stopped",
530                    reason = "cancelled",
531                    "Stopped rebalance worker"
532                );
533                let err = Error::OperationCanceled;
534                final_result = Err(resolve_rebalance_terminal_error(
535                    err.clone(),
536                    send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
537                ));
538                break;
539            }
540
541            let next_bucket = match self.next_rebal_bucket(pool_index, rebalance_id.as_ref()).await {
542                Ok(bucket) => bucket,
543                Err(err) => {
544                    error!(
545                        event = EVENT_REBALANCE_BUCKET,
546                        component = LOG_COMPONENT_ECSTORE,
547                        subsystem = LOG_SUBSYSTEM_REBALANCE,
548                        pool_index,
549                        state = "next_bucket_failed",
550                        error = ?err,
551                        "Rebalance next bucket lookup failed"
552                    );
553                    final_result = Err(resolve_rebalance_terminal_error(
554                        err.clone(),
555                        send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
556                    ));
557                    break;
558                }
559            };
560
561            if let Some(bucket) = next_bucket {
562                debug!(
563                    event = EVENT_REBALANCE_BUCKET,
564                    component = LOG_COMPONENT_ECSTORE,
565                    subsystem = LOG_SUBSYSTEM_REBALANCE,
566                    pool_index,
567                    bucket = %bucket,
568                    state = "started",
569                    "Starting rebalance bucket"
570                );
571
572                let outcome = match resolve_rebalance_bucket_result(
573                    self.rebalance_bucket(rx.clone(), bucket.clone(), pool_index, Arc::clone(&rebalance_id))
574                        .await,
575                    pool_index,
576                    &bucket,
577                ) {
578                    Ok(outcome) => outcome,
579                    Err(err) => {
580                        error!(
581                            event = EVENT_REBALANCE_BUCKET,
582                            component = LOG_COMPONENT_ECSTORE,
583                            subsystem = LOG_SUBSYSTEM_REBALANCE,
584                            pool_index,
585                            bucket = %bucket,
586                            state = "bucket_failed",
587                            error = ?err,
588                            "Rebalance bucket failed"
589                        );
590                        final_result = Err(resolve_rebalance_terminal_error(
591                            err.clone(),
592                            send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
593                        ));
594                        break;
595                    }
596                };
597
598                if let RebalanceBucketOutcome::Deferred { last_error } = outcome {
599                    let source_cleanup_deferred = last_error.starts_with(REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX);
600                    if should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, &bucket, source_cleanup_deferred) {
601                        let err = Error::other(format!(
602                            "rebalance bucket {bucket} deferred repeatedly due to transient object failures: {last_error}"
603                        ));
604                        error!(
605                            event = EVENT_REBALANCE_BUCKET,
606                            component = LOG_COMPONENT_ECSTORE,
607                            subsystem = LOG_SUBSYSTEM_REBALANCE,
608                            pool_index,
609                            bucket = %bucket,
610                            state = "bucket_deferred_repeatedly",
611                            error = ?err,
612                            "Rebalance bucket failed after repeated deferral"
613                        );
614                        final_result = Err(resolve_rebalance_terminal_error(
615                            err.clone(),
616                            send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
617                        ));
618                        break;
619                    }
620
621                    let source_cleanup_attempt = if source_cleanup_deferred {
622                        source_cleanup_defer_attempt(&mut source_cleanup_deferred_attempts, &bucket)
623                    } else {
624                        0
625                    };
626                    warn!(
627                        event = EVENT_REBALANCE_BUCKET,
628                        component = LOG_COMPONENT_ECSTORE,
629                        subsystem = LOG_SUBSYSTEM_REBALANCE,
630                        pool_index,
631                        bucket = %bucket,
632                        state = "deferred",
633                        error = %last_error,
634                        "Deferred rebalance bucket after transient object failures"
635                    );
636                    if let Err(err) = self
637                        .defer_rebalance_bucket(pool_index, bucket.clone(), last_error.clone(), rebalance_id.as_ref())
638                        .await
639                    {
640                        error!(
641                            event = EVENT_REBALANCE_BUCKET,
642                            component = LOG_COMPONENT_ECSTORE,
643                            subsystem = LOG_SUBSYSTEM_REBALANCE,
644                            pool_index,
645                            bucket = %bucket,
646                            state = "defer_failed",
647                            error = ?err,
648                            "Rebalance bucket defer failed"
649                        );
650                        final_result = Err(resolve_rebalance_terminal_error(
651                            err.clone(),
652                            send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
653                        ));
654                        break;
655                    }
656                    if source_cleanup_deferred {
657                        if source_cleanup_attempt >= super::REBALANCE_SOURCE_CLEANUP_MAX_DEFERS {
658                            let err = Error::other(format!(
659                                "rebalance bucket {bucket} source cleanup remained unstable after {} deferrals: {last_error}",
660                                super::REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
661                            ));
662                            warn!(
663                                event = EVENT_REBALANCE_BUCKET,
664                                component = LOG_COMPONENT_ECSTORE,
665                                subsystem = LOG_SUBSYSTEM_REBALANCE,
666                                pool_index,
667                                bucket = %bucket,
668                                state = "source_cleanup_defer_limit",
669                                error = ?err,
670                                "Rebalance bucket failed after repeated source cleanup conflicts"
671                            );
672                            final_result = Err(resolve_rebalance_terminal_error(
673                                err.clone(),
674                                send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
675                            ));
676                            break;
677                        }
678                        if let Err(err) =
679                            super::worker::wait_rebalance_listing_retry(&rx, REBALANCE_LISTING_RETRY_BASE_DELAY).await
680                        {
681                            final_result = Err(resolve_rebalance_terminal_error(
682                                err.clone(),
683                                send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
684                            ));
685                            break;
686                        }
687                    }
688                    continue;
689                }
690
691                debug!(
692                    event = EVENT_REBALANCE_BUCKET,
693                    component = LOG_COMPONENT_ECSTORE,
694                    subsystem = LOG_SUBSYSTEM_REBALANCE,
695                    pool_index,
696                    bucket = %bucket,
697                    state = "completed",
698                    "Completed rebalance bucket"
699                );
700                source_cleanup_deferred_attempts.remove(&bucket);
701                if let Err(err) = self.bucket_rebalance_done(pool_index, bucket, rebalance_id.as_ref()).await {
702                    error!(
703                        event = EVENT_REBALANCE_BUCKET,
704                        component = LOG_COMPONENT_ECSTORE,
705                        subsystem = LOG_SUBSYSTEM_REBALANCE,
706                        pool_index,
707                        state = "bucket_done_mark_failed",
708                        error = ?err,
709                        "Rebalance bucket completion mark failed"
710                    );
711                    final_result = Err(resolve_rebalance_terminal_error(
712                        err.clone(),
713                        send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
714                    ));
715                    break;
716                }
717            } else {
718                debug!(
719                    event = EVENT_REBALANCE_BUCKET,
720                    component = LOG_COMPONENT_ECSTORE,
721                    subsystem = LOG_SUBSYSTEM_REBALANCE,
722                    pool_index,
723                    state = "idle",
724                    reason = "no_bucket_to_rebalance",
725                    "No rebalance bucket available"
726                );
727                break;
728            }
729        }
730
731        info!(
732            event = EVENT_REBALANCE_STATE,
733            component = LOG_COMPONENT_ECSTORE,
734            subsystem = LOG_SUBSYSTEM_REBALANCE,
735            pool_index,
736            state = "pool_done",
737            "Rebalance worker finished"
738        );
739
740        if final_result.is_ok()
741            && let Err(err) = send_rebalance_done_signal(&done_tx, Ok(()), pool_index).await
742        {
743            final_result = Err(err);
744        }
745        drop(done_tx);
746        if let Err(err) = resolve_rebalance_save_task_result(pool_index, save_task.await)
747            && final_result.is_ok()
748        {
749            final_result = Err(err);
750        }
751        debug!(
752            event = EVENT_REBALANCE_STATE,
753            component = LOG_COMPONENT_ECSTORE,
754            subsystem = LOG_SUBSYSTEM_REBALANCE,
755            pool_index,
756            state = "pool_result_returned",
757            "Rebalance worker result returned"
758        );
759        final_result
760    }
761
762    pub(super) async fn check_if_rebalance_done(&self, pool_index: usize, expected_id: &str) -> Result<bool> {
763        let mut rebalance_meta = self.rebalance_meta.write().await;
764        super::control::ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "check rebalance completion")?;
765
766        if let Some(meta) = rebalance_meta.as_mut()
767            && let Some(pool_stat) = meta.pool_stats.get_mut(pool_index)
768        {
769            // Check if the pool's rebalance status is already completed
770            if pool_stat.info.status == RebalStatus::Completed {
771                debug!(
772                    event = EVENT_REBALANCE_STATE,
773                    component = LOG_COMPONENT_ECSTORE,
774                    subsystem = LOG_SUBSYSTEM_REBALANCE,
775                    pool_index,
776                    state = "already_completed",
777                    "Rebalance pool is already completed"
778                );
779                return Ok(true);
780            }
781
782            // Mark pool rebalance as done only after it reaches the PercentFreeGoal.
783            let pfi = if pool_stat.init_capacity == 0 {
784                0.0
785            } else {
786                (pool_stat.init_free_space + pool_stat.bytes) as f64 / pool_stat.init_capacity as f64
787            };
788
789            if !has_deferred_rebalance_error(pool_stat)
790                && !has_rebalance_cleanup_warnings(pool_stat)
791                && rebalance_goal_reached(
792                    pool_stat.init_free_space,
793                    pool_stat.init_capacity,
794                    pool_stat.bytes,
795                    meta.percent_free_goal,
796                )
797            {
798                info!(
799                    event = EVENT_REBALANCE_STATE,
800                    component = LOG_COMPONENT_ECSTORE,
801                    subsystem = LOG_SUBSYSTEM_REBALANCE,
802                    pool_index,
803                    state = "completion_ready",
804                    percent_free = pfi,
805                    "Rebalance pool reached completion goal"
806                );
807                return Ok(true);
808            }
809        }
810
811        Ok(false)
812    }
813}
814
815impl ECStore {
816    #[tracing::instrument(skip(self))]
817    pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
818        let movement_gate = self.ctx.data_movement_operation_gate();
819        let _movement_guard = movement_gate.write().await;
820        self.save_rebalance_stats_inner(pool_idx, opt, None).await
821    }
822
823    pub async fn save_rebalance_stats_for_id(&self, pool_idx: usize, opt: RebalSaveOpt, expected_id: &str) -> Result<()> {
824        let movement_gate = self.ctx.data_movement_operation_gate();
825        let _movement_guard = movement_gate.write().await;
826        self.save_rebalance_stats_inner(pool_idx, opt, Some(expected_id)).await
827    }
828
829    pub(super) async fn save_rebalance_stats_inner(
830        &self,
831        pool_idx: usize,
832        opt: RebalSaveOpt,
833        expected_id: Option<&str>,
834    ) -> Result<()> {
835        let now = match opt {
836            RebalSaveOpt::Stats => OffsetDateTime::now_utc(),
837            RebalSaveOpt::StoppedAt => self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await,
838        };
839        let meta_to_save = {
840            let mut rebalance_meta = self.rebalance_meta.write().await;
841            if let Some(expected_id) = expected_id {
842                super::control::ensure_rebalance_run_id(rebalance_meta.as_ref(), expected_id, "save rebalance stats")?;
843            }
844            let Some(meta) = rebalance_meta.as_mut() else {
845                return Ok(());
846            };
847
848            apply_rebalance_save_option(meta, pool_idx, opt, now);
849            meta.clone()
850        };
851
852        let pool = clone_first_arc(&self.pools, "save_rebalance_stats: no pools available")?;
853
854        debug!(
855            event = EVENT_REBALANCE_STATE,
856            component = LOG_COMPONENT_ECSTORE,
857            subsystem = LOG_SUBSYSTEM_REBALANCE,
858            pool_index = pool_idx,
859            save_opt = ?opt,
860            state = "metadata_save_requested",
861            "Rebalance metadata save requested"
862        );
863        let stage = format!("save_rebalance_stats for pool {pool_idx} opt {opt:?}");
864        let save_result = match expected_id {
865            Some(expected_id) => {
866                self.save_rebalance_meta_for_id_with_merge(pool, &meta_to_save, stage.as_str(), expected_id)
867                    .await
868            }
869            None => self.save_rebalance_meta_with_merge(pool, &meta_to_save, stage.as_str()).await,
870        };
871        resolve_rebalance_meta_save_result(save_result, stage.as_str())?;
872
873        Ok(())
874    }
875}