1use super::control::RebalanceWorkerActivationFence;
2use super::meta::{
3 apply_rebalance_save_option, apply_rebalance_terminal_event, classify_rebalance_terminal_event, clone_first_arc,
4 complete_rebalance_pools_at_goal, complete_rebalance_pools_with_empty_queue, ensure_valid_rebalance_pool_index,
5 has_deferred_rebalance_error, has_rebalance_cleanup_warnings, is_rebalance_in_progress, rebalance_goal_reached,
6 resolve_rebalance_participants, should_preserve_rebalance_stopped_state, should_skip_start_rebalance,
7 validate_start_rebalance_state,
8};
9use super::worker::{
10 resolve_rebalance_bucket_result, resolve_rebalance_meta_save_result, resolve_rebalance_save_task_result,
11 resolve_rebalance_terminal_error, send_rebalance_done_signal,
12};
13use super::{
14 EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
15 REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, RebalSaveOpt, RebalStatus,
16 RebalanceBucketOutcome,
17};
18use crate::error::{Error, Result};
19use crate::runtime::sources as runtime_sources;
20use crate::store::ECStore;
21use std::collections::HashMap;
22use std::collections::HashSet;
23use std::sync::Arc;
24use time::OffsetDateTime;
25use tokio::time::{Duration, Instant};
26use tokio_util::sync::CancellationToken;
27use tracing::{debug, error, info, warn};
28
29pub(super) fn should_fail_repeated_rebalance_bucket_defer(
30 deferred_buckets: &mut HashSet<String>,
31 bucket: &str,
32 source_cleanup_deferred: bool,
33) -> bool {
34 !source_cleanup_deferred && !deferred_buckets.insert(bucket.to_string())
35}
36
37pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap<String, usize>, bucket: &str) -> usize {
38 let attempts = deferred_attempts.entry(bucket.to_string()).or_default();
39 *attempts = attempts.saturating_add(1);
40 *attempts
41}
42
43#[derive(Clone, Copy, Debug, PartialEq, Eq)]
44pub(super) enum RebalanceLocalActivationOutcome {
45 Started,
46 NotStartedTerminal,
47}
48
49pub(super) fn commit_local_rebalance_worker_activation(
50 meta: &mut super::RebalanceMeta,
51 expected_id: &str,
52 cancel: CancellationToken,
53) -> Result<RebalanceLocalActivationOutcome> {
54 if meta.id != expected_id {
55 return Err(Error::other(format!(
56 "rebalance metadata changed before local worker activation: expected {expected_id}, found {}",
57 meta.id
58 )));
59 }
60 if meta.stopped_at.is_some() || meta.stop_requested || !is_rebalance_in_progress(meta) {
61 return Ok(RebalanceLocalActivationOutcome::NotStartedTerminal);
62 }
63 meta.cancel = Some(cancel);
64 Ok(RebalanceLocalActivationOutcome::Started)
65}
66
67pub(super) fn stage_local_rebalance_worker_activation(
68 meta: &super::RebalanceMeta,
69 expected_id: &str,
70 cancel: CancellationToken,
71 now: OffsetDateTime,
72) -> Result<(super::RebalanceMeta, RebalanceLocalActivationOutcome, bool)> {
73 let mut candidate = meta.clone();
74 let completed_at_goal = complete_rebalance_pools_at_goal(&mut candidate, now);
75 let completed_empty_queue = complete_rebalance_pools_with_empty_queue(&mut candidate, now);
76 let outcome = commit_local_rebalance_worker_activation(&mut candidate, expected_id, cancel)?;
77 let must_persist =
78 completed_at_goal || completed_empty_queue || outcome == RebalanceLocalActivationOutcome::NotStartedTerminal;
79 Ok((candidate, outcome, must_persist))
80}
81
82pub(super) fn commit_local_rebalance_worker_activation_candidate(
83 current: &mut super::RebalanceMeta,
84 expected_id: &str,
85 expected_cancel: Option<&CancellationToken>,
86 candidate: super::RebalanceMeta,
87) -> Result<()> {
88 if current.id != expected_id || candidate.id != expected_id {
89 return Err(Error::other(format!(
90 "rebalance metadata changed before local worker activation commit: expected {expected_id}, found {}",
91 current.id
92 )));
93 }
94 if !Arc::ptr_eq(¤t.activation_gate, &candidate.activation_gate) {
95 return Err(Error::other(format!(
96 "rebalance activation gate changed before local worker activation commit: {expected_id}"
97 )));
98 }
99 if current.cancel.as_ref() != expected_cancel {
100 return Err(Error::other(format!(
101 "rebalance worker token changed before local worker activation commit: {expected_id}"
102 )));
103 }
104 *current = candidate;
105 Ok(())
106}
107
108pub(super) fn rollback_local_rebalance_worker_activation(
109 meta: Option<&mut super::RebalanceMeta>,
110 expected_id: &str,
111 activation_token: &CancellationToken,
112) -> bool {
113 let Some(meta) = meta else {
114 return false;
115 };
116 if meta.id != expected_id || meta.cancel.as_ref() != Some(activation_token) {
117 return false;
118 }
119 if let Some(cancel) = meta.cancel.take() {
120 cancel.cancel();
121 return true;
122 }
123 false
124}
125
126impl ECStore {
127 #[tracing::instrument(skip_all)]
128 pub async fn start_rebalance(self: &Arc<Self>) -> Result<()> {
129 let _start_guard = self.start_gate.lock().await;
130 let _activation_guard = self.rebalance_activation_write_guard(None, "start rebalance").await?;
131 let movement_gate = self.ctx.data_movement_operation_gate();
132 let _movement_guard = movement_gate.write().await;
133 if self.start_rebalance_under_gate().await? {
134 self.ctx.advance_data_movement_operation_epoch();
135 }
136 Ok(())
137 }
138
139 pub(super) async fn start_rebalance_under_gate(self: &Arc<Self>) -> Result<bool> {
140 info!(
141 event = EVENT_REBALANCE_STATE,
142 component = LOG_COMPONENT_ECSTORE,
143 subsystem = LOG_SUBSYSTEM_REBALANCE,
144 state = "starting",
145 "Starting rebalance"
146 );
147 let expected_id: Arc<str> = {
148 let rebalance_meta = self.rebalance_meta.read().await;
149 Arc::from(rebalance_meta.as_ref().ok_or(Error::ConfigNotFound)?.id.as_str())
150 };
151 let pool = clone_first_arc(self.pools.as_slice(), "start_rebalance: no pools available")?;
152 let activation_fence = match self
153 .fence_rebalance_worker_activation(pool.clone(), expected_id.as_ref())
154 .await?
155 {
156 RebalanceWorkerActivationFence::Ready(fence) => fence,
157 RebalanceWorkerActivationFence::NotStartedTerminal => return Ok(false),
158 };
159
160 let decommission_running = self.is_decommission_running().await;
161
162 let cancel_tx = CancellationToken::new();
163 let rx = cancel_tx.clone();
164 let activation_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
165 let activation_outcome;
166 let candidate;
167 let expected_cancel;
168 let must_persist;
169
170 {
171 let mut rebalance_meta = self.rebalance_meta.write().await;
172 validate_start_rebalance_state(decommission_running, rebalance_meta.is_some())?;
173
174 let Some(meta) = rebalance_meta.as_mut() else {
175 return Err(Error::ConfigNotFound);
176 };
177 if should_skip_start_rebalance(meta.cancel.is_some(), is_rebalance_in_progress(meta)) {
178 debug!(
179 event = EVENT_REBALANCE_STATE,
180 component = LOG_COMPONENT_ECSTORE,
181 subsystem = LOG_SUBSYSTEM_REBALANCE,
182 state = "start_skipped",
183 reason = "already_in_progress",
184 "Skipped duplicate rebalance start"
185 );
186 return Ok(false);
187 }
188 expected_cancel = meta.cancel.clone();
189 (candidate, activation_outcome, must_persist) =
190 stage_local_rebalance_worker_activation(meta, expected_id.as_ref(), cancel_tx.clone(), activation_at)?;
191 if let Err(err) = activation_fence.ensure_held() {
192 cancel_tx.cancel();
193 return Err(err);
194 }
195 if !must_persist
196 && let Err(err) = commit_local_rebalance_worker_activation_candidate(
197 meta,
198 expected_id.as_ref(),
199 expected_cancel.as_ref(),
200 candidate.clone(),
201 )
202 {
203 cancel_tx.cancel();
204 return Err(err);
205 }
206 }
207
208 if must_persist {
209 let save_result = resolve_rebalance_meta_save_result(
210 self.save_rebalance_meta_under_activation_fence(
211 pool,
212 &candidate,
213 "start_rebalance persist activation candidate",
214 activation_fence.as_ref(),
215 expected_id.as_ref(),
216 )
217 .await,
218 "start_rebalance persist activation candidate",
219 );
220 if let Err(err) = save_result {
221 cancel_tx.cancel();
222 return Err(err);
223 }
224 let mut rebalance_meta = self.rebalance_meta.write().await;
225 let Some(meta) = rebalance_meta.as_mut() else {
226 cancel_tx.cancel();
227 return Err(Error::ConfigNotFound);
228 };
229 if let Err(err) = commit_local_rebalance_worker_activation_candidate(
230 meta,
231 expected_id.as_ref(),
232 expected_cancel.as_ref(),
233 candidate,
234 ) {
235 cancel_tx.cancel();
236 return Err(err);
237 }
238 }
239 if !must_persist && let Err(err) = activation_fence.ensure_held() {
240 let mut rebalance_meta = self.rebalance_meta.write().await;
241 rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
242 return Err(err);
243 }
244 drop(activation_fence);
245
246 if activation_outcome != RebalanceLocalActivationOutcome::Started {
247 return Ok(must_persist);
248 }
249
250 let participants = if let Some(ref meta) = *self.rebalance_meta.read().await {
251 resolve_rebalance_participants(meta.pool_stats.as_slice(), self.pools.len())
252 } else {
253 debug!(
254 event = EVENT_REBALANCE_STATE,
255 component = LOG_COMPONENT_ECSTORE,
256 subsystem = LOG_SUBSYSTEM_REBALANCE,
257 state = "start_skipped",
258 reason = "metadata_missing",
259 "Skipped rebalance start because metadata is unavailable"
260 );
261 Vec::new()
262 };
263
264 if !participants.iter().any(|participating| *participating) {
265 let mut rebalance_meta = self.rebalance_meta.write().await;
266 rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
267 debug!(
268 event = EVENT_REBALANCE_STATE,
269 component = LOG_COMPONENT_ECSTORE,
270 subsystem = LOG_SUBSYSTEM_REBALANCE,
271 state = "start_skipped",
272 reason = "no_participants",
273 "Skipped rebalance start because no pools are participating"
274 );
275 return Ok(must_persist);
276 }
277
278 #[cfg(test)]
279 let endpoints = self.instance_endpoints().unwrap_or_else(|| self.endpoints());
280 #[cfg(not(test))]
281 let endpoints = self.endpoints();
282
283 let mut workers_started = 0usize;
284 for (idx, participating) in participants.iter().enumerate() {
285 if !*participating {
286 debug!(
287 event = EVENT_REBALANCE_STATE,
288 component = LOG_COMPONENT_ECSTORE,
289 subsystem = LOG_SUBSYSTEM_REBALANCE,
290 pool_index = idx,
291 state = "pool_skipped",
292 reason = "not_participating",
293 "Skipped rebalance pool"
294 );
295 continue;
296 }
297
298 if !runtime_sources::endpoint_pool_is_local(&endpoints, idx) {
299 debug!(
300 event = EVENT_REBALANCE_STATE,
301 component = LOG_COMPONENT_ECSTORE,
302 subsystem = LOG_SUBSYSTEM_REBALANCE,
303 pool_index = idx,
304 state = "pool_skipped",
305 reason = "not_local",
306 "Skipped non-local rebalance pool"
307 );
308 continue;
309 }
310
311 let pool_idx = idx;
312 let store = self.clone();
313 let rx_clone = rx.clone();
314 let worker_id = Arc::clone(&expected_id);
315 workers_started += 1;
316 tokio::spawn(async move {
317 if let Err(err) = store.rebalance_buckets(rx_clone, pool_idx, worker_id).await {
318 error!(
319 event = EVENT_REBALANCE_STATE,
320 component = LOG_COMPONENT_ECSTORE,
321 subsystem = LOG_SUBSYSTEM_REBALANCE,
322 pool_index = pool_idx,
323 state = "pool_failed",
324 error = %err,
325 "Rebalance pool failed"
326 );
327 } else {
328 debug!(
329 event = EVENT_REBALANCE_STATE,
330 component = LOG_COMPONENT_ECSTORE,
331 subsystem = LOG_SUBSYSTEM_REBALANCE,
332 pool_index = pool_idx,
333 state = "completed",
334 "Rebalance pool completed"
335 );
336 }
337 });
338 }
339
340 if workers_started == 0 {
341 let mut rebalance_meta = self.rebalance_meta.write().await;
342 rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
343 debug!(
344 event = EVENT_REBALANCE_STATE,
345 component = LOG_COMPONENT_ECSTORE,
346 subsystem = LOG_SUBSYSTEM_REBALANCE,
347 state = "start_skipped",
348 reason = "no_local_participants",
349 "Skipped rebalance start because no local pools are participating"
350 );
351 return Ok(must_persist);
352 }
353
354 info!(
355 event = EVENT_REBALANCE_STATE,
356 component = LOG_COMPONENT_ECSTORE,
357 subsystem = LOG_SUBSYSTEM_REBALANCE,
358 state = "started",
359 worker_count = workers_started,
360 "Rebalance started"
361 );
362 Ok(true)
363 }
364
365 #[tracing::instrument(skip(self, rx))]
366 async fn rebalance_buckets(self: &Arc<Self>, rx: CancellationToken, pool_index: usize, rebalance_id: Arc<str>) -> Result<()> {
367 ensure_valid_rebalance_pool_index(self.pools.len(), pool_index)?;
368
369 let (done_tx, mut done_rx) = tokio::sync::mpsc::channel::<Result<()>>(1);
370
371 let store = self.clone();
373 let save_rebalance_id = Arc::clone(&rebalance_id);
374 let save_task = tokio::spawn(async move {
375 let mut timer = tokio::time::interval_at(Instant::now() + Duration::from_secs(30), Duration::from_secs(10));
376 let mut msg: String;
377 let mut quit = false;
378
379 loop {
380 let mut terminal_state_saved = false;
381 tokio::select! {
382 result = done_rx.recv() => {
383 quit = true;
384 let movement_gate = store.ctx.data_movement_operation_gate();
385 let movement_guard = movement_gate.write().await;
386 let terminal_at = store.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
387 let terminal_event = classify_rebalance_terminal_event(result, terminal_at);
388 msg = terminal_event.message().to_string();
389 let previous_meta = store.rebalance_meta.read().await.clone();
390 let terminal_state_present = {
391 let mut rebalance_meta = store.rebalance_meta.write().await;
392 super::control::ensure_rebalance_run_id(
393 rebalance_meta.as_ref(),
394 save_rebalance_id.as_ref(),
395 "apply rebalance terminal event",
396 )?;
397 if let Some(meta) = rebalance_meta.as_mut() {
398 let meta_stopped = meta.stopped_at.is_some();
399 if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
400 if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
401 && has_rebalance_cleanup_warnings(pool_stat)
402 {
403 pool_stat.info.stopping = false;
404 pool_stat.info.status = RebalStatus::Failed;
405 pool_stat.info.end_time = Some(terminal_at);
406 pool_stat.info.last_error = Some(
407 pool_stat
408 .cleanup_warnings
409 .last_message
410 .clone()
411 .unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
412 );
413 } else if should_preserve_rebalance_stopped_state(
414 meta_stopped,
415 pool_stat.info.status,
416 &terminal_event,
417 ) {
418 debug!(
419 event = EVENT_REBALANCE_STATE,
420 component = LOG_COMPONENT_ECSTORE,
421 subsystem = LOG_SUBSYSTEM_REBALANCE,
422 pool_index,
423 state = "stopped_preserved",
424 "Preserved stopped rebalance status"
425 );
426 } else {
427 pool_stat.info.stopping = false;
428 apply_rebalance_terminal_event(
429 &mut pool_stat.info.status,
430 &mut pool_stat.info.end_time,
431 &mut pool_stat.info.last_error,
432 terminal_event,
433 terminal_at,
434 );
435 }
436 true
437 } else {
438 false
439 }
440 } else {
441 false
442 }
443 };
444
445 if terminal_state_present {
446 if let Err(err) = store
447 .save_rebalance_stats_inner(
448 pool_index,
449 RebalSaveOpt::Stats,
450 Some(save_rebalance_id.as_ref()),
451 )
452 .await
453 {
454 let mut rebalance_meta = store.rebalance_meta.write().await;
455 *rebalance_meta = previous_meta;
456 drop(movement_guard);
457 return Err(Error::other(format!(
458 "rebalance terminal state save failed for pool {pool_index}: {err}"
459 )));
460 }
461 store.ctx.advance_data_movement_operation_epoch();
462 terminal_state_saved = true;
463 }
464 }
465 _ = timer.tick() => {
466 let now = OffsetDateTime::now_utc();
467 msg = format!("Saving rebalance metadata at {now:?}");
468 }
469 }
470
471 if !terminal_state_saved
472 && let Err(err) = store
473 .save_rebalance_stats_for_id(pool_index, RebalSaveOpt::Stats, save_rebalance_id.as_ref())
474 .await
475 {
476 let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}"));
477 error!("{} err: {:?}", msg, wrapped);
478 if quit {
479 return Err(wrapped);
480 }
481 } else {
482 debug!(
483 event = EVENT_REBALANCE_STATE,
484 component = LOG_COMPONENT_ECSTORE,
485 subsystem = LOG_SUBSYSTEM_REBALANCE,
486 pool_index,
487 state = "metadata_saved",
488 message = %msg,
489 "Saved rebalance metadata"
490 );
491 }
492
493 if quit {
494 debug!(
495 event = EVENT_REBALANCE_STATE,
496 component = LOG_COMPONENT_ECSTORE,
497 subsystem = LOG_SUBSYSTEM_REBALANCE,
498 pool_index,
499 state = "save_task_exiting",
500 message = %msg,
501 "Exiting rebalance save task"
502 );
503 return Ok(());
504 }
505
506 timer.reset();
507 }
508 });
509
510 info!(
511 event = EVENT_REBALANCE_STATE,
512 component = LOG_COMPONENT_ECSTORE,
513 subsystem = LOG_SUBSYSTEM_REBALANCE,
514 pool_index,
515 state = "pool_started",
516 "Rebalance worker started"
517 );
518 let mut final_result: Result<()> = Ok(());
519 let mut deferred_buckets = HashSet::new();
520 let mut source_cleanup_deferred_attempts = HashMap::new();
521
522 loop {
523 if rx.is_cancelled() {
524 info!(
525 event = EVENT_REBALANCE_STATE,
526 component = LOG_COMPONENT_ECSTORE,
527 subsystem = LOG_SUBSYSTEM_REBALANCE,
528 pool_index,
529 state = "pool_stopped",
530 reason = "cancelled",
531 "Stopped rebalance worker"
532 );
533 let err = Error::OperationCanceled;
534 final_result = Err(resolve_rebalance_terminal_error(
535 err.clone(),
536 send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
537 ));
538 break;
539 }
540
541 let next_bucket = match self.next_rebal_bucket(pool_index, rebalance_id.as_ref()).await {
542 Ok(bucket) => bucket,
543 Err(err) => {
544 error!(
545 event = EVENT_REBALANCE_BUCKET,
546 component = LOG_COMPONENT_ECSTORE,
547 subsystem = LOG_SUBSYSTEM_REBALANCE,
548 pool_index,
549 state = "next_bucket_failed",
550 error = ?err,
551 "Rebalance next bucket lookup failed"
552 );
553 final_result = Err(resolve_rebalance_terminal_error(
554 err.clone(),
555 send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
556 ));
557 break;
558 }
559 };
560
561 if let Some(bucket) = next_bucket {
562 debug!(
563 event = EVENT_REBALANCE_BUCKET,
564 component = LOG_COMPONENT_ECSTORE,
565 subsystem = LOG_SUBSYSTEM_REBALANCE,
566 pool_index,
567 bucket = %bucket,
568 state = "started",
569 "Starting rebalance bucket"
570 );
571
572 let outcome = match resolve_rebalance_bucket_result(
573 self.rebalance_bucket(rx.clone(), bucket.clone(), pool_index, Arc::clone(&rebalance_id))
574 .await,
575 pool_index,
576 &bucket,
577 ) {
578 Ok(outcome) => outcome,
579 Err(err) => {
580 error!(
581 event = EVENT_REBALANCE_BUCKET,
582 component = LOG_COMPONENT_ECSTORE,
583 subsystem = LOG_SUBSYSTEM_REBALANCE,
584 pool_index,
585 bucket = %bucket,
586 state = "bucket_failed",
587 error = ?err,
588 "Rebalance bucket failed"
589 );
590 final_result = Err(resolve_rebalance_terminal_error(
591 err.clone(),
592 send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
593 ));
594 break;
595 }
596 };
597
598 if let RebalanceBucketOutcome::Deferred { last_error } = outcome {
599 let source_cleanup_deferred = last_error.starts_with(REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX);
600 if should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, &bucket, source_cleanup_deferred) {
601 let err = Error::other(format!(
602 "rebalance bucket {bucket} deferred repeatedly due to transient object failures: {last_error}"
603 ));
604 error!(
605 event = EVENT_REBALANCE_BUCKET,
606 component = LOG_COMPONENT_ECSTORE,
607 subsystem = LOG_SUBSYSTEM_REBALANCE,
608 pool_index,
609 bucket = %bucket,
610 state = "bucket_deferred_repeatedly",
611 error = ?err,
612 "Rebalance bucket failed after repeated deferral"
613 );
614 final_result = Err(resolve_rebalance_terminal_error(
615 err.clone(),
616 send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
617 ));
618 break;
619 }
620
621 let source_cleanup_attempt = if source_cleanup_deferred {
622 source_cleanup_defer_attempt(&mut source_cleanup_deferred_attempts, &bucket)
623 } else {
624 0
625 };
626 warn!(
627 event = EVENT_REBALANCE_BUCKET,
628 component = LOG_COMPONENT_ECSTORE,
629 subsystem = LOG_SUBSYSTEM_REBALANCE,
630 pool_index,
631 bucket = %bucket,
632 state = "deferred",
633 error = %last_error,
634 "Deferred rebalance bucket after transient object failures"
635 );
636 if let Err(err) = self
637 .defer_rebalance_bucket(pool_index, bucket.clone(), last_error.clone(), rebalance_id.as_ref())
638 .await
639 {
640 error!(
641 event = EVENT_REBALANCE_BUCKET,
642 component = LOG_COMPONENT_ECSTORE,
643 subsystem = LOG_SUBSYSTEM_REBALANCE,
644 pool_index,
645 bucket = %bucket,
646 state = "defer_failed",
647 error = ?err,
648 "Rebalance bucket defer failed"
649 );
650 final_result = Err(resolve_rebalance_terminal_error(
651 err.clone(),
652 send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
653 ));
654 break;
655 }
656 if source_cleanup_deferred {
657 if source_cleanup_attempt >= super::REBALANCE_SOURCE_CLEANUP_MAX_DEFERS {
658 let err = Error::other(format!(
659 "rebalance bucket {bucket} source cleanup remained unstable after {} deferrals: {last_error}",
660 super::REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
661 ));
662 warn!(
663 event = EVENT_REBALANCE_BUCKET,
664 component = LOG_COMPONENT_ECSTORE,
665 subsystem = LOG_SUBSYSTEM_REBALANCE,
666 pool_index,
667 bucket = %bucket,
668 state = "source_cleanup_defer_limit",
669 error = ?err,
670 "Rebalance bucket failed after repeated source cleanup conflicts"
671 );
672 final_result = Err(resolve_rebalance_terminal_error(
673 err.clone(),
674 send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
675 ));
676 break;
677 }
678 if let Err(err) =
679 super::worker::wait_rebalance_listing_retry(&rx, REBALANCE_LISTING_RETRY_BASE_DELAY).await
680 {
681 final_result = Err(resolve_rebalance_terminal_error(
682 err.clone(),
683 send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
684 ));
685 break;
686 }
687 }
688 continue;
689 }
690
691 debug!(
692 event = EVENT_REBALANCE_BUCKET,
693 component = LOG_COMPONENT_ECSTORE,
694 subsystem = LOG_SUBSYSTEM_REBALANCE,
695 pool_index,
696 bucket = %bucket,
697 state = "completed",
698 "Completed rebalance bucket"
699 );
700 source_cleanup_deferred_attempts.remove(&bucket);
701 if let Err(err) = self.bucket_rebalance_done(pool_index, bucket, rebalance_id.as_ref()).await {
702 error!(
703 event = EVENT_REBALANCE_BUCKET,
704 component = LOG_COMPONENT_ECSTORE,
705 subsystem = LOG_SUBSYSTEM_REBALANCE,
706 pool_index,
707 state = "bucket_done_mark_failed",
708 error = ?err,
709 "Rebalance bucket completion mark failed"
710 );
711 final_result = Err(resolve_rebalance_terminal_error(
712 err.clone(),
713 send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
714 ));
715 break;
716 }
717 } else {
718 debug!(
719 event = EVENT_REBALANCE_BUCKET,
720 component = LOG_COMPONENT_ECSTORE,
721 subsystem = LOG_SUBSYSTEM_REBALANCE,
722 pool_index,
723 state = "idle",
724 reason = "no_bucket_to_rebalance",
725 "No rebalance bucket available"
726 );
727 break;
728 }
729 }
730
731 info!(
732 event = EVENT_REBALANCE_STATE,
733 component = LOG_COMPONENT_ECSTORE,
734 subsystem = LOG_SUBSYSTEM_REBALANCE,
735 pool_index,
736 state = "pool_done",
737 "Rebalance worker finished"
738 );
739
740 if final_result.is_ok()
741 && let Err(err) = send_rebalance_done_signal(&done_tx, Ok(()), pool_index).await
742 {
743 final_result = Err(err);
744 }
745 drop(done_tx);
746 if let Err(err) = resolve_rebalance_save_task_result(pool_index, save_task.await)
747 && final_result.is_ok()
748 {
749 final_result = Err(err);
750 }
751 debug!(
752 event = EVENT_REBALANCE_STATE,
753 component = LOG_COMPONENT_ECSTORE,
754 subsystem = LOG_SUBSYSTEM_REBALANCE,
755 pool_index,
756 state = "pool_result_returned",
757 "Rebalance worker result returned"
758 );
759 final_result
760 }
761
762 pub(super) async fn check_if_rebalance_done(&self, pool_index: usize, expected_id: &str) -> Result<bool> {
763 let mut rebalance_meta = self.rebalance_meta.write().await;
764 super::control::ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "check rebalance completion")?;
765
766 if let Some(meta) = rebalance_meta.as_mut()
767 && let Some(pool_stat) = meta.pool_stats.get_mut(pool_index)
768 {
769 if pool_stat.info.status == RebalStatus::Completed {
771 debug!(
772 event = EVENT_REBALANCE_STATE,
773 component = LOG_COMPONENT_ECSTORE,
774 subsystem = LOG_SUBSYSTEM_REBALANCE,
775 pool_index,
776 state = "already_completed",
777 "Rebalance pool is already completed"
778 );
779 return Ok(true);
780 }
781
782 let pfi = if pool_stat.init_capacity == 0 {
784 0.0
785 } else {
786 (pool_stat.init_free_space + pool_stat.bytes) as f64 / pool_stat.init_capacity as f64
787 };
788
789 if !has_deferred_rebalance_error(pool_stat)
790 && !has_rebalance_cleanup_warnings(pool_stat)
791 && rebalance_goal_reached(
792 pool_stat.init_free_space,
793 pool_stat.init_capacity,
794 pool_stat.bytes,
795 meta.percent_free_goal,
796 )
797 {
798 info!(
799 event = EVENT_REBALANCE_STATE,
800 component = LOG_COMPONENT_ECSTORE,
801 subsystem = LOG_SUBSYSTEM_REBALANCE,
802 pool_index,
803 state = "completion_ready",
804 percent_free = pfi,
805 "Rebalance pool reached completion goal"
806 );
807 return Ok(true);
808 }
809 }
810
811 Ok(false)
812 }
813}
814
815impl ECStore {
816 #[tracing::instrument(skip(self))]
817 pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
818 let movement_gate = self.ctx.data_movement_operation_gate();
819 let _movement_guard = movement_gate.write().await;
820 self.save_rebalance_stats_inner(pool_idx, opt, None).await
821 }
822
823 pub async fn save_rebalance_stats_for_id(&self, pool_idx: usize, opt: RebalSaveOpt, expected_id: &str) -> Result<()> {
824 let movement_gate = self.ctx.data_movement_operation_gate();
825 let _movement_guard = movement_gate.write().await;
826 self.save_rebalance_stats_inner(pool_idx, opt, Some(expected_id)).await
827 }
828
829 pub(super) async fn save_rebalance_stats_inner(
830 &self,
831 pool_idx: usize,
832 opt: RebalSaveOpt,
833 expected_id: Option<&str>,
834 ) -> Result<()> {
835 let now = match opt {
836 RebalSaveOpt::Stats => OffsetDateTime::now_utc(),
837 RebalSaveOpt::StoppedAt => self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await,
838 };
839 let meta_to_save = {
840 let mut rebalance_meta = self.rebalance_meta.write().await;
841 if let Some(expected_id) = expected_id {
842 super::control::ensure_rebalance_run_id(rebalance_meta.as_ref(), expected_id, "save rebalance stats")?;
843 }
844 let Some(meta) = rebalance_meta.as_mut() else {
845 return Ok(());
846 };
847
848 apply_rebalance_save_option(meta, pool_idx, opt, now);
849 meta.clone()
850 };
851
852 let pool = clone_first_arc(&self.pools, "save_rebalance_stats: no pools available")?;
853
854 debug!(
855 event = EVENT_REBALANCE_STATE,
856 component = LOG_COMPONENT_ECSTORE,
857 subsystem = LOG_SUBSYSTEM_REBALANCE,
858 pool_index = pool_idx,
859 save_opt = ?opt,
860 state = "metadata_save_requested",
861 "Rebalance metadata save requested"
862 );
863 let stage = format!("save_rebalance_stats for pool {pool_idx} opt {opt:?}");
864 let save_result = match expected_id {
865 Some(expected_id) => {
866 self.save_rebalance_meta_for_id_with_merge(pool, &meta_to_save, stage.as_str(), expected_id)
867 .await
868 }
869 None => self.save_rebalance_meta_with_merge(pool, &meta_to_save, stage.as_str()).await,
870 };
871 resolve_rebalance_meta_save_result(save_result, stage.as_str())?;
872
873 Ok(())
874 }
875}