1use super::control::RebalanceWorkerActivationFence;
2use super::meta::{
3 apply_rebalance_save_option, apply_rebalance_terminal_event, classify_rebalance_terminal_event, clone_first_arc,
4 complete_rebalance_pools_at_goal, complete_rebalance_pools_with_empty_queue, ensure_valid_rebalance_pool_index,
5 has_deferred_rebalance_error, has_rebalance_cleanup_warnings, is_rebalance_in_progress, rebalance_goal_reached,
6 resolve_rebalance_participants, should_preserve_rebalance_stopped_state, should_skip_start_rebalance,
7 validate_start_rebalance_state,
8};
9use super::worker::{
10 rebalance_max_attempts, resolve_rebalance_bucket_result, resolve_rebalance_meta_save_result,
11 resolve_rebalance_save_task_result, resolve_rebalance_terminal_error, retry_rebalance_metadata_access,
12 send_rebalance_done_signal,
13};
14use super::{
15 EVENT_REBALANCE_BUCKET, EVENT_REBALANCE_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REBALANCE,
16 REBALANCE_LISTING_RETRY_BASE_DELAY, REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX, REBALANCE_SOURCE_CLEANUP_MAX_DEFERS,
17 RebalSaveOpt, RebalStatus, RebalanceBucketOutcome, RebalanceDeferKind,
18};
19use crate::error::{Error, Result};
20use crate::runtime::sources as runtime_sources;
21use crate::store::ECStore;
22use std::collections::HashMap;
23use std::collections::HashSet;
24use std::sync::Arc;
25use time::OffsetDateTime;
26use tokio::time::{Duration, Instant};
27use tokio_util::sync::CancellationToken;
28use tracing::{debug, error, info, warn};
29
30pub(super) fn should_fail_repeated_rebalance_bucket_defer(
31 deferred_buckets: &mut HashSet<String>,
32 bucket: &str,
33 source_cleanup_deferred: bool,
34) -> bool {
35 !source_cleanup_deferred && !deferred_buckets.insert(bucket.to_string())
36}
37
38pub(super) fn source_cleanup_defer_attempt(deferred_attempts: &mut HashMap<String, usize>, bucket: &str) -> usize {
39 let attempts = deferred_attempts.entry(bucket.to_string()).or_default();
40 *attempts = attempts.saturating_add(1);
41 *attempts
42}
43
44pub(super) fn reached_rebalance_source_cleanup_defer_limit(attempt: usize) -> bool {
47 attempt >= REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
48}
49
50#[derive(Clone, Copy, Debug, PartialEq, Eq)]
51pub(super) enum RebalanceLocalActivationOutcome {
52 Started,
53 NotStartedTerminal,
54}
55
56pub(super) fn commit_local_rebalance_worker_activation(
57 meta: &mut super::RebalanceMeta,
58 expected_id: &str,
59 cancel: CancellationToken,
60) -> Result<RebalanceLocalActivationOutcome> {
61 if meta.id != expected_id {
62 return Err(Error::other(format!(
63 "rebalance metadata changed before local worker activation: expected {expected_id}, found {}",
64 meta.id
65 )));
66 }
67 if meta.stopped_at.is_some() || meta.stop_requested || !is_rebalance_in_progress(meta) {
68 return Ok(RebalanceLocalActivationOutcome::NotStartedTerminal);
69 }
70 meta.cancel = Some(cancel);
71 Ok(RebalanceLocalActivationOutcome::Started)
72}
73
74pub(super) fn stage_local_rebalance_worker_activation(
75 meta: &super::RebalanceMeta,
76 expected_id: &str,
77 cancel: CancellationToken,
78 now: OffsetDateTime,
79) -> Result<(super::RebalanceMeta, RebalanceLocalActivationOutcome, bool)> {
80 let mut candidate = meta.clone();
81 let completed_at_goal = complete_rebalance_pools_at_goal(&mut candidate, now);
82 let completed_empty_queue = complete_rebalance_pools_with_empty_queue(&mut candidate, now);
83 let outcome = commit_local_rebalance_worker_activation(&mut candidate, expected_id, cancel)?;
84 let must_persist =
85 completed_at_goal || completed_empty_queue || outcome == RebalanceLocalActivationOutcome::NotStartedTerminal;
86 Ok((candidate, outcome, must_persist))
87}
88
89pub(super) fn commit_local_rebalance_worker_activation_candidate(
90 current: &mut super::RebalanceMeta,
91 expected_id: &str,
92 expected_cancel: Option<&CancellationToken>,
93 candidate: super::RebalanceMeta,
94) -> Result<()> {
95 if current.id != expected_id || candidate.id != expected_id {
96 return Err(Error::other(format!(
97 "rebalance metadata changed before local worker activation commit: expected {expected_id}, found {}",
98 current.id
99 )));
100 }
101 if !Arc::ptr_eq(¤t.activation_gate, &candidate.activation_gate) {
102 return Err(Error::other(format!(
103 "rebalance activation gate changed before local worker activation commit: {expected_id}"
104 )));
105 }
106 if current.cancel.as_ref() != expected_cancel {
107 return Err(Error::other(format!(
108 "rebalance worker token changed before local worker activation commit: {expected_id}"
109 )));
110 }
111 *current = candidate;
112 Ok(())
113}
114
115pub(super) fn rollback_local_rebalance_worker_activation(
116 meta: Option<&mut super::RebalanceMeta>,
117 expected_id: &str,
118 activation_token: &CancellationToken,
119) -> bool {
120 let Some(meta) = meta else {
121 return false;
122 };
123 if meta.id != expected_id || meta.cancel.as_ref() != Some(activation_token) {
124 return false;
125 }
126 if let Some(cancel) = meta.cancel.take() {
127 cancel.cancel();
128 return true;
129 }
130 false
131}
132
133impl ECStore {
134 #[tracing::instrument(skip_all)]
135 pub async fn start_rebalance(self: &Arc<Self>) -> Result<()> {
136 let _start_guard = self.start_gate.lock().await;
137 let _activation_guard = self.rebalance_activation_write_guard(None, "start rebalance").await?;
138 let movement_gate = self.ctx.data_movement_operation_gate();
139 let _movement_guard = movement_gate.write().await;
140 if self.start_rebalance_under_gate().await? {
141 self.ctx.advance_data_movement_operation_epoch();
142 }
143 Ok(())
144 }
145
146 pub(super) async fn start_rebalance_under_gate(self: &Arc<Self>) -> Result<bool> {
147 info!(
148 event = EVENT_REBALANCE_STATE,
149 component = LOG_COMPONENT_ECSTORE,
150 subsystem = LOG_SUBSYSTEM_REBALANCE,
151 state = "starting",
152 "Starting rebalance"
153 );
154 let expected_id: Arc<str> = {
155 let rebalance_meta = self.rebalance_meta.read().await;
156 Arc::from(rebalance_meta.as_ref().ok_or(Error::ConfigNotFound)?.id.as_str())
157 };
158 let pool = clone_first_arc(self.pools.as_slice(), "start_rebalance: no pools available")?;
159 let activation_fence = match self
160 .fence_rebalance_worker_activation(pool.clone(), expected_id.as_ref())
161 .await?
162 {
163 RebalanceWorkerActivationFence::Ready(fence) => fence,
164 RebalanceWorkerActivationFence::NotStartedTerminal => return Ok(false),
165 };
166
167 let decommission_running = self.is_decommission_running().await;
168
169 let cancel_tx = CancellationToken::new();
170 let rx = cancel_tx.clone();
171 let activation_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
172 let activation_outcome;
173 let candidate;
174 let expected_cancel;
175 let must_persist;
176
177 {
178 let mut rebalance_meta = self.rebalance_meta.write().await;
179 validate_start_rebalance_state(decommission_running, rebalance_meta.is_some())?;
180
181 let Some(meta) = rebalance_meta.as_mut() else {
182 return Err(Error::ConfigNotFound);
183 };
184 if should_skip_start_rebalance(meta.cancel.is_some(), is_rebalance_in_progress(meta)) {
185 debug!(
186 event = EVENT_REBALANCE_STATE,
187 component = LOG_COMPONENT_ECSTORE,
188 subsystem = LOG_SUBSYSTEM_REBALANCE,
189 state = "start_skipped",
190 reason = "already_in_progress",
191 "Skipped duplicate rebalance start"
192 );
193 return Ok(false);
194 }
195 expected_cancel = meta.cancel.clone();
196 (candidate, activation_outcome, must_persist) =
197 stage_local_rebalance_worker_activation(meta, expected_id.as_ref(), cancel_tx.clone(), activation_at)?;
198 if let Err(err) = activation_fence.ensure_held() {
199 cancel_tx.cancel();
200 return Err(err);
201 }
202 if !must_persist
203 && let Err(err) = commit_local_rebalance_worker_activation_candidate(
204 meta,
205 expected_id.as_ref(),
206 expected_cancel.as_ref(),
207 candidate.clone(),
208 )
209 {
210 cancel_tx.cancel();
211 return Err(err);
212 }
213 }
214
215 if must_persist {
216 let save_result = resolve_rebalance_meta_save_result(
217 self.save_rebalance_meta_under_activation_fence(
218 pool,
219 &candidate,
220 "start_rebalance persist activation candidate",
221 activation_fence.as_ref(),
222 expected_id.as_ref(),
223 )
224 .await,
225 "start_rebalance persist activation candidate",
226 );
227 if let Err(err) = save_result {
228 cancel_tx.cancel();
229 return Err(err);
230 }
231 let mut rebalance_meta = self.rebalance_meta.write().await;
232 let Some(meta) = rebalance_meta.as_mut() else {
233 cancel_tx.cancel();
234 return Err(Error::ConfigNotFound);
235 };
236 if let Err(err) = commit_local_rebalance_worker_activation_candidate(
237 meta,
238 expected_id.as_ref(),
239 expected_cancel.as_ref(),
240 candidate,
241 ) {
242 cancel_tx.cancel();
243 return Err(err);
244 }
245 }
246 if !must_persist && let Err(err) = activation_fence.ensure_held() {
247 let mut rebalance_meta = self.rebalance_meta.write().await;
248 rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
249 return Err(err);
250 }
251 drop(activation_fence);
252
253 if activation_outcome != RebalanceLocalActivationOutcome::Started {
254 return Ok(must_persist);
255 }
256
257 let participants = if let Some(ref meta) = *self.rebalance_meta.read().await {
258 resolve_rebalance_participants(meta.pool_stats.as_slice(), self.pools.len())
259 } else {
260 debug!(
261 event = EVENT_REBALANCE_STATE,
262 component = LOG_COMPONENT_ECSTORE,
263 subsystem = LOG_SUBSYSTEM_REBALANCE,
264 state = "start_skipped",
265 reason = "metadata_missing",
266 "Skipped rebalance start because metadata is unavailable"
267 );
268 Vec::new()
269 };
270
271 if !participants.iter().any(|participating| *participating) {
272 let mut rebalance_meta = self.rebalance_meta.write().await;
273 rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
274 debug!(
275 event = EVENT_REBALANCE_STATE,
276 component = LOG_COMPONENT_ECSTORE,
277 subsystem = LOG_SUBSYSTEM_REBALANCE,
278 state = "start_skipped",
279 reason = "no_participants",
280 "Skipped rebalance start because no pools are participating"
281 );
282 return Ok(must_persist);
283 }
284
285 #[cfg(test)]
286 let endpoints = self.instance_endpoints().unwrap_or_else(|| self.endpoints());
287 #[cfg(not(test))]
288 let endpoints = self.endpoints();
289
290 let mut workers_started = 0usize;
291 for (idx, participating) in participants.iter().enumerate() {
292 if !*participating {
293 debug!(
294 event = EVENT_REBALANCE_STATE,
295 component = LOG_COMPONENT_ECSTORE,
296 subsystem = LOG_SUBSYSTEM_REBALANCE,
297 pool_index = idx,
298 state = "pool_skipped",
299 reason = "not_participating",
300 "Skipped rebalance pool"
301 );
302 continue;
303 }
304
305 if !runtime_sources::endpoint_pool_is_local(&endpoints, idx) {
306 debug!(
307 event = EVENT_REBALANCE_STATE,
308 component = LOG_COMPONENT_ECSTORE,
309 subsystem = LOG_SUBSYSTEM_REBALANCE,
310 pool_index = idx,
311 state = "pool_skipped",
312 reason = "not_local",
313 "Skipped non-local rebalance pool"
314 );
315 continue;
316 }
317
318 let pool_idx = idx;
319 let store = self.clone();
320 let rx_clone = rx.clone();
321 let worker_id = Arc::clone(&expected_id);
322 workers_started += 1;
323 tokio::spawn(async move {
324 if let Err(err) = store.rebalance_buckets(rx_clone, pool_idx, worker_id).await {
325 error!(
326 event = EVENT_REBALANCE_STATE,
327 component = LOG_COMPONENT_ECSTORE,
328 subsystem = LOG_SUBSYSTEM_REBALANCE,
329 pool_index = pool_idx,
330 state = "pool_failed",
331 error = %err,
332 "Rebalance pool failed"
333 );
334 } else {
335 debug!(
336 event = EVENT_REBALANCE_STATE,
337 component = LOG_COMPONENT_ECSTORE,
338 subsystem = LOG_SUBSYSTEM_REBALANCE,
339 pool_index = pool_idx,
340 state = "completed",
341 "Rebalance pool completed"
342 );
343 }
344 });
345 }
346
347 if workers_started == 0 {
348 let mut rebalance_meta = self.rebalance_meta.write().await;
349 rollback_local_rebalance_worker_activation(rebalance_meta.as_mut(), expected_id.as_ref(), &rx);
350 debug!(
351 event = EVENT_REBALANCE_STATE,
352 component = LOG_COMPONENT_ECSTORE,
353 subsystem = LOG_SUBSYSTEM_REBALANCE,
354 state = "start_skipped",
355 reason = "no_local_participants",
356 "Skipped rebalance start because no local pools are participating"
357 );
358 return Ok(must_persist);
359 }
360
361 info!(
362 event = EVENT_REBALANCE_STATE,
363 component = LOG_COMPONENT_ECSTORE,
364 subsystem = LOG_SUBSYSTEM_REBALANCE,
365 state = "started",
366 worker_count = workers_started,
367 "Rebalance started"
368 );
369 Ok(true)
370 }
371
372 #[tracing::instrument(skip(self, rx))]
373 async fn rebalance_buckets(self: &Arc<Self>, rx: CancellationToken, pool_index: usize, rebalance_id: Arc<str>) -> Result<()> {
374 ensure_valid_rebalance_pool_index(self.pools.len(), pool_index)?;
375
376 let (done_tx, mut done_rx) = tokio::sync::mpsc::channel::<Result<()>>(1);
377
378 let store = self.clone();
380 let save_rebalance_id = Arc::clone(&rebalance_id);
381 let save_task = tokio::spawn(async move {
382 let mut timer = tokio::time::interval_at(Instant::now() + Duration::from_secs(30), Duration::from_secs(10));
383 let mut msg: String;
384 let mut quit = false;
385
386 loop {
387 let mut terminal_state_saved = false;
388 tokio::select! {
389 result = done_rx.recv() => {
390 quit = true;
391 let movement_gate = store.ctx.data_movement_operation_gate();
392 let movement_guard = movement_gate.write().await;
393 let terminal_at = store.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
394 let terminal_event = classify_rebalance_terminal_event(result, terminal_at);
395 msg = terminal_event.message().to_string();
396 let previous_meta = store.rebalance_meta.read().await.clone();
397 let terminal_state_present = {
398 let mut rebalance_meta = store.rebalance_meta.write().await;
399 super::control::ensure_rebalance_run_id(
400 rebalance_meta.as_ref(),
401 save_rebalance_id.as_ref(),
402 "apply rebalance terminal event",
403 )?;
404 if let Some(meta) = rebalance_meta.as_mut() {
405 let meta_stopped = meta.stopped_at.is_some();
406 if let Some(pool_stat) = meta.pool_stats.get_mut(pool_index) {
407 if matches!(&terminal_event, super::meta::RebalanceTerminalEvent::Completed { .. })
408 && has_rebalance_cleanup_warnings(pool_stat)
409 {
410 pool_stat.info.stopping = false;
411 pool_stat.info.status = RebalStatus::Failed;
412 pool_stat.info.end_time = Some(terminal_at);
413 pool_stat.info.last_error = Some(
414 pool_stat
415 .cleanup_warnings
416 .last_message
417 .clone()
418 .unwrap_or_else(|| "rebalance source cleanup warnings prevented completion".to_string()),
419 );
420 } else if should_preserve_rebalance_stopped_state(
421 meta_stopped,
422 pool_stat.info.status,
423 &terminal_event,
424 ) {
425 debug!(
426 event = EVENT_REBALANCE_STATE,
427 component = LOG_COMPONENT_ECSTORE,
428 subsystem = LOG_SUBSYSTEM_REBALANCE,
429 pool_index,
430 state = "stopped_preserved",
431 "Preserved stopped rebalance status"
432 );
433 } else {
434 pool_stat.info.stopping = false;
435 apply_rebalance_terminal_event(
436 &mut pool_stat.info.status,
437 &mut pool_stat.info.end_time,
438 &mut pool_stat.info.last_error,
439 terminal_event,
440 terminal_at,
441 );
442 }
443 true
444 } else {
445 false
446 }
447 } else {
448 false
449 }
450 };
451
452 if terminal_state_present {
453 if let Err(err) = retry_rebalance_metadata_access(None, rebalance_max_attempts(), || {
454 store.save_rebalance_stats_inner(
455 pool_index,
456 RebalSaveOpt::Stats,
457 Some(save_rebalance_id.as_ref()),
458 )
459 })
460 .await
461 {
462 let mut rebalance_meta = store.rebalance_meta.write().await;
463 *rebalance_meta = previous_meta;
464 drop(movement_guard);
465 return Err(Error::other(format!(
466 "rebalance terminal state save failed for pool {pool_index}: {err}"
467 )));
468 }
469 store.ctx.advance_data_movement_operation_epoch();
470 terminal_state_saved = true;
471 }
472 }
473 _ = timer.tick() => {
474 let now = OffsetDateTime::now_utc();
475 msg = format!("Saving rebalance metadata at {now:?}");
476 }
477 }
478
479 if !terminal_state_saved
480 && let Err(err) = retry_rebalance_metadata_access(None, rebalance_max_attempts(), || {
481 store.save_rebalance_stats_for_id(pool_index, RebalSaveOpt::Stats, save_rebalance_id.as_ref())
482 })
483 .await
484 {
485 let wrapped = Error::other(format!("rebalance save_task stats save failed for pool {pool_index}: {err}"));
486 error!("{} err: {:?}", msg, wrapped);
487 if quit {
488 return Err(wrapped);
489 }
490 } else {
491 debug!(
492 event = EVENT_REBALANCE_STATE,
493 component = LOG_COMPONENT_ECSTORE,
494 subsystem = LOG_SUBSYSTEM_REBALANCE,
495 pool_index,
496 state = "metadata_saved",
497 message = %msg,
498 "Saved rebalance metadata"
499 );
500 }
501
502 if quit {
503 debug!(
504 event = EVENT_REBALANCE_STATE,
505 component = LOG_COMPONENT_ECSTORE,
506 subsystem = LOG_SUBSYSTEM_REBALANCE,
507 pool_index,
508 state = "save_task_exiting",
509 message = %msg,
510 "Exiting rebalance save task"
511 );
512 return Ok(());
513 }
514
515 timer.reset();
516 }
517 });
518
519 info!(
520 event = EVENT_REBALANCE_STATE,
521 component = LOG_COMPONENT_ECSTORE,
522 subsystem = LOG_SUBSYSTEM_REBALANCE,
523 pool_index,
524 state = "pool_started",
525 "Rebalance worker started"
526 );
527 let mut final_result: Result<()> = Ok(());
528 let mut deferred_buckets = HashSet::new();
529 let mut source_cleanup_deferred_attempts = HashMap::new();
530
531 loop {
532 if rx.is_cancelled() {
533 info!(
534 event = EVENT_REBALANCE_STATE,
535 component = LOG_COMPONENT_ECSTORE,
536 subsystem = LOG_SUBSYSTEM_REBALANCE,
537 pool_index,
538 state = "pool_stopped",
539 reason = "cancelled",
540 "Stopped rebalance worker"
541 );
542 let err = Error::OperationCanceled;
543 final_result = Err(resolve_rebalance_terminal_error(
544 err.clone(),
545 send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
546 ));
547 break;
548 }
549
550 let next_bucket = match self.next_rebal_bucket(pool_index, rebalance_id.as_ref()).await {
551 Ok(bucket) => bucket,
552 Err(err) => {
553 error!(
554 event = EVENT_REBALANCE_BUCKET,
555 component = LOG_COMPONENT_ECSTORE,
556 subsystem = LOG_SUBSYSTEM_REBALANCE,
557 pool_index,
558 state = "next_bucket_failed",
559 error = ?err,
560 "Rebalance next bucket lookup failed"
561 );
562 final_result = Err(resolve_rebalance_terminal_error(
563 err.clone(),
564 send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
565 ));
566 break;
567 }
568 };
569
570 if let Some(bucket) = next_bucket {
571 debug!(
572 event = EVENT_REBALANCE_BUCKET,
573 component = LOG_COMPONENT_ECSTORE,
574 subsystem = LOG_SUBSYSTEM_REBALANCE,
575 pool_index,
576 bucket = %bucket,
577 state = "started",
578 "Starting rebalance bucket"
579 );
580
581 let outcome = match resolve_rebalance_bucket_result(
582 self.rebalance_bucket(rx.clone(), bucket.clone(), pool_index, Arc::clone(&rebalance_id))
583 .await,
584 pool_index,
585 &bucket,
586 ) {
587 Ok(outcome) => outcome,
588 Err(err) => {
589 error!(
590 event = EVENT_REBALANCE_BUCKET,
591 component = LOG_COMPONENT_ECSTORE,
592 subsystem = LOG_SUBSYSTEM_REBALANCE,
593 pool_index,
594 bucket = %bucket,
595 state = "bucket_failed",
596 error = ?err,
597 "Rebalance bucket failed"
598 );
599 final_result = Err(resolve_rebalance_terminal_error(
600 err.clone(),
601 send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
602 ));
603 break;
604 }
605 };
606
607 if let RebalanceBucketOutcome::Deferred { last_error } = outcome {
608 let source_cleanup_deferred = last_error.starts_with(REBALANCE_SOURCE_CLEANUP_DEFERRED_ERROR_PREFIX);
609 if should_fail_repeated_rebalance_bucket_defer(&mut deferred_buckets, &bucket, source_cleanup_deferred) {
610 let err = Error::other(format!(
611 "rebalance bucket {bucket} deferred repeatedly due to transient object failures: {last_error}"
612 ));
613 error!(
614 event = EVENT_REBALANCE_BUCKET,
615 component = LOG_COMPONENT_ECSTORE,
616 subsystem = LOG_SUBSYSTEM_REBALANCE,
617 pool_index,
618 bucket = %bucket,
619 state = "bucket_deferred_repeatedly",
620 error = ?err,
621 "Rebalance bucket failed after repeated deferral"
622 );
623 final_result = Err(resolve_rebalance_terminal_error(
624 err.clone(),
625 send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
626 ));
627 break;
628 }
629
630 let source_cleanup_attempt = if source_cleanup_deferred {
631 source_cleanup_defer_attempt(&mut source_cleanup_deferred_attempts, &bucket)
632 } else {
633 0
634 };
635 let defer_kind = if source_cleanup_deferred {
636 RebalanceDeferKind::SourceCleanup
637 } else {
638 RebalanceDeferKind::Entry
639 };
640 warn!(
641 event = EVENT_REBALANCE_BUCKET,
642 component = LOG_COMPONENT_ECSTORE,
643 subsystem = LOG_SUBSYSTEM_REBALANCE,
644 pool_index,
645 bucket = %bucket,
646 state = "deferred",
647 error = %last_error,
648 "Deferred rebalance bucket after transient object failures"
649 );
650 if let Err(err) = self
651 .defer_rebalance_bucket(pool_index, bucket.clone(), last_error.clone(), rebalance_id.as_ref(), defer_kind)
652 .await
653 {
654 error!(
655 event = EVENT_REBALANCE_BUCKET,
656 component = LOG_COMPONENT_ECSTORE,
657 subsystem = LOG_SUBSYSTEM_REBALANCE,
658 pool_index,
659 bucket = %bucket,
660 state = "defer_failed",
661 error = ?err,
662 "Rebalance bucket defer failed"
663 );
664 final_result = Err(resolve_rebalance_terminal_error(
665 err.clone(),
666 send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
667 ));
668 break;
669 }
670 if source_cleanup_deferred {
671 if reached_rebalance_source_cleanup_defer_limit(source_cleanup_attempt) {
672 let err = Error::other(format!(
673 "rebalance bucket {bucket} source cleanup remained unstable after {} deferrals: {last_error}",
674 REBALANCE_SOURCE_CLEANUP_MAX_DEFERS
675 ));
676 warn!(
677 event = EVENT_REBALANCE_BUCKET,
678 component = LOG_COMPONENT_ECSTORE,
679 subsystem = LOG_SUBSYSTEM_REBALANCE,
680 pool_index,
681 bucket = %bucket,
682 state = "source_cleanup_defer_limit",
683 error = ?err,
684 "Rebalance bucket failed after repeated source cleanup conflicts"
685 );
686 final_result = Err(resolve_rebalance_terminal_error(
687 err.clone(),
688 send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
689 ));
690 break;
691 }
692 if let Err(err) =
693 super::worker::wait_rebalance_listing_retry(&rx, REBALANCE_LISTING_RETRY_BASE_DELAY).await
694 {
695 final_result = Err(resolve_rebalance_terminal_error(
696 err.clone(),
697 send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
698 ));
699 break;
700 }
701 }
702 continue;
703 }
704
705 debug!(
706 event = EVENT_REBALANCE_BUCKET,
707 component = LOG_COMPONENT_ECSTORE,
708 subsystem = LOG_SUBSYSTEM_REBALANCE,
709 pool_index,
710 bucket = %bucket,
711 state = "completed",
712 "Completed rebalance bucket"
713 );
714 source_cleanup_deferred_attempts.remove(&bucket);
715 if let Err(err) = self.bucket_rebalance_done(pool_index, bucket, rebalance_id.as_ref()).await {
716 error!(
717 event = EVENT_REBALANCE_BUCKET,
718 component = LOG_COMPONENT_ECSTORE,
719 subsystem = LOG_SUBSYSTEM_REBALANCE,
720 pool_index,
721 state = "bucket_done_mark_failed",
722 error = ?err,
723 "Rebalance bucket completion mark failed"
724 );
725 final_result = Err(resolve_rebalance_terminal_error(
726 err.clone(),
727 send_rebalance_done_signal(&done_tx, Err(err.clone()), pool_index).await,
728 ));
729 break;
730 }
731 } else {
732 debug!(
733 event = EVENT_REBALANCE_BUCKET,
734 component = LOG_COMPONENT_ECSTORE,
735 subsystem = LOG_SUBSYSTEM_REBALANCE,
736 pool_index,
737 state = "idle",
738 reason = "no_bucket_to_rebalance",
739 "No rebalance bucket available"
740 );
741 break;
742 }
743 }
744
745 info!(
746 event = EVENT_REBALANCE_STATE,
747 component = LOG_COMPONENT_ECSTORE,
748 subsystem = LOG_SUBSYSTEM_REBALANCE,
749 pool_index,
750 state = "pool_done",
751 "Rebalance worker finished"
752 );
753
754 if final_result.is_ok()
755 && let Err(err) = send_rebalance_done_signal(&done_tx, Ok(()), pool_index).await
756 {
757 final_result = Err(err);
758 }
759 drop(done_tx);
760 if let Err(err) = resolve_rebalance_save_task_result(pool_index, save_task.await)
761 && final_result.is_ok()
762 {
763 final_result = Err(err);
764 }
765 debug!(
766 event = EVENT_REBALANCE_STATE,
767 component = LOG_COMPONENT_ECSTORE,
768 subsystem = LOG_SUBSYSTEM_REBALANCE,
769 pool_index,
770 state = "pool_result_returned",
771 "Rebalance worker result returned"
772 );
773 final_result
774 }
775
776 pub(super) async fn check_if_rebalance_done(&self, pool_index: usize, expected_id: &str) -> Result<bool> {
777 let mut rebalance_meta = self.rebalance_meta.write().await;
778 super::control::ensure_rebalance_worker_active(rebalance_meta.as_ref(), expected_id, "check rebalance completion")?;
779
780 if let Some(meta) = rebalance_meta.as_mut()
781 && let Some(pool_stat) = meta.pool_stats.get_mut(pool_index)
782 {
783 if pool_stat.info.status == RebalStatus::Completed {
785 debug!(
786 event = EVENT_REBALANCE_STATE,
787 component = LOG_COMPONENT_ECSTORE,
788 subsystem = LOG_SUBSYSTEM_REBALANCE,
789 pool_index,
790 state = "already_completed",
791 "Rebalance pool is already completed"
792 );
793 return Ok(true);
794 }
795
796 let pfi = if pool_stat.init_capacity == 0 {
798 0.0
799 } else {
800 (pool_stat.init_free_space + pool_stat.bytes) as f64 / pool_stat.init_capacity as f64
801 };
802
803 if !has_deferred_rebalance_error(pool_stat)
804 && !has_rebalance_cleanup_warnings(pool_stat)
805 && rebalance_goal_reached(
806 pool_stat.init_free_space,
807 pool_stat.init_capacity,
808 pool_stat.bytes,
809 meta.percent_free_goal,
810 )
811 {
812 info!(
813 event = EVENT_REBALANCE_STATE,
814 component = LOG_COMPONENT_ECSTORE,
815 subsystem = LOG_SUBSYSTEM_REBALANCE,
816 pool_index,
817 state = "completion_ready",
818 percent_free = pfi,
819 "Rebalance pool reached completion goal"
820 );
821 return Ok(true);
822 }
823 }
824
825 Ok(false)
826 }
827}
828
829impl ECStore {
830 #[tracing::instrument(skip(self))]
831 pub async fn save_rebalance_stats(&self, pool_idx: usize, opt: RebalSaveOpt) -> Result<()> {
832 let movement_gate = self.ctx.data_movement_operation_gate();
833 let _movement_guard = movement_gate.write().await;
834 self.save_rebalance_stats_inner(pool_idx, opt, None).await
835 }
836
837 pub async fn save_rebalance_stats_for_id(&self, pool_idx: usize, opt: RebalSaveOpt, expected_id: &str) -> Result<()> {
838 let movement_gate = self.ctx.data_movement_operation_gate();
839 let _movement_guard = movement_gate.write().await;
840 self.save_rebalance_stats_inner(pool_idx, opt, Some(expected_id)).await
841 }
842
843 pub(super) async fn save_rebalance_stats_inner(
844 &self,
845 pool_idx: usize,
846 opt: RebalSaveOpt,
847 expected_id: Option<&str>,
848 ) -> Result<()> {
849 let now = match opt {
850 RebalSaveOpt::Stats => OffsetDateTime::now_utc(),
851 RebalSaveOpt::StoppedAt => self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await,
852 };
853 let meta_to_save = {
854 let mut rebalance_meta = self.rebalance_meta.write().await;
855 if let Some(expected_id) = expected_id {
856 super::control::ensure_rebalance_run_id(rebalance_meta.as_ref(), expected_id, "save rebalance stats")?;
857 }
858 let Some(meta) = rebalance_meta.as_mut() else {
859 return Ok(());
860 };
861
862 apply_rebalance_save_option(meta, pool_idx, opt, now);
863 meta.clone()
864 };
865
866 let pool = clone_first_arc(&self.pools, "save_rebalance_stats: no pools available")?;
867
868 debug!(
869 event = EVENT_REBALANCE_STATE,
870 component = LOG_COMPONENT_ECSTORE,
871 subsystem = LOG_SUBSYSTEM_REBALANCE,
872 pool_index = pool_idx,
873 save_opt = ?opt,
874 state = "metadata_save_requested",
875 "Rebalance metadata save requested"
876 );
877 let stage = format!("save_rebalance_stats for pool {pool_idx} opt {opt:?}");
878 let save_result = match expected_id {
879 Some(expected_id) => {
880 self.save_rebalance_meta_for_id_with_merge(pool, &meta_to_save, stage.as_str(), expected_id)
881 .await
882 }
883 None => self.save_rebalance_meta_with_merge(pool, &meta_to_save, stage.as_str()).await,
884 };
885 resolve_rebalance_meta_save_result(save_result, stage.as_str())?;
886
887 Ok(())
888 }
889}