1use crate::bucket::replication::replication_state_from_filemeta;
16#[cfg(all(test, feature = "test-util"))]
17use crate::bucket::utils::is_meta_bucketname;
18use crate::bucket::versioning::VersioningApi as _;
19use crate::bucket::versioning_sys::BucketVersioningSys;
20use crate::bucket::{
21 lifecycle::{
22 DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
23 ValidatedDurableIlmRecord,
24 bucket_lifecycle_audit::LcEventSrc,
25 bucket_lifecycle_ops::{
26 LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_for_data_movement, apply_expiry_rule_in,
27 eval_action_from_lifecycle, lifecycle_delete_all_versions_blocked_by_replication,
28 },
29 classify_durable_ilm_record, get_expiry_configs,
30 lifecycle::IlmAction,
31 tier_delete_journal::durable_ilm_v6_topology_generation,
32 validate_durable_ilm_record,
33 },
34 metadata_sys,
35};
36use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
37use crate::config::com::{
38 CONFIG_PREFIX, delete_config, read_config_limited_preserve_empty, read_config_limited_preserve_empty_with_metadata,
39 read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty, save_config_with_opts,
40 save_config_with_opts_and_metadata,
41};
42use crate::data_movement;
43use crate::data_movement::backpressure::{self, DataMovementOperation};
44use crate::data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_cache};
45use crate::disk::error::DiskError;
46use crate::disk::{BUCKET_META_PREFIX, DiskAPI, RUSTFS_META_BUCKET};
47use crate::error::{Error, Result};
48use crate::error::{
49 StorageError, is_err_bucket_exists, is_err_bucket_not_found, is_err_data_movement_overwrite, is_err_invalid_upload_id,
50 is_err_object_not_found, is_err_operation_canceled, is_err_version_not_found,
51};
52use crate::layout::endpoints::EndpointServerPools;
53use crate::object_api::{DecommissionCapacityOptions, GetObjectReader, ObjectInfo, ObjectOptions};
54use crate::runtime::sources as runtime_sources;
55use crate::services::notification_sys::{
56 acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches, tier_delete_journal_topology_generation,
57};
58use crate::services::rebalance::{REBAL_META_NAME, RebalanceMeta, is_rebalance_conflicting_with_decommission};
59use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
60use crate::storage_api_contracts::{
61 admin::StorageAdminApi,
62 bucket::{BucketOperations, BucketOptions, MakeBucketOptions},
63 heal::HealOperations as _,
64 list::ListOperations as _,
65 namespace::NamespaceLocking as _,
66 object::{EcstoreObjectIO, HTTPPreconditions, ObjectIO as _, ObjectOperations as _},
67};
68use crate::store::PoolMetaWriteGateStatus;
69use crate::{core::sets::Sets, store::ECStore};
70use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
71use futures::{
72 StreamExt,
73 future::{BoxFuture, join_all},
74 stream::FuturesUnordered,
75};
76use http::HeaderMap;
77use rand::RngExt as _;
78#[cfg(test)]
79use rmp_serde::Deserializer;
80use rmp_serde::Serializer;
81use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
82use rustfs_heal_contracts::heal_channel::HealOpts;
83use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
84use rustfs_utils::path::{
85 decode_dir_object, encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path,
86};
87use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ReplicationConfiguration};
88use serde::{Deserialize, Serialize};
89use sha2::{Digest, Sha256};
90use std::collections::{HashMap, HashSet};
91use std::fmt::Display;
92#[cfg(test)]
93use std::io::Cursor;
94use std::io::Write;
95use std::path::PathBuf;
96use std::sync::{
97 Arc,
98 atomic::{AtomicBool, AtomicUsize, Ordering},
99};
100use time::{Duration, OffsetDateTime};
101use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc};
102use tokio_util::sync::CancellationToken;
103use tracing::{debug, error, info, warn};
104
105const LOG_COMPONENT_ECSTORE: &str = "ecstore";
106const LOG_SUBSYSTEM_POOLS: &str = "pools";
107const EVENT_DECOMMISSION_STATE: &str = "decommission_state";
108const EVENT_DECOMMISSION_BUCKET: &str = "decommission_bucket";
109const EVENT_DECOMMISSION_ENTRY: &str = "decommission_entry";
110const POOL_ACTIVATION_FLEET_PROOF_REQUIRED: &str = "pool activation requires a live fleet capability proof";
111const POOL_ACTIVATION_FLEET_PROOF_EXPIRED: &str = "pool activation fleet capability proof expired before commit";
112const DECOMMISSION_TARGET_FLEET_PROOF_EXPIRED: &str =
113 "decommission target fence fleet capability proof expired before reservation commit";
114const DECOMMISSION_STAGE_MIGRATE_OBJECT: &str = "migrate_object";
115const DECOMMISSION_STAGE_CLEANUP_PREFLIGHT: &str = "cleanup_preflight";
116const DECOMMISSION_STAGE_SOURCE_CLEANUP: &str = "source_cleanup";
117const DECOMMISSION_STAGE_ENTRY_FINISHED: &str = "entry_finished";
118const DECOMMISSION_PROGRESS_SAVE_INTERVAL: Duration = Duration::seconds(30);
119const DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD: usize = 1000;
120const DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF: Duration = Duration::seconds(1);
121const DECOMMISSION_BUCKET_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_BUCKET_CONCURRENCY";
122const DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP: usize = 4;
123const DECOMMISSION_ENTRY_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_ENTRY_CONCURRENCY";
124const DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP: usize = 8;
125const DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP: usize = 64;
126const DECOMMISSION_ENTRY_WORKERS_PER_SET: usize = 2;
127const DECOMMISSION_META_PREFIXES: [&str; 3] = [CONFIG_PREFIX, BUCKET_META_PREFIX, ILM_META_PREFIX];
128const DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION: u16 = 1;
129const DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION: u16 = 2;
130#[cfg(test)]
131const DECOMMISSION_CAPACITY_MODEL_VERSION: u16 = DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION;
132const DECOMMISSION_CAPACITY_TEMPORARY_COPIES: usize = 1;
133const DECOMMISSION_CAPACITY_RESERVATION_TTL: Duration = Duration::minutes(10);
134const DECOMMISSION_CAPACITY_RELEASE_CANCELED: &str = "canceled";
135const DECOMMISSION_CAPACITY_RELEASE_FAILED: &str = "failed";
136const DECOMMISSION_CAPACITY_RELEASE_COMPLETED: &str = "completed";
137pub(crate) const DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX: &str = "decommission/capacity-target";
138const DECOMMISSION_CAPACITY_TARGET_LOCK_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(250);
139const DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS: usize = 12;
140pub(crate) const DECOMMISSION_MUTATION_GATE_MAX_INLINE_ATTEMPTS: usize = 3;
141static DECOMMISSION_OBJECT_ATTEMPTS_MAX: std::sync::Mutex<u64> = std::sync::Mutex::new(0);
142const DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX: &str = "target pool ";
143const DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX: &str = " target capacity mutation gate is busy";
144const METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL: &str = "rustfs_decommission_capacity_conflicts_total";
145const METRIC_DECOMMISSION_CAPACITY_GATE_RETRIES_TOTAL: &str = "rustfs_decommission_capacity_gate_retries_total";
146const METRIC_DECOMMISSION_OBJECT_ATTEMPTS_TOTAL: &str = "rustfs_decommission_object_attempts_total";
149const METRIC_DECOMMISSION_OBJECT_ATTEMPTS_MAX: &str = "rustfs_decommission_object_attempts_max";
152const METRIC_DECOMMISSION_CAPACITY_GATE_WAIT_SECONDS: &str = "rustfs_decommission_capacity_gate_wait_seconds";
154const METRIC_DECOMMISSION_CAPACITY_PREDICTED_BYTES: &str = "rustfs_decommission_capacity_predicted_physical_bytes";
155const METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES: &str = "rustfs_decommission_capacity_reserved_physical_bytes";
156const METRIC_DECOMMISSION_CAPACITY_PREDICTION_ERROR_BYTES: &str = "rustfs_decommission_capacity_prediction_error_bytes";
157const METRIC_DECOMMISSION_CAPACITY_PREDICTION_ABSOLUTE_ERROR_BYTES: &str =
158 "rustfs_decommission_capacity_prediction_absolute_error_bytes";
159const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3;
160const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
161pub(crate) const DECOMMISSION_ENTRY_MAX_ATTEMPTS: usize = 3;
162const DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS: usize = 12;
163const DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS: usize = 12;
168const DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(100);
169pub(crate) const DECOMMISSION_VERSION_COPY_ATTEMPTS: usize = 3;
170const DECOMMISSION_COPY_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(50);
171const DECOMMISSION_CAPACITY_GATE_RETRY_BASE: std::time::Duration = std::time::Duration::from_millis(100);
172const DECOMMISSION_RETRY_BACKOFF_CAP: std::time::Duration = std::time::Duration::from_millis(1500);
173const DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT: usize = 100;
174const DECOMMISSION_TERMINAL_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(1);
175const DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS: usize = 3;
176
177fn decommission_capacity_target_gate_retry_exhausted(attempt: usize) -> bool {
178 attempt.saturating_add(1) >= DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS
179}
180
181fn record_decommission_capacity_gate_retry(target_pool_index: usize) {
182 metrics::counter!(
183 METRIC_DECOMMISSION_CAPACITY_GATE_RETRIES_TOTAL,
184 "target_pool_index" => target_pool_index.to_string()
185 )
186 .increment(1);
187}
188
189fn record_decommission_object_attempt(attempts: &mut usize) {
192 *attempts = attempts.saturating_add(1);
193 metrics::counter!(METRIC_DECOMMISSION_OBJECT_ATTEMPTS_TOTAL).increment(1);
194 let attempt = u64::try_from(*attempts).unwrap_or(u64::MAX);
195 let mut peak = DECOMMISSION_OBJECT_ATTEMPTS_MAX.lock().unwrap_or_else(|err| err.into_inner());
198 if *peak < attempt {
199 *peak = attempt;
200 metrics::gauge!(METRIC_DECOMMISSION_OBJECT_ATTEMPTS_MAX).set(attempt as f64);
201 }
202}
203
204fn record_decommission_capacity_gate_wait(delay: std::time::Duration) {
205 metrics::histogram!(METRIC_DECOMMISSION_CAPACITY_GATE_WAIT_SECONDS).record(delay.as_secs_f64());
206}
207const DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT: &str = "decommission/ilm-receipts";
208const DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT: &str = "decommission/ilm-manifests";
209const DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA: &str = "v2";
210const DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA: &str = "v1";
211const DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE: usize = 16 * 1024;
212const DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE: usize = 4 * 1024;
213const DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS: usize = 3;
214const DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
217
218pub const POOL_META_NAME: &str = "pool.bin";
219pub(crate) const POOL_META_IDENTITY_NAME: &str = "pool.bin.identity";
220pub const POOL_META_FORMAT: u16 = 1;
221const POOL_META_V1_VERSION: u16 = 1;
222pub const POOL_META_VERSION: u16 = 2;
223const POOL_META_GENERATION_VERSION: u16 = 3;
224const POOL_META_IDENTITY_FORMAT: u16 = 1;
225const POOL_META_IDENTITY_VERSION: u16 = 1;
226const POOL_META_INITIAL_EPOCH: u64 = 1;
227const POOL_META_CAS_MAX_ATTEMPTS: usize = 3;
228const METRIC_POOL_META_STALE_WRITE_REJECTIONS_TOTAL: &str = "rustfs_pool_meta_stale_write_rejections_total";
229
230fn record_pool_meta_stale_write_rejection(reason: &'static str) {
231 metrics::counter!(METRIC_POOL_META_STALE_WRITE_REJECTIONS_TOTAL, "reason" => reason).increment(1);
232}
233
234fn pool_meta_v2_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool {
235 requested && fleet_confirmed
236}
237
238fn pool_meta_v2_writer_enabled() -> bool {
239 pool_meta_v2_writer_enabled_for(
240 rustfs_utils::get_env_bool(rustfs_config::ENV_POOL_META_V2_WRITE, rustfs_config::DEFAULT_POOL_META_V2_WRITE),
241 rustfs_utils::get_env_bool(
242 rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
243 rustfs_config::DEFAULT_POOL_META_V2_FLEET_CONFIRMED,
244 ),
245 )
246}
247
248fn pool_meta_v3_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool {
249 requested && fleet_confirmed
250}
251
252fn pool_meta_v3_writer_enabled() -> bool {
253 pool_meta_v3_writer_enabled_for(
254 rustfs_utils::get_env_bool(rustfs_config::ENV_POOL_META_V3_WRITE, rustfs_config::DEFAULT_POOL_META_V3_WRITE),
255 rustfs_utils::get_env_bool(
256 rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED,
257 rustfs_config::DEFAULT_POOL_META_V3_FLEET_CONFIRMED,
258 ),
259 )
260}
261
262fn decommission_capacity_writer_supported_for(version: u16, v2_writer_enabled: bool, v3_writer_enabled: bool) -> bool {
263 matches!(version, POOL_META_VERSION | POOL_META_GENERATION_VERSION) || v2_writer_enabled || v3_writer_enabled
264}
265
266fn ensure_decommission_ledger_persistence_supported_for(
267 version: u16,
268 v2_writer_enabled: bool,
269 v3_writer_enabled: bool,
270) -> Result<()> {
271 if decommission_capacity_writer_supported_for(version, v2_writer_enabled, v3_writer_enabled) {
272 return Ok(());
273 }
274
275 Err(Error::InvalidArgument(
276 "decommission".to_string(),
277 "pool-metadata-version".to_string(),
278 format!(
279 "durable unresolved-entry recovery requires pool metadata V2 or V3; enable either the {} + {} V2 gate or the {} + {} V3 gate only after every reader and writer supports that format",
280 rustfs_config::ENV_POOL_META_V2_WRITE,
281 rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
282 rustfs_config::ENV_POOL_META_V3_WRITE,
283 rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED,
284 ),
285 ))
286}
287
288fn ensure_decommission_ledger_persistence_supported(pool_meta: &PoolMeta) -> Result<()> {
289 ensure_decommission_ledger_persistence_supported_for(
290 pool_meta.version,
291 pool_meta_v2_writer_enabled(),
292 pool_meta_v3_writer_enabled(),
293 )
294}
295
296#[derive(Clone, Debug)]
297pub struct DecommissionCanceler {
298 operation: Arc<DecommissionOperation>,
299}
300
301#[derive(Debug)]
302struct DecommissionOperation {
303 token: CancellationToken,
304 active: AtomicBool,
305}
306
307impl DecommissionCanceler {
308 pub(crate) fn new(token: CancellationToken) -> Self {
309 Self {
310 operation: Arc::new(DecommissionOperation {
311 token,
312 active: AtomicBool::new(true),
313 }),
314 }
315 }
316
317 #[cfg(test)]
318 pub(crate) fn new_for_test(token: CancellationToken) -> Self {
319 Self::new(token)
320 }
321
322 fn token(&self) -> &CancellationToken {
323 &self.operation.token
324 }
325
326 pub(crate) fn is_active(&self) -> bool {
327 self.operation.active.load(Ordering::Acquire)
328 }
329
330 #[cfg(test)]
331 fn is_cancelled(&self) -> bool {
332 self.token().is_cancelled()
333 }
334
335 fn cancel(&self) {
336 self.token().cancel();
337 }
338
339 fn release(&self) {
340 self.cancel();
341 self.operation.active.store(false, Ordering::Release);
342 }
343
344 fn owns_same_operation(&self, other: &Self) -> bool {
345 Arc::ptr_eq(&self.operation, &other.operation)
346 }
347}
348
349struct DecommissionCancelerGuard {
350 canceler: DecommissionCanceler,
351}
352
353impl DecommissionCancelerGuard {
354 fn new(canceler: DecommissionCanceler) -> Self {
355 Self { canceler }
356 }
357
358 fn canceler(&self) -> &DecommissionCanceler {
359 &self.canceler
360 }
361}
362
363impl Drop for DecommissionCancelerGuard {
364 fn drop(&mut self) {
365 self.canceler.release();
366 }
367}
368
369fn dedup_indices(indices: &[usize]) -> Vec<usize> {
370 let mut seen = HashSet::with_capacity(indices.len());
371 let mut output = Vec::with_capacity(indices.len());
372 for idx in indices {
373 if seen.insert(*idx) {
374 output.push(*idx);
375 }
376 }
377
378 output
379}
380
381fn bind_decommission_cancelers(
382 indices: &[usize],
383 parent: &CancellationToken,
384 cancelers: &mut [Option<DecommissionCanceler>],
385) -> Vec<(usize, DecommissionCanceler)> {
386 let mut bound = Vec::with_capacity(indices.len());
387
388 for idx in indices {
389 if let Some(slot) = cancelers.get_mut(*idx) {
390 if let Some(existing) = slot.take() {
391 existing.release();
392 }
393 let canceler = DecommissionCanceler::new(parent.child_token());
394 *slot = Some(canceler.clone());
395 bound.push((*idx, canceler));
396 }
397 }
398
399 bound
400}
401
402fn bind_missing_decommission_cancelers(
403 indices: &[usize],
404 parent: &CancellationToken,
405 cancelers: &mut [Option<DecommissionCanceler>],
406) -> Vec<(usize, DecommissionCanceler)> {
407 let mut bound = Vec::with_capacity(indices.len());
408
409 for idx in indices {
410 let Some(slot) = cancelers.get_mut(*idx) else {
411 continue;
412 };
413 if slot.as_ref().is_some_and(DecommissionCanceler::is_active) {
414 break;
415 }
416 if let Some(stale) = slot.take() {
417 stale.release();
418 }
419 let canceler = DecommissionCanceler::new(parent.child_token());
420 *slot = Some(canceler.clone());
421 bound.push((*idx, canceler));
422 }
423
424 bound
425}
426
427fn take_decommission_canceler(cancelers: &mut [Option<DecommissionCanceler>], idx: usize) -> Option<DecommissionCanceler> {
428 cancelers.get_mut(idx).and_then(Option::take)
429}
430
431fn take_decommission_canceler_for_operation(
432 cancelers: &mut [Option<DecommissionCanceler>],
433 idx: usize,
434 owner: &DecommissionCanceler,
435) -> Option<DecommissionCanceler> {
436 let slot = cancelers.get_mut(idx)?;
437 if slot.as_ref().is_some_and(|canceler| canceler.owns_same_operation(owner)) {
438 slot.take()
439 } else {
440 None
441 }
442}
443
444fn decommission_canceler_is_owned_by(
445 cancelers: &[Option<DecommissionCanceler>],
446 idx: usize,
447 owner: &DecommissionCanceler,
448) -> bool {
449 cancelers
450 .get(idx)
451 .and_then(Option::as_ref)
452 .is_some_and(|canceler| canceler.owns_same_operation(owner))
453}
454
455fn update_decommission_for_operation<T>(
456 cancelers: &[Option<DecommissionCanceler>],
457 pool_meta: &mut PoolMeta,
458 idx: usize,
459 owner: Option<&DecommissionCanceler>,
460 update: impl FnOnce(&mut PoolMeta) -> T,
461) -> Option<T> {
462 if let Some(owner) = owner
463 && !decommission_canceler_is_owned_by(cancelers, idx, owner)
464 {
465 owner.release();
466 return None;
467 }
468
469 Some(update(pool_meta))
470}
471
472fn has_active_decommission_canceler(cancelers: &[Option<DecommissionCanceler>]) -> bool {
473 cancelers.iter().flatten().any(DecommissionCanceler::is_active)
474}
475
476fn cancel_decommission_canceler(canceler: Option<DecommissionCanceler>) -> bool {
477 if let Some(canceler) = canceler {
478 canceler.release();
479 true
480 } else {
481 false
482 }
483}
484
485fn take_and_cancel_decommission_canceler(cancelers: &mut [Option<DecommissionCanceler>], idx: usize) -> bool {
486 let canceler = take_decommission_canceler(cancelers, idx);
487 cancel_decommission_canceler(canceler)
488}
489
490fn take_and_cancel_decommission_canceler_for_operation(
491 cancelers: &mut [Option<DecommissionCanceler>],
492 idx: usize,
493 owner: &DecommissionCanceler,
494) -> bool {
495 let canceler = take_decommission_canceler_for_operation(cancelers, idx, owner);
496 if canceler.is_none() {
497 owner.release();
498 return false;
499 }
500 cancel_decommission_canceler(canceler)
501}
502
503fn ensure_decommission_routines_scheduled(bound_count: usize, expected_count: usize) -> Result<()> {
504 if bound_count == 0 || bound_count != expected_count {
505 return Err(Error::other(format!(
506 "failed to start decommission routines: scheduled {bound_count} of {expected_count} expected workers"
507 )));
508 }
509
510 Ok(())
511}
512
513fn guard_decommission_cancelers(index_cancelers: Vec<(usize, DecommissionCanceler)>) -> Vec<(usize, DecommissionCancelerGuard)> {
514 index_cancelers
515 .into_iter()
516 .map(|(idx, canceler)| (idx, DecommissionCancelerGuard::new(canceler)))
517 .collect()
518}
519
520async fn await_decommission_worker(idx: usize, worker: tokio::task::JoinHandle<Result<()>>) -> Result<()> {
521 worker
522 .await
523 .map_err(|err| Error::other(format!("decommission worker {idx} task join error: {err}")))?
524}
525
526fn reserve_decommission_start_cancelers(
527 pool_meta: &PoolMeta,
528 indices: &[usize],
529 local_indices: &[usize],
530 parent: &CancellationToken,
531 cancelers: &mut [Option<DecommissionCanceler>],
532) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
533 ensure_decommission_start_pool_states(pool_meta, indices)?;
534 if local_indices.is_empty() {
535 return Ok(Vec::new());
536 }
537 let bound = bind_decommission_cancelers(local_indices, parent, cancelers);
538 let guards = guard_decommission_cancelers(bound);
539 ensure_decommission_routines_scheduled(guards.len(), local_indices.len())?;
540 Ok(guards)
541}
542
543fn default_decommission_bucket_concurrency(cpu_count: usize) -> usize {
544 cpu_count.clamp(1, DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP)
545}
546
547fn decommission_bucket_concurrency_limit() -> usize {
548 let default_limit = default_decommission_bucket_concurrency(num_cpus::get());
549 rustfs_utils::get_env_usize(DECOMMISSION_BUCKET_CONCURRENCY_ENV, default_limit).max(1)
550}
551
552fn default_decommission_entry_concurrency(cpu_count: usize) -> usize {
553 cpu_count.clamp(1, DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP)
554}
555
556fn clamp_decommission_entry_concurrency(limit: usize) -> usize {
557 limit.clamp(1, DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP)
558}
559
560fn decommission_entry_concurrency_limit() -> usize {
561 let default_limit = default_decommission_entry_concurrency(num_cpus::get());
562 clamp_decommission_entry_concurrency(rustfs_utils::get_env_usize(DECOMMISSION_ENTRY_CONCURRENCY_ENV, default_limit))
563}
564
565fn is_decommission_meta_bucket(bucket: &DecomBucketInfo) -> bool {
566 bucket.name == RUSTFS_META_BUCKET
567}
568
569fn decommission_meta_buckets() -> [DecomBucketInfo; DECOMMISSION_META_PREFIXES.len()] {
570 DECOMMISSION_META_PREFIXES.map(|prefix| DecomBucketInfo {
571 name: RUSTFS_META_BUCKET.to_owned(),
572 prefix: prefix.to_owned(),
573 })
574}
575
576fn reconcile_decommission_meta_buckets(meta: &mut PoolMeta, idx: usize) -> bool {
577 let before = meta.pending_buckets(idx).len();
578 meta.queue_buckets(idx, decommission_meta_buckets().into());
579 meta.pending_buckets(idx).len() != before
580}
581
582fn split_decommission_buckets(buckets: Vec<DecomBucketInfo>) -> (Vec<DecomBucketInfo>, Vec<DecomBucketInfo>) {
583 let mut regular = Vec::with_capacity(buckets.len());
584 let mut meta = Vec::new();
585
586 for bucket in buckets {
587 if is_decommission_meta_bucket(&bucket) {
588 meta.push(bucket);
589 } else {
590 regular.push(bucket);
591 }
592 }
593
594 regular.shrink_to_fit();
595 (regular, meta)
596}
597
598fn ensure_decommission_not_rebalancing(rebalance_running: bool) -> Result<()> {
599 if rebalance_running {
600 return Err(Error::RebalanceAlreadyRunning);
601 }
602
603 Ok(())
604}
605
606fn ensure_decommission_start_rebalance_meta_allowed(meta: Option<&RebalanceMeta>) -> Result<()> {
607 ensure_decommission_not_rebalancing(meta.is_some_and(is_rebalance_conflicting_with_decommission))
608}
609
610#[allow(dead_code, reason = "leader precondition asserted by this file's tests (backlog#1823)")]
611fn ensure_local_decommission_pool_leaders(endpoints: &EndpointServerPools, indices: &[usize]) -> Result<()> {
612 for idx in indices {
613 ensure_local_decommission_pool_leader(endpoints, *idx)?;
614 }
615
616 Ok(())
617}
618
619fn ensure_local_decommission_pool_leader(endpoints: &EndpointServerPools, idx: usize) -> Result<()> {
620 let pool = endpoints
621 .as_ref()
622 .get(idx)
623 .ok_or_else(|| invalid_decommission_pool_index_error(endpoints.as_ref().len(), idx))?;
624 let endpoint = pool
625 .endpoints
626 .as_ref()
627 .first()
628 .ok_or_else(|| Error::other(format!("decommission pool {idx} has no configured endpoints")))?;
629
630 if !endpoint.is_local {
631 return Err(Error::other(format!(
632 "decommission for pool {idx} must run on the pool first endpoint {endpoint}"
633 )));
634 }
635
636 Ok(())
637}
638
639fn decommission_pool_first_endpoint_is_local(endpoints: &EndpointServerPools, idx: usize) -> Result<bool> {
640 let pool = endpoints
641 .as_ref()
642 .get(idx)
643 .ok_or_else(|| invalid_decommission_pool_index_error(endpoints.as_ref().len(), idx))?;
644 let endpoint = pool
645 .endpoints
646 .as_ref()
647 .first()
648 .ok_or_else(|| Error::other(format!("decommission pool {idx} has no configured endpoints")))?;
649
650 Ok(endpoint.is_local)
651}
652
653pub(crate) fn local_decommission_queue_prefix(endpoints: &EndpointServerPools, indices: &[usize]) -> Result<Vec<usize>> {
654 let mut local = Vec::with_capacity(indices.len());
655
656 for idx in indices {
657 if decommission_pool_first_endpoint_is_local(endpoints, *idx)? {
658 local.push(*idx);
659 } else {
660 break;
661 }
662 }
663
664 Ok(local)
665}
666
667fn resumable_decommission_queue_indices(meta: &PoolMeta) -> Vec<usize> {
668 let mut indices = Vec::new();
669 for (idx, pool) in meta.pools.iter().enumerate() {
670 if let Some(decommission) = &pool.decommission {
671 if !decommission.has_decommission_state() {
672 continue;
673 }
674 if decommission.complete || decommission.failed || decommission.canceled {
675 continue;
676 }
677 indices.push(idx);
678 }
679 }
680
681 indices
682}
683
684fn missing_decommission_worker_prefix(indices: &[usize], cancelers: &[Option<DecommissionCanceler>]) -> Vec<usize> {
685 let mut missing = Vec::with_capacity(indices.len());
686
687 for idx in indices {
688 if cancelers
689 .get(*idx)
690 .and_then(Option::as_ref)
691 .is_some_and(DecommissionCanceler::is_active)
692 {
693 break;
694 }
695 missing.push(*idx);
696 }
697
698 missing
699}
700
701fn ensure_decommission_start_local_leader(endpoints: &EndpointServerPools, indices: &[usize]) -> Result<()> {
702 if let Some(first) = indices.first() {
703 ensure_local_decommission_pool_leader(endpoints, *first)?;
704 }
705
706 Ok(())
707}
708
709fn build_decommission_start_state(
710 pi: PoolSpaceInfo,
711 queued: bool,
712 now: OffsetDateTime,
713 previous: Option<&PoolDecommissionInfo>,
714) -> PoolDecommissionInfo {
715 let mut info = PoolDecommissionInfo {
716 start_time: if queued { None } else { Some(now) },
717 start_size: pi.free,
718 total_size: pi.total,
719 current_size: pi.free,
720 queued,
721 ..Default::default()
722 };
723
724 if let Some(previous) = previous
725 && (previous.failed || previous.canceled)
726 {
727 info.decommissioned_buckets = previous.decommissioned_buckets.clone();
728 info.items_decommissioned = previous.items_decommissioned;
729 info.bytes_done = previous.bytes_done;
730 info.unresolved_entries = previous.unresolved_entries.clone();
731 if let Some(generation) = info.start_time {
732 for entry in &mut info.unresolved_entries {
733 entry.source_generation = generation;
734 }
735 }
736 info.mark_progress_saved();
737 }
738
739 info
740}
741
742fn spawn_decommission_index_cancelers(
743 store: Arc<ECStore>,
744 rx: CancellationToken,
745 index_cancelers: Vec<(usize, DecommissionCancelerGuard)>,
746 entry_budget: Arc<Semaphore>,
747) -> tokio::task::JoinHandle<()> {
748 tokio::spawn(async move {
749 let mut stop_queue = false;
750
751 for (idx, canceler_guard) in index_cancelers {
752 let canceler = canceler_guard.canceler().clone();
753 if stop_queue || rx.is_cancelled() {
754 canceler.cancel();
755 store.retry_decommission_cancel_for_operation(idx, &canceler).await;
756 continue;
757 }
758
759 let worker = tokio::spawn({
760 let store = store.clone();
761 let canceler = canceler.clone();
762 let entry_budget = entry_budget.clone();
763 async move { store.do_decommission_in_routine(canceler, idx, entry_budget).await }
764 });
765 if let Err(err) = await_decommission_worker(idx, worker).await {
766 if is_decommission_capacity_blocked_error(&err) || is_decommission_target_capacity_error(&err) {
767 warn!(
768 event = EVENT_DECOMMISSION_STATE,
769 component = LOG_COMPONENT_ECSTORE,
770 subsystem = LOG_SUBSYSTEM_POOLS,
771 pool_index = idx,
772 state = "capacity_blocked",
773 error = %err,
774 "Decommission routine paused without a terminal transition"
775 );
776 store.release_decommission_canceler_slot(idx, &canceler).await;
777 stop_queue = true;
778 continue;
779 }
780 if let Err(blocked) = store
781 .ensure_pool_meta_side_effects_safe("decommission paused because pool metadata requires recovery")
782 .await
783 {
784 warn!(
785 event = EVENT_DECOMMISSION_STATE,
786 component = LOG_COMPONENT_ECSTORE,
787 subsystem = LOG_SUBSYSTEM_POOLS,
788 pool_index = idx,
789 state = "routine_blocked",
790 error = %blocked,
791 "Decommission routine paused without changing terminal state"
792 );
793 store.release_decommission_canceler_slot(idx, &canceler).await;
794 stop_queue = true;
795 continue;
796 }
797 error!(
798 event = EVENT_DECOMMISSION_STATE,
799 component = LOG_COMPONENT_ECSTORE,
800 subsystem = LOG_SUBSYSTEM_POOLS,
801 pool_index = idx,
802 state = "routine_failed",
803 error = %err,
804 "Decommission routine failed"
805 );
806 store.quiesce_decommission_worker_after_join_error(&canceler).await;
807 store.retry_decommission_failed_for_operation(idx, &canceler).await;
808 stop_queue = true;
809 continue;
810 }
811
812 stop_queue = {
813 let pool_meta = store.pool_meta.read().await;
814 !should_continue_decommission_queue(&pool_meta, idx)
815 };
816 }
817 })
818}
819
820fn decommission_meta_bucket_options() -> MakeBucketOptions {
821 MakeBucketOptions {
822 force_create: true,
823 ..Default::default()
824 }
825}
826
827fn is_decommission_active(complete: bool, failed: bool, canceled: bool) -> bool {
828 !complete && !failed && !canceled
829}
830
831pub(crate) fn pool_meta_has_active_decommission(meta: &PoolMeta) -> bool {
832 meta.pools.iter().any(|pool| {
833 pool.decommission.as_ref().is_some_and(|info| {
834 info.has_decommission_state() && is_decommission_active(info.complete, info.failed, info.canceled)
835 })
836 })
837}
838
839fn is_decommission_suspended(info: &PoolDecommissionInfo) -> bool {
840 info.has_decommission_state()
841}
842
843fn validate_decommission_terminal_state(complete: bool, failed: bool, canceled: bool) -> Result<()> {
844 let terminal_count = [complete, failed, canceled].into_iter().filter(|terminal| *terminal).count();
845 if terminal_count > 1 {
846 return Err(Error::other(format!(
847 "pool metadata load failed: invalid decommission terminal state complete={complete} failed={failed} canceled={canceled}"
848 )));
849 }
850 Ok(())
851}
852
853fn invalid_decommission_pool_index_error(pool_count: usize, idx: usize) -> Error {
854 Error::other(format!("invalid decommission pool index {idx} for {pool_count} pools"))
855}
856
857#[derive(Debug, Clone, Copy, PartialEq, Eq)]
858enum DecommissionStartPoolState {
859 Missing,
860 Active,
861 Retryable,
862 Decommissioning,
863 Decommissioned,
864 Blocked,
865}
866
867fn decommission_start_pool_state(pool: Option<&PoolStatus>) -> DecommissionStartPoolState {
868 let Some(pool) = pool else {
869 return DecommissionStartPoolState::Missing;
870 };
871 let Some(info) = pool.decommission.as_ref() else {
872 return DecommissionStartPoolState::Active;
873 };
874 if !info.has_decommission_state() {
875 return DecommissionStartPoolState::Active;
876 }
877
878 if info.complete {
879 DecommissionStartPoolState::Decommissioned
880 } else if info.failed || info.canceled {
881 if info.unresolved_entries.is_empty() {
882 DecommissionStartPoolState::Blocked
883 } else {
884 DecommissionStartPoolState::Retryable
885 }
886 } else {
887 DecommissionStartPoolState::Decommissioning
888 }
889}
890
891fn is_decommission_start_active_pool(pool: &PoolStatus) -> bool {
892 decommission_start_pool_state(Some(pool)) == DecommissionStartPoolState::Active
893}
894
895fn invalid_decommission_request(reason: impl Into<String>) -> Error {
896 Error::InvalidArgument("decommission".to_string(), "pool-state".to_string(), reason.into())
897}
898
899fn ensure_decommission_start_allowed(state: DecommissionStartPoolState) -> Result<()> {
900 match state {
901 DecommissionStartPoolState::Missing => {
902 Err(invalid_decommission_request("failed to start decommission: target pool was not found"))
903 }
904 DecommissionStartPoolState::Active | DecommissionStartPoolState::Retryable => Ok(()),
905 DecommissionStartPoolState::Decommissioning => Err(StorageError::DecommissionAlreadyRunning),
906 DecommissionStartPoolState::Decommissioned => Err(invalid_decommission_request(
907 "failed to start decommission: target pool is already decommissioned",
908 )),
909 DecommissionStartPoolState::Blocked => Err(invalid_decommission_request(
910 "failed to start decommission: target pool decommission is blocked; clear failed or canceled metadata before starting again",
911 )),
912 }
913}
914
915fn ensure_decommission_start_keeps_active_pool(meta: &PoolMeta, indices: &[usize]) -> Result<()> {
916 let active_count = meta
917 .pools
918 .iter()
919 .filter(|pool| is_decommission_start_active_pool(pool))
920 .count();
921 let active_target_count = indices
922 .iter()
923 .filter(|idx| meta.pools.get(**idx).is_some_and(is_decommission_start_active_pool))
924 .count();
925 if active_count.saturating_sub(active_target_count) == 0 {
926 return Err(invalid_decommission_request(
927 "failed to start decommission: at least one active pool must remain after decommission start",
928 ));
929 }
930
931 Ok(())
932}
933
934fn ensure_decommission_start_pool_states(meta: &PoolMeta, indices: &[usize]) -> Result<()> {
935 for idx in indices.iter().copied() {
936 ensure_decommission_start_allowed(decommission_start_pool_state(meta.pools.get(idx)))?;
937 }
938 ensure_decommission_start_keeps_active_pool(meta, indices)
939}
940
941fn capacity_mul_div_ceil(value: usize, multiplier: usize, divisor: usize) -> usize {
942 if value == 0 || multiplier == 0 {
943 return 0;
944 }
945 if divisor == 0 {
946 return usize::MAX;
947 }
948 value
949 .checked_mul(multiplier)
950 .map(|product| product.div_ceil(divisor))
951 .unwrap_or(usize::MAX)
952}
953
954fn capacity_source_data_equivalent(physical_bytes: usize, layout: DecommissionErasureLayout) -> Result<usize> {
955 if !layout.is_valid() {
956 return Err(Error::DecommissionCapacity(format!(
957 "failed to model decommission capacity: invalid source layout data={} parity={}",
958 layout.data, layout.parity
959 )));
960 }
961 Ok(capacity_mul_div_ceil(physical_bytes, layout.data, layout.width()))
962}
963
964fn capacity_target_physical_bytes(data_bytes: usize, layout: DecommissionErasureLayout) -> Result<usize> {
965 if !layout.is_valid() {
966 return Err(Error::DecommissionCapacity(format!(
967 "failed to model decommission capacity: invalid target layout data={} parity={}",
968 layout.data, layout.parity
969 )));
970 }
971 Ok(capacity_mul_div_ceil(data_bytes, layout.width(), layout.data))
972}
973
974async fn decommission_owned_physical_usage(sets: &Sets, layout: DecommissionErasureLayout) -> HashMap<i32, usize> {
975 let mut usage = HashMap::with_capacity(sets.disk_set.len());
976 for (set_index, set) in sets.disk_set.iter().enumerate() {
977 let Ok(cache) = load_data_usage_cache(set.as_ref(), DATA_USAGE_CACHE_NAME).await else {
978 continue;
979 };
980 let data_usage = cache.dui(DATA_USAGE_ROOT, &[]);
981 if !data_usage.is_complete_bucket_usage_snapshot() {
982 continue;
983 }
984 let Ok(logical_bytes) = usize::try_from(data_usage.objects_total_size) else {
985 continue;
986 };
987 let Ok(set_index) = i32::try_from(set_index) else {
988 continue;
989 };
990 usage.insert(set_index, capacity_mul_div_ceil(logical_bytes, layout.width(), layout.data));
993 }
994 usage
995}
996
997fn worst_decommission_target_layout(targets: &[DecommissionPoolCapacityInfo]) -> Result<DecommissionErasureLayout> {
998 targets
999 .iter()
1000 .map(|target| target.layout)
1001 .filter(|layout| layout.is_valid())
1002 .max_by(|left, right| {
1003 ((left.width() as u128) * (right.data as u128)).cmp(&((right.width() as u128) * (left.data as u128)))
1004 })
1005 .ok_or_else(|| Error::other("failed to start decommission: no valid target erasure layout is available"))
1006}
1007
1008fn decommission_capacity_writer_supported(meta: &PoolMeta) -> bool {
1009 decommission_capacity_writer_supported_for(meta.version, pool_meta_v2_writer_enabled(), pool_meta_v3_writer_enabled())
1010}
1011
1012fn ensure_decommission_capacity_writer_supported(meta: &PoolMeta) -> Result<()> {
1013 if decommission_capacity_writer_supported(meta) {
1014 return Ok(());
1015 }
1016 Err(Error::DecommissionCapacity(format!(
1017 "failed to start decommission: durable capacity reservations require pool metadata V2 or V3; enable either the {} + {} V2 gate or the {} + {} V3 gate only after every reader and writer supports that format",
1018 rustfs_config::ENV_POOL_META_V2_WRITE,
1019 rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
1020 rustfs_config::ENV_POOL_META_V3_WRITE,
1021 rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED,
1022 )))
1023}
1024
1025fn decommission_capacity_blocked_error(message: impl Display) -> Error {
1026 Error::DecommissionCapacityBlocked {
1027 message: message.to_string(),
1028 }
1029}
1030
1031fn is_decommission_capacity_blocked_error(err: &Error) -> bool {
1032 if matches!(err, Error::DecommissionCapacityBlocked { .. }) {
1033 return true;
1034 }
1035 data_movement::data_movement_stage_source(err).is_some_and(is_decommission_capacity_blocked_error)
1036}
1037
1038fn is_decommission_capacity_intent_conflict(err: &Error) -> bool {
1039 if let Error::DecommissionCapacityBlocked { message } = err {
1040 return message.contains("unresolved target capacity intent")
1041 || message.contains("pending capacity intent belongs to another mutation");
1042 }
1043 data_movement::data_movement_stage_source(err).is_some_and(is_decommission_capacity_intent_conflict)
1044 || err.to_string().contains("unresolved target capacity intent")
1045}
1046
1047pub(crate) fn decommission_capacity_target_gate_busy_index(err: &Error) -> Option<usize> {
1048 if let Error::DecommissionCapacityBlocked { message } = err {
1049 return message
1050 .strip_prefix(DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX)?
1051 .strip_suffix(DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX)?
1052 .parse()
1053 .ok();
1054 }
1055 data_movement::data_movement_stage_source(err).and_then(decommission_capacity_target_gate_busy_index)
1056}
1057
1058pub(crate) fn is_decommission_capacity_target_gate_busy(err: &Error) -> bool {
1059 decommission_capacity_target_gate_busy_index(err).is_some()
1060}
1061
1062#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1068pub(crate) enum DecommissionCapacityFailure {
1069 TargetGateBusy { target_pool_index: usize },
1072 IntentConflict,
1075 BenignContention,
1078 Fatal,
1080}
1081
1082fn is_decommission_benign_contention_error(err: &Error) -> bool {
1089 if is_decommission_copy_cleanup_safe_error(err) || is_err_data_movement_overwrite(err) || is_err_invalid_upload_id(err) {
1090 return true;
1091 }
1092 data_movement::data_movement_stage_source(err).is_some_and(is_decommission_benign_contention_error)
1093}
1094
1095fn classify_decommission_capacity_failure(err: &Error) -> DecommissionCapacityFailure {
1096 if let Some(target_pool_index) = decommission_capacity_target_gate_busy_index(err) {
1097 return DecommissionCapacityFailure::TargetGateBusy { target_pool_index };
1098 }
1099 if is_decommission_capacity_intent_conflict(err) {
1100 return DecommissionCapacityFailure::IntentConflict;
1101 }
1102 if is_decommission_benign_contention_error(err) {
1103 return DecommissionCapacityFailure::BenignContention;
1104 }
1105 DecommissionCapacityFailure::Fatal
1106}
1107
1108#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1109enum DecommissionCapacityRetryKind {
1110 IntentConflict,
1111}
1112
1113fn decommission_capacity_retry_kind(err: &Error, intent_conflict_attempt: usize) -> Option<DecommissionCapacityRetryKind> {
1114 (intent_conflict_attempt < DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS
1115 && matches!(classify_decommission_capacity_failure(err), DecommissionCapacityFailure::IntentConflict))
1116 .then_some(DecommissionCapacityRetryKind::IntentConflict)
1117}
1118
1119#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1120enum DecommissionScannerBacklogRetryKind {
1121 HandoffConflict,
1122}
1123
1124fn is_decommission_scanner_backlog_handoff_conflict(err: &Error) -> bool {
1125 data_movement::data_movement_stage_source_as::<data_movement::scanner_backlog::ScannerPauseBacklogRetirementError>(err)
1126 .is_some_and(data_movement::scanner_backlog::ScannerPauseBacklogRetirementError::is_retryable_handoff_conflict)
1127}
1128
1129fn decommission_scanner_backlog_retry_kind(
1135 err: &Error,
1136 handoff_conflict_attempt: usize,
1137) -> Option<DecommissionScannerBacklogRetryKind> {
1138 (handoff_conflict_attempt < DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS
1139 && is_decommission_scanner_backlog_handoff_conflict(err))
1140 .then_some(DecommissionScannerBacklogRetryKind::HandoffConflict)
1141}
1142
1143fn ensure_decommission_capacity_target_fence(
1144 guard: &rustfs_lock::NamespaceLockGuard,
1145 target_pool_index: usize,
1146 phase: &str,
1147) -> Result<()> {
1148 if guard.is_lock_lost() {
1149 return Err(Error::DecommissionCapacity(format!(
1150 "target pool {target_pool_index} capacity mutation fence was lost during {phase}"
1151 )));
1152 }
1153 Ok(())
1154}
1155
1156fn ensure_decommission_capacity_mutation_intent_current(
1157 meta: &PoolMeta,
1158 owner: DecommissionCapacityOwner,
1159 target_pool_index: usize,
1160 expected_target_physical_bytes: usize,
1161 mutation_id: uuid::Uuid,
1162 temporary_release: bool,
1163 model_version: u16,
1164) -> Result<()> {
1165 let reservation = meta
1166 .pools
1167 .get(owner.source_pool_index)
1168 .and_then(|pool| pool.decommission.as_ref())
1169 .and_then(|info| info.capacity_reservation.as_ref())
1170 .filter(|reservation| {
1171 reservation.active()
1176 && reservation.source_pool_index == owner.source_pool_index
1177 && reservation.operation_id == owner.operation_id
1178 && reservation.generation == owner.generation
1179 && reservation.model_version == model_version
1180 })
1181 .ok_or_else(|| {
1182 decommission_capacity_blocked_error("decommission target mutation owner changed before capacity finalize")
1183 })?;
1184 let target = reservation
1185 .targets
1186 .iter()
1187 .find(|target| target.pool_index == target_pool_index)
1188 .ok_or_else(|| decommission_capacity_blocked_error("decommission target allocation changed before capacity finalize"))?;
1189
1190 if temporary_release {
1191 return Ok(());
1196 }
1197
1198 if expected_target_physical_bytes == 0
1199 || (target.pending_mutation_id == Some(mutation_id) && target.pending_physical_bytes >= expected_target_physical_bytes)
1200 {
1201 return Ok(());
1202 }
1203 Err(decommission_capacity_blocked_error(
1204 "pending capacity intent belongs to another mutation before capacity finalize",
1205 ))
1206}
1207
1208fn validate_decommission_capacity_reservation(reservation: Option<&DecommissionCapacityReservation>) -> Result<()> {
1209 let Some(reservation) = reservation else {
1210 return Ok(());
1211 };
1212 if !matches!(
1213 reservation.model_version,
1214 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
1215 ) {
1216 return Err(Error::DecommissionCapacity(format!(
1217 "pool metadata load failed: unsupported decommission capacity model version {}",
1218 reservation.model_version
1219 )));
1220 }
1221 if reservation.operation_id.is_nil() || reservation.generation == 0 || reservation.owner_nonce.is_nil() {
1222 return Err(Error::other(
1223 "pool metadata load failed: decommission capacity reservation identity is invalid",
1224 ));
1225 }
1226 if !reservation.source_layout.is_valid() || reservation.targets.iter().any(|target| !target.layout.is_valid()) {
1227 return Err(Error::other(
1228 "pool metadata load failed: decommission capacity reservation contains an invalid erasure layout",
1229 ));
1230 }
1231 if reservation.temporary_copies != DECOMMISSION_CAPACITY_TEMPORARY_COPIES {
1232 return Err(Error::other(
1233 "pool metadata load failed: decommission capacity reservation temporary-copy model is invalid",
1234 ));
1235 }
1236 if reservation.source_data_equivalent_bytes
1237 != capacity_source_data_equivalent(reservation.source_physical_bytes, reservation.source_layout)?
1238 {
1239 return Err(Error::other(
1240 "pool metadata load failed: decommission capacity reservation source estimate is invalid",
1241 ));
1242 }
1243 if reservation.peak_physical_bytes
1244 != reservation
1245 .predicted_physical_bytes
1246 .saturating_add(reservation.temporary_physical_bytes)
1247 || reservation.temporary_physical_bytes
1248 != reservation
1249 .predicted_physical_bytes
1250 .saturating_mul(reservation.temporary_copies)
1251 {
1252 return Err(Error::other(
1253 "pool metadata load failed: decommission capacity reservation peak does not match its physical model",
1254 ));
1255 }
1256 if reservation.committed_data_bytes > reservation.source_data_equivalent_bytes {
1257 return Err(Error::other(
1258 "pool metadata load failed: decommission capacity committed bytes exceed the source estimate",
1259 ));
1260 }
1261 if reservation.consumed_target_physical_bytes > reservation.predicted_physical_bytes {
1262 return Err(Error::other(
1263 "pool metadata load failed: decommission capacity consumed target bytes exceed the prediction",
1264 ));
1265 }
1266 let mut target_indices = HashSet::with_capacity(reservation.targets.len());
1267 let mut reserved_physical_bytes = 0usize;
1268 let mut consumed_physical_bytes = 0usize;
1269 let mut observed_physical_bytes = 0usize;
1270 let mut inflight_physical_bytes = 0usize;
1271 let mut pending_physical_bytes = 0usize;
1272 for target in &reservation.targets {
1273 let mut temporary_mutation_ids = HashSet::with_capacity(target.temporary_mutations.len());
1274 let temporary_mutation_bytes = target.temporary_mutations.iter().try_fold(0usize, |total, mutation| {
1275 if mutation.mutation_id.is_nil()
1276 || (mutation.physical_bytes == 0 && reservation.model_version != DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION)
1277 || !temporary_mutation_ids.insert(mutation.mutation_id)
1278 {
1279 return Err(Error::other(
1280 "pool metadata load failed: decommission capacity temporary mutation is invalid",
1281 ));
1282 }
1283 Ok(total.saturating_add(mutation.physical_bytes))
1284 })?;
1285 if target.pool_index == reservation.source_pool_index
1286 || !target_indices.insert(target.pool_index)
1287 || target.reserved_physical_bytes == 0
1288 || target.reserved_physical_bytes > target.physical_free_at_reservation
1289 || target.physical_free_at_reservation > target.physical_total_at_reservation
1290 || target.consumed_physical_bytes > reservation.consumed_target_physical_bytes
1291 || target.observed_physical_bytes > reservation.observed_target_physical_bytes
1292 || target.inflight_physical_bytes > target.observed_physical_bytes
1293 || temporary_mutation_bytes > target.inflight_physical_bytes
1294 {
1295 return Err(Error::other(
1296 "pool metadata load failed: decommission capacity reservation target allocation is invalid",
1297 ));
1298 }
1299 if reservation.predicted_physical_bytes
1300 < capacity_target_physical_bytes(reservation.source_data_equivalent_bytes, target.layout)?
1301 {
1302 return Err(Error::other(
1303 "pool metadata load failed: decommission capacity reservation target estimate is unsafe",
1304 ));
1305 }
1306 reserved_physical_bytes =
1307 reserved_physical_bytes.saturating_add(target.remaining_reserved_physical_bytes(reservation.temporary_copies));
1308 consumed_physical_bytes = consumed_physical_bytes.saturating_add(target.consumed_physical_bytes);
1309 observed_physical_bytes = observed_physical_bytes.saturating_add(target.observed_physical_bytes);
1310 inflight_physical_bytes = inflight_physical_bytes.saturating_add(target.inflight_physical_bytes);
1311 pending_physical_bytes = pending_physical_bytes.saturating_add(target.pending_physical_bytes);
1312 }
1313 if reserved_physical_bytes != reservation.remaining_peak_physical_bytes() {
1314 return Err(Error::other(
1315 "pool metadata load failed: decommission capacity reservation target allocation is incomplete",
1316 ));
1317 }
1318 if inflight_physical_bytes != reservation.inflight_target_physical_bytes {
1319 return Err(Error::other(
1320 "pool metadata load failed: decommission capacity inflight target accounting is incomplete",
1321 ));
1322 }
1323 if pending_physical_bytes != reservation.pending_target_physical_bytes {
1324 return Err(Error::other(
1325 "pool metadata load failed: decommission capacity pending target accounting is incomplete",
1326 ));
1327 }
1328 if consumed_physical_bytes != reservation.consumed_target_physical_bytes
1329 || observed_physical_bytes != reservation.observed_target_physical_bytes
1330 {
1331 return Err(Error::other(
1332 "pool metadata load failed: decommission capacity target progress accounting is incomplete",
1333 ));
1334 }
1335 if reservation.released_at.is_some() != reservation.release_reason.is_some() {
1336 return Err(Error::other(
1337 "pool metadata load failed: decommission capacity reservation release state is incomplete",
1338 ));
1339 }
1340 Ok(())
1341}
1342
1343fn active_decommission_capacity_model(meta: &PoolMeta) -> Result<Option<u16>> {
1344 let mut active_model = None;
1345 for reservation in meta
1346 .pools
1347 .iter()
1348 .filter_map(|pool| pool.decommission.as_ref()?.capacity_reservation.as_ref())
1349 .filter(|reservation| reservation.active())
1350 {
1351 if !matches!(
1352 reservation.model_version,
1353 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
1354 ) {
1355 return Err(Error::DecommissionCapacity(format!(
1356 "pool metadata load failed: unsupported active decommission capacity model version {}",
1357 reservation.model_version
1358 )));
1359 }
1360 match active_model {
1361 Some(model_version) if model_version != reservation.model_version => {
1362 return Err(Error::DecommissionCapacity(
1363 "pool metadata load failed: active decommission capacity reservations use mixed lock models".to_string(),
1364 ));
1365 }
1366 Some(_) => {}
1367 None => active_model = Some(reservation.model_version),
1368 }
1369 }
1370 Ok(active_model)
1371}
1372
1373fn validate_decommission_capacity_model_cohort(meta: &PoolMeta) -> Result<()> {
1374 active_decommission_capacity_model(meta).map(|_| ())
1375}
1376
1377fn select_decommission_capacity_model(meta: &PoolMeta, target_fence_proof_available: bool) -> Result<u16> {
1378 match active_decommission_capacity_model(meta)? {
1379 Some(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION) => Ok(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION),
1380 Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) if target_fence_proof_available => {
1381 Ok(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION)
1382 }
1383 Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) => Err(Error::DecommissionCapacity(
1384 "failed to start decommission: an active per-target capacity cohort requires a live all-v4 fleet proof".to_string(),
1385 )),
1386 Some(version) => Err(Error::DecommissionCapacity(format!(
1387 "failed to start decommission: unsupported active capacity model version {version}"
1388 ))),
1389 None if target_fence_proof_available => Ok(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION),
1390 None => Ok(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION),
1391 }
1392}
1393
1394fn ensure_decommission_target_fence_fleet_proof(
1395 proof: Option<&crate::services::notification_sys::DecommissionTargetFenceFleetProofToken>,
1396 required: bool,
1397) -> Result<()> {
1398 if !required {
1399 return Ok(());
1400 }
1401 if proof.is_some_and(crate::services::notification_sys::decommission_target_fence_fleet_proof_matches) {
1402 return Ok(());
1403 }
1404 Err(Error::other(DECOMMISSION_TARGET_FLEET_PROOF_EXPIRED))
1405}
1406
1407fn release_decommission_capacity_reservation(info: &mut PoolDecommissionInfo, reason: &str, now: OffsetDateTime) -> bool {
1408 let Some(reservation) = info.capacity_reservation.as_mut() else {
1409 return false;
1410 };
1411 if !reservation.active() {
1412 return false;
1413 }
1414 reservation.released_at = Some(now);
1415 reservation.release_reason = Some(reason.to_string());
1416 metrics::gauge!(
1417 METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES,
1418 "pool_index" => reservation.source_pool_index.to_string()
1419 )
1420 .set(0.0);
1421 true
1422}
1423
1424fn renew_decommission_capacity_reservation(
1425 reservation: &mut DecommissionCapacityReservation,
1426 now: OffsetDateTime,
1427 recover_expired: bool,
1428) -> bool {
1429 if !reservation.active() {
1430 return false;
1431 }
1432 let expired = reservation.expires_at <= now;
1433 if expired && recover_expired {
1434 reservation.owner_nonce = uuid::Uuid::new_v4();
1435 reservation.recovered_at = Some(now);
1436 }
1437 reservation.renewed_at = now;
1438 reservation.expires_at = now + DECOMMISSION_CAPACITY_RESERVATION_TTL;
1439 true
1440}
1441
1442fn next_decommission_capacity_generation(meta: &PoolMeta) -> Result<u64> {
1443 meta.pools
1444 .iter()
1445 .filter_map(|pool| pool.decommission.as_ref()?.capacity_reservation.as_ref())
1446 .map(|reservation| reservation.generation)
1447 .max()
1448 .unwrap_or_default()
1449 .checked_add(1)
1450 .ok_or_else(|| Error::other("failed to start decommission: capacity reservation generation overflow"))
1451}
1452
1453fn active_decommission_source_indices(meta: &PoolMeta) -> HashSet<usize> {
1454 meta.pools
1455 .iter()
1456 .enumerate()
1457 .filter_map(|(idx, pool)| {
1458 pool.decommission.as_ref().and_then(|info| {
1459 (info.has_decommission_state() && is_decommission_active(info.complete, info.failed, info.canceled))
1460 .then_some(idx)
1461 })
1462 })
1463 .collect()
1464}
1465
1466#[cfg(test)]
1467fn build_decommission_capacity_reservation(
1468 source: DecommissionPoolCapacityInfo,
1469 target_layout: DecommissionErasureLayout,
1470 operation_id: uuid::Uuid,
1471 generation: u64,
1472 now: OffsetDateTime,
1473) -> Result<DecommissionCapacityReservation> {
1474 build_decommission_capacity_reservation_with_model(
1475 source,
1476 target_layout,
1477 operation_id,
1478 generation,
1479 now,
1480 DECOMMISSION_CAPACITY_MODEL_VERSION,
1481 )
1482}
1483
1484fn build_decommission_capacity_reservation_with_model(
1485 source: DecommissionPoolCapacityInfo,
1486 target_layout: DecommissionErasureLayout,
1487 operation_id: uuid::Uuid,
1488 generation: u64,
1489 now: OffsetDateTime,
1490 model_version: u16,
1491) -> Result<DecommissionCapacityReservation> {
1492 if !matches!(
1493 model_version,
1494 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
1495 ) {
1496 return Err(Error::DecommissionCapacity(format!(
1497 "failed to build decommission capacity reservation: unsupported lock model {model_version}"
1498 )));
1499 }
1500 let source_data_equivalent_bytes = capacity_source_data_equivalent(source.physical_used, source.layout)?;
1501 let predicted_physical_bytes = capacity_target_physical_bytes(source_data_equivalent_bytes, target_layout)?;
1502 let temporary_physical_bytes = predicted_physical_bytes.saturating_mul(DECOMMISSION_CAPACITY_TEMPORARY_COPIES);
1503 Ok(DecommissionCapacityReservation {
1504 model_version,
1505 operation_id,
1506 generation,
1507 owner_nonce: uuid::Uuid::new_v4(),
1508 source_pool_index: source.pool_index,
1509 source_layout: source.layout,
1510 source_physical_total_bytes: source.physical_total,
1511 source_physical_bytes: source.physical_used,
1512 source_data_equivalent_bytes,
1513 predicted_physical_bytes,
1514 temporary_copies: DECOMMISSION_CAPACITY_TEMPORARY_COPIES,
1515 temporary_physical_bytes,
1516 peak_physical_bytes: predicted_physical_bytes.saturating_add(temporary_physical_bytes),
1517 committed_data_bytes: 0,
1518 consumed_target_physical_bytes: 0,
1519 observed_target_physical_bytes: 0,
1520 inflight_target_physical_bytes: 0,
1521 pending_target_physical_bytes: 0,
1522 prediction_error_bytes: 0,
1523 targets: Vec::new(),
1524 created_at: now,
1525 renewed_at: now,
1526 expires_at: now + DECOMMISSION_CAPACITY_RESERVATION_TTL,
1527 recovered_at: None,
1528 released_at: None,
1529 release_reason: None,
1530 })
1531}
1532
1533fn reserve_decommission_start_target_capacity(
1534 meta: &mut PoolMeta,
1535 requested_indices: &[usize],
1536 capacity_infos: &[DecommissionPoolCapacityInfo],
1537 operation_id: uuid::Uuid,
1538 generation: u64,
1539 now: OffsetDateTime,
1540 model_version: u16,
1541) -> Result<()> {
1542 validate_decommission_capacity_model_cohort(meta)?;
1543 let active_sources = active_decommission_source_indices(meta);
1544 let targets = capacity_infos
1545 .iter()
1546 .copied()
1547 .filter(|capacity| {
1548 !active_sources.contains(&capacity.pool_index)
1549 && meta
1550 .pools
1551 .get(capacity.pool_index)
1552 .is_some_and(is_decommission_start_active_pool)
1553 })
1554 .collect::<Vec<_>>();
1555 let target_layout = worst_decommission_target_layout(&targets)?;
1556 let requested = requested_indices.iter().copied().collect::<HashSet<_>>();
1557 let mut reservations = Vec::with_capacity(active_sources.len());
1558
1559 let mut source_indices = active_sources.into_iter().collect::<Vec<_>>();
1560 source_indices.sort_unstable();
1561 for source_index in source_indices {
1562 let source = capacity_infos
1563 .iter()
1564 .copied()
1565 .find(|capacity| capacity.pool_index == source_index)
1566 .ok_or_else(|| {
1567 Error::DecommissionCapacity(format!(
1568 "failed to start decommission: capacity snapshot is missing source pool {source_index}"
1569 ))
1570 })?;
1571 let pool = meta
1572 .pools
1573 .get(source_index)
1574 .ok_or_else(|| invalid_decommission_pool_index_error(meta.pools.len(), source_index))?;
1575 let info = pool
1576 .decommission
1577 .as_ref()
1578 .ok_or_else(|| decommission_metadata_not_initialized_error("reserve decommission capacity"))?;
1579 let new_reservation = requested.contains(&source_index);
1580 let mut reservation = if new_reservation {
1581 build_decommission_capacity_reservation_with_model(
1582 source,
1583 target_layout,
1584 operation_id,
1585 generation,
1586 now,
1587 model_version,
1588 )?
1589 } else {
1590 info.capacity_reservation.clone().ok_or_else(|| {
1591 Error::DecommissionCapacity(format!(
1592 "failed to start decommission: active source pool {source_index} has no durable capacity reservation"
1593 ))
1594 })?
1595 };
1596 if !reservation.active() {
1597 return Err(Error::DecommissionCapacity(format!(
1598 "failed to start decommission: active source pool {source_index} has a released capacity reservation"
1599 )));
1600 }
1601 renew_decommission_capacity_reservation(&mut reservation, now, true);
1602 reservations.push((source_index, reservation, new_reservation));
1603 }
1604
1605 let mut remaining_target_capacity = targets
1606 .iter()
1607 .map(|target| (target.pool_index, target.physical_free))
1608 .collect::<HashMap<_, _>>();
1609 for (_, reservation, new_reservation) in &reservations {
1610 if *new_reservation {
1611 continue;
1612 }
1613 for target in &reservation.targets {
1614 let available = remaining_target_capacity.entry(target.pool_index).or_default();
1615 *available = available
1616 .saturating_add(target.inflight_physical_bytes)
1617 .saturating_add(target.pending_physical_bytes);
1618 let required = target.remaining_reserved_physical_bytes(reservation.temporary_copies);
1619 if *available < required {
1620 return Err(decommission_capacity_blocked_error(format!(
1621 "failed to start decommission: existing reservation for source pool {} requires {required} bytes on target pool {}, but only {} bytes remain",
1622 reservation.source_pool_index, target.pool_index, *available
1623 )));
1624 }
1625 *available -= required;
1626 }
1627 }
1628 let required = reservations
1629 .iter()
1630 .filter(|(_, _, new_reservation)| *new_reservation)
1631 .fold(0usize, |total, (_, reservation, _)| {
1632 total.saturating_add(reservation.remaining_peak_physical_bytes())
1633 });
1634 let available = remaining_target_capacity
1635 .values()
1636 .copied()
1637 .fold(0usize, usize::saturating_add);
1638 if available < required {
1639 metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => "activation").increment(1);
1640 return Err(decommission_capacity_blocked_error(format!(
1641 "failed to start decommission: insufficient reserved physical target capacity: operation {operation_id} generation {generation} requires {required} bytes, but {available} bytes are available after source/target parity and temporary-copy accounting"
1642 )));
1643 }
1644
1645 for (_, reservation, new_reservation) in &mut reservations {
1646 if !*new_reservation {
1647 continue;
1648 }
1649 let mut remaining = reservation.remaining_peak_physical_bytes();
1650 for target in &targets {
1651 if remaining == 0 {
1652 break;
1653 }
1654 let target_remaining = remaining_target_capacity.entry(target.pool_index).or_default();
1655 let allocated = remaining.min(*target_remaining);
1656 if allocated == 0 {
1657 continue;
1658 }
1659 *target_remaining -= allocated;
1660 remaining -= allocated;
1661 reservation.targets.push(DecommissionCapacityTarget {
1662 pool_index: target.pool_index,
1663 layout: target.layout,
1664 physical_total_at_reservation: target.physical_total,
1665 physical_free_at_reservation: target.physical_free,
1666 reserved_physical_bytes: allocated,
1667 consumed_physical_bytes: 0,
1668 observed_physical_bytes: 0,
1669 inflight_physical_bytes: 0,
1670 pending_physical_bytes: 0,
1671 pending_mutation_id: None,
1672 temporary_mutations: Vec::new(),
1673 });
1674 }
1675 debug_assert_eq!(remaining, 0);
1676 }
1677
1678 for (source_index, reservation, _) in reservations {
1679 metrics::gauge!(
1680 METRIC_DECOMMISSION_CAPACITY_PREDICTED_BYTES,
1681 "pool_index" => source_index.to_string()
1682 )
1683 .set(reservation.predicted_physical_bytes as f64);
1684 metrics::gauge!(
1685 METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES,
1686 "pool_index" => source_index.to_string()
1687 )
1688 .set(reservation.remaining_peak_physical_bytes() as f64);
1689 let pool_count = meta.pools.len();
1690 let info = meta
1691 .pools
1692 .get_mut(source_index)
1693 .and_then(|pool| pool.decommission.as_mut())
1694 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_index))?;
1695 info.capacity_blocked_reason = None;
1696 info.capacity_reservation = Some(reservation);
1697 }
1698 if meta.version != POOL_META_GENERATION_VERSION {
1699 meta.version = POOL_META_VERSION;
1700 }
1701 validate_decommission_capacity_model_cohort(meta)?;
1702 Ok(())
1703}
1704
1705fn ensure_decommission_start_target_capacity(
1706 meta: &PoolMeta,
1707 indices: &[usize],
1708 capacity_infos: &[DecommissionPoolCapacityInfo],
1709 target_fence_proof_available: bool,
1710) -> Result<()> {
1711 let generation = next_decommission_capacity_generation(meta)?;
1712 let mut projected = meta.clone();
1713 let first_idx = indices.first().copied();
1714 for idx in indices.iter().copied() {
1715 let capacity = capacity_infos
1716 .iter()
1717 .find(|capacity| capacity.pool_index == idx)
1718 .ok_or_else(|| Error::DecommissionCapacity(format!("decommission capacity snapshot is missing pool {idx}")))?;
1719 if Some(idx) == first_idx {
1720 projected.decommission(idx, capacity.space)?;
1721 } else {
1722 projected.queue_decommission(idx, capacity.space)?;
1723 }
1724 }
1725 let model_version = select_decommission_capacity_model(&projected, target_fence_proof_available)?;
1726 reserve_decommission_start_target_capacity(
1727 &mut projected,
1728 indices,
1729 capacity_infos,
1730 uuid::Uuid::new_v4(),
1731 generation,
1732 OffsetDateTime::now_utc(),
1733 model_version,
1734 )
1735}
1736
1737fn recover_decommission_capacity_reservations(
1738 meta: &mut PoolMeta,
1739 capacity_infos: &[DecommissionPoolCapacityInfo],
1740 now: OffsetDateTime,
1741 target_fence_proof_available: bool,
1742) -> Result<Vec<usize>> {
1743 ensure_decommission_capacity_writer_supported(meta)?;
1744 let mut active_indices = active_decommission_source_indices(meta).into_iter().collect::<Vec<_>>();
1745 active_indices.sort_unstable();
1746 let missing_indices = active_indices
1747 .iter()
1748 .copied()
1749 .filter(|idx| {
1750 meta.pools
1751 .get(*idx)
1752 .and_then(|pool| pool.decommission.as_ref())
1753 .and_then(|info| info.capacity_reservation.as_ref())
1754 .is_none_or(|reservation| !reservation.active())
1755 })
1756 .collect::<Vec<_>>();
1757 if missing_indices.is_empty() {
1758 validate_decommission_capacity_model_cohort(meta)?;
1759 return Ok(active_indices);
1760 }
1761 let generation = next_decommission_capacity_generation(meta)?;
1762 let model_version = select_decommission_capacity_model(meta, target_fence_proof_available)?;
1763 reserve_decommission_start_target_capacity(
1764 meta,
1765 &missing_indices,
1766 capacity_infos,
1767 uuid::Uuid::new_v4(),
1768 generation,
1769 now,
1770 model_version,
1771 )?;
1772 for idx in missing_indices {
1773 if let Some(reservation) = meta
1774 .pools
1775 .get_mut(idx)
1776 .and_then(|pool| pool.decommission.as_mut())
1777 .and_then(|info| info.capacity_reservation.as_mut())
1778 {
1779 reservation.recovered_at = Some(now);
1780 }
1781 }
1782 Ok(active_indices)
1783}
1784
1785fn signed_capacity_difference(observed: usize, predicted: usize) -> i64 {
1786 if observed >= predicted {
1787 i64::try_from(observed.saturating_sub(predicted)).unwrap_or(i64::MAX)
1788 } else {
1789 -i64::try_from(predicted.saturating_sub(observed)).unwrap_or(i64::MAX)
1790 }
1791}
1792
1793fn active_decommission_target_reservations(meta: &PoolMeta) -> HashMap<usize, usize> {
1794 let mut target_reservations = HashMap::new();
1795 for reservation in meta
1796 .pools
1797 .iter()
1798 .filter_map(|pool| pool.decommission.as_ref()?.capacity_reservation.as_ref())
1799 .filter(|reservation| reservation.active())
1800 {
1801 for target in &reservation.targets {
1802 let reserved = target_reservations.entry(target.pool_index).or_insert(0usize);
1803 *reserved = reserved.saturating_add(target.remaining_reserved_physical_bytes(reservation.temporary_copies));
1804 }
1805 }
1806 target_reservations
1807}
1808
1809fn observe_decommission_capacity_reservation(
1810 info: &mut PoolDecommissionInfo,
1811 _capacity_infos: &[DecommissionPoolCapacityInfo],
1812 _active_target_reservations: &HashMap<usize, usize>,
1813) {
1814 let Some(reservation) = info.capacity_reservation.as_mut() else {
1815 return;
1816 };
1817 reservation.prediction_error_bytes =
1818 signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes);
1819 metrics::gauge!(
1820 METRIC_DECOMMISSION_CAPACITY_PREDICTION_ERROR_BYTES,
1821 "pool_index" => reservation.source_pool_index.to_string()
1822 )
1823 .set(reservation.prediction_error_bytes as f64);
1824 metrics::histogram!(
1825 METRIC_DECOMMISSION_CAPACITY_PREDICTION_ABSOLUTE_ERROR_BYTES,
1826 "pool_index" => reservation.source_pool_index.to_string()
1827 )
1828 .record(reservation.prediction_error_bytes.unsigned_abs() as f64);
1829}
1830
1831fn ensure_decommission_capacity_reservations_available(
1832 meta: &PoolMeta,
1833 capacity_infos: &[DecommissionPoolCapacityInfo],
1834 phase: &'static str,
1835) -> Result<()> {
1836 let mut required_by_target = HashMap::<usize, usize>::new();
1837 let mut inflight_by_target = HashMap::<usize, usize>::new();
1838 let mut pending_by_target = HashMap::<usize, usize>::new();
1839 for source_index in active_decommission_source_indices(meta) {
1840 let info = meta.pools[source_index]
1841 .decommission
1842 .as_ref()
1843 .ok_or_else(|| decommission_metadata_not_initialized_error("check decommission capacity reservation"))?;
1844 let reservation = info.capacity_reservation.as_ref().ok_or_else(|| {
1845 decommission_capacity_blocked_error(format!(
1846 "active decommission source pool {source_index} has no durable capacity reservation"
1847 ))
1848 })?;
1849 if !reservation.active() {
1850 return Err(decommission_capacity_blocked_error(format!(
1851 "active decommission source pool {source_index} has a released capacity reservation"
1852 )));
1853 }
1854 metrics::gauge!(
1855 METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES,
1856 "pool_index" => source_index.to_string()
1857 )
1858 .set(reservation.remaining_peak_physical_bytes() as f64);
1859 for target in &reservation.targets {
1860 let temporary_budget = target.remaining_reserved_physical_bytes(reservation.temporary_copies)
1861 / 1usize.saturating_add(reservation.temporary_copies);
1862 if target.inflight_physical_bytes > temporary_budget {
1863 return Err(decommission_capacity_blocked_error(format!(
1864 "source pool {source_index} target pool {} has {} inflight physical bytes, exceeding its {temporary_budget}-byte temporary-copy reservation during {phase}",
1865 target.pool_index, target.inflight_physical_bytes
1866 )));
1867 }
1868 if target.pending_physical_bytes > temporary_budget {
1869 return Err(decommission_capacity_blocked_error(format!(
1870 "source pool {source_index} target pool {} has {} pending physical bytes, exceeding its {temporary_budget}-byte committed-copy reservation during {phase}",
1871 target.pool_index, target.pending_physical_bytes
1872 )));
1873 }
1874 let required = required_by_target.entry(target.pool_index).or_default();
1875 *required = required.saturating_add(target.remaining_reserved_physical_bytes(reservation.temporary_copies));
1876 let inflight = inflight_by_target.entry(target.pool_index).or_default();
1877 *inflight = inflight.saturating_add(target.inflight_physical_bytes);
1878 let pending = pending_by_target.entry(target.pool_index).or_default();
1879 *pending = pending.saturating_add(target.pending_physical_bytes);
1880 }
1881 }
1882 for (target_pool_index, required) in required_by_target {
1883 let available = capacity_infos
1884 .iter()
1885 .find(|capacity| capacity.pool_index == target_pool_index)
1886 .map(|capacity| capacity.physical_free)
1887 .ok_or_else(|| {
1888 decommission_capacity_blocked_error(format!(
1889 "capacity snapshot is missing target pool {target_pool_index} during {phase}"
1890 ))
1891 })?
1892 .saturating_add(inflight_by_target.get(&target_pool_index).copied().unwrap_or_default())
1893 .saturating_add(pending_by_target.get(&target_pool_index).copied().unwrap_or_default());
1894 if available >= required {
1895 continue;
1896 }
1897 metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => phase).increment(1);
1898 return Err(decommission_capacity_blocked_error(format!(
1899 "target pool {target_pool_index} reservation requires {required} physical bytes, but only {available} bytes remain during {phase}"
1900 )));
1901 }
1902 Ok(())
1903}
1904
1905#[derive(Clone, Copy)]
1906pub(crate) enum DecommissionCapacityAdmission {
1907 Mutation,
1908 ExistingMultipart,
1909 ScannerBacklog,
1910 BatchDelete,
1911 Heal,
1912}
1913
1914impl DecommissionCapacityAdmission {
1915 fn phase(self) -> &'static str {
1916 match self {
1917 Self::Mutation => "mutation",
1918 Self::ExistingMultipart => "existing_multipart",
1919 Self::ScannerBacklog => "scanner_backlog",
1920 Self::BatchDelete => "batch_delete",
1921 Self::Heal => "heal",
1922 }
1923 }
1924}
1925
1926fn ensure_external_decommission_target_admission(
1927 meta: &PoolMeta,
1928 target_pool_index: usize,
1929 admission: DecommissionCapacityAdmission,
1930) -> Result<()> {
1931 let phase = admission.phase();
1932 if matches!(admission, DecommissionCapacityAdmission::ScannerBacklog)
1936 && !meta.scanner_pause_backlog_pool_writable(target_pool_index)
1937 {
1938 return Err(Error::SlowDown);
1939 }
1940 let active_sources = active_decommission_source_indices(meta);
1941 if meta.is_suspended(target_pool_index) {
1942 let active_source = active_sources.contains(&target_pool_index);
1943 if !matches!(
1944 admission,
1945 DecommissionCapacityAdmission::Heal | DecommissionCapacityAdmission::ScannerBacklog
1946 ) && !(matches!(admission, DecommissionCapacityAdmission::ExistingMultipart) && active_source)
1947 {
1948 return Err(Error::SlowDown);
1949 }
1950 }
1951 if active_sources.into_iter().any(|source_pool_index| {
1952 meta.pools
1953 .get(source_pool_index)
1954 .and_then(|pool| pool.decommission.as_ref())
1955 .and_then(|info| info.capacity_reservation.as_ref())
1956 .is_none_or(|reservation| !reservation.active())
1957 }) {
1958 metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => phase).increment(1);
1959 return Err(Error::SlowDown);
1960 }
1961 if !matches!(admission, DecommissionCapacityAdmission::Heal) {
1966 return Ok(());
1967 }
1968 let reserved = active_decommission_target_reservations(meta)
1969 .get(&target_pool_index)
1970 .copied()
1971 .unwrap_or_default();
1972 if reserved == 0 {
1973 return Ok(());
1974 }
1975 metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => phase).increment(1);
1976 Err(Error::SlowDown)
1977}
1978
1979fn grow_decommission_target_reservation(
1980 meta: &mut PoolMeta,
1981 capacity_infos: &[DecommissionPoolCapacityInfo],
1982 source_pool_index: usize,
1983 target_pool_index: usize,
1984 expected_data_bytes: usize,
1985 mutation_id: uuid::Uuid,
1986 now: OffsetDateTime,
1987) -> Result<bool> {
1988 let original = meta.clone();
1989 let result = (|| {
1990 let target_capacity = capacity_infos
1991 .iter()
1992 .find(|capacity| capacity.pool_index == target_pool_index)
1993 .ok_or_else(|| {
1994 decommission_capacity_blocked_error("target capacity snapshot is missing during reservation growth")
1995 })?;
1996 let pool_count = meta.pools.len();
1997 let pool = meta
1998 .pools
1999 .get_mut(source_pool_index)
2000 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2001 let reservation = pool
2002 .decommission
2003 .as_mut()
2004 .and_then(|info| info.capacity_reservation.as_mut())
2005 .filter(|reservation| reservation.active())
2006 .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during reservation growth"))?;
2007 let target_layout = reservation
2008 .targets
2009 .iter()
2010 .find(|target| target.pool_index == target_pool_index)
2011 .map(|target| target.layout)
2012 .unwrap_or(target_capacity.layout);
2013 let data_bytes = expected_data_bytes.max(1);
2014 let required_target_bytes = capacity_target_physical_bytes(data_bytes, target_layout)?;
2015 let required_peak = required_target_bytes.saturating_mul(1usize.saturating_add(reservation.temporary_copies));
2016 let existing_target = reservation
2017 .targets
2018 .iter()
2019 .find(|target| target.pool_index == target_pool_index);
2020 if let Some(target) = existing_target
2021 && target.pending_physical_bytes > 0
2022 && target.pending_mutation_id != Some(mutation_id)
2023 {
2024 return Err(decommission_capacity_blocked_error(format!(
2025 "source pool {source_pool_index} target pool {target_pool_index} has an unresolved target capacity intent"
2026 )));
2027 }
2028 let existing_remaining = existing_target
2029 .map(|target| target.remaining_reserved_physical_bytes(reservation.temporary_copies))
2030 .unwrap_or_default();
2031 if required_peak <= existing_remaining {
2032 return Ok(false);
2033 }
2034 if reservation.source_data_equivalent_bytes > 0 {
2035 return Err(decommission_capacity_blocked_error(format!(
2036 "source pool {source_pool_index} reservation estimate is exhausted"
2037 )));
2038 }
2039 if let Some(target) = existing_target
2040 && target_capacity.physical_free <= target.physical_free_at_reservation
2041 {
2042 return Err(decommission_capacity_blocked_error(format!(
2043 "source pool {source_pool_index} target pool {target_pool_index} reservation is exhausted"
2044 )));
2045 }
2046 let current_peak = reservation.remaining_peak_physical_bytes();
2047 let additional_peak = required_peak.saturating_sub(existing_remaining);
2048 let additional_predicted = additional_peak.div_ceil(1usize.saturating_add(reservation.temporary_copies));
2049 let additional_source_data = capacity_source_data_equivalent(additional_predicted, target_layout)?;
2050 reservation.source_data_equivalent_bytes = reservation
2051 .source_data_equivalent_bytes
2052 .saturating_add(additional_source_data);
2053 reservation.source_physical_bytes = reservation
2054 .source_physical_bytes
2055 .saturating_add(capacity_target_physical_bytes(additional_source_data, reservation.source_layout)?);
2056 reservation.predicted_physical_bytes = reservation
2057 .predicted_physical_bytes
2058 .saturating_add(additional_predicted)
2059 .max(capacity_target_physical_bytes(reservation.source_data_equivalent_bytes, target_layout)?);
2060 reservation.temporary_physical_bytes = reservation
2061 .predicted_physical_bytes
2062 .saturating_mul(reservation.temporary_copies);
2063 reservation.peak_physical_bytes = reservation
2064 .predicted_physical_bytes
2065 .saturating_add(reservation.temporary_physical_bytes);
2066 let allocation_growth = reservation.remaining_peak_physical_bytes().saturating_sub(current_peak);
2067 if let Some(target) = reservation
2068 .targets
2069 .iter_mut()
2070 .find(|target| target.pool_index == target_pool_index)
2071 {
2072 target.reserved_physical_bytes = target.reserved_physical_bytes.saturating_add(allocation_growth);
2073 target.physical_total_at_reservation = target.physical_total_at_reservation.max(target_capacity.physical_total);
2074 target.physical_free_at_reservation = target.physical_free_at_reservation.max(target_capacity.physical_free);
2075 } else {
2076 reservation.targets.push(DecommissionCapacityTarget {
2077 pool_index: target_pool_index,
2078 layout: target_layout,
2079 physical_total_at_reservation: target_capacity.physical_total,
2080 physical_free_at_reservation: target_capacity.physical_free,
2081 reserved_physical_bytes: allocation_growth,
2082 consumed_physical_bytes: 0,
2083 observed_physical_bytes: 0,
2084 inflight_physical_bytes: 0,
2085 pending_physical_bytes: 0,
2086 pending_mutation_id: None,
2087 temporary_mutations: Vec::new(),
2088 });
2089 }
2090 renew_decommission_capacity_reservation(reservation, now, true);
2091 pool.last_update = pool.last_update.max(now);
2092 Ok(true)
2093 })();
2094 if result.is_ok() {
2095 if let Err(err) = ensure_decommission_capacity_reservations_available(meta, capacity_infos, "reservation growth") {
2096 *meta = original;
2097 return Err(err);
2098 }
2099 } else {
2100 *meta = original;
2101 }
2102 result
2103}
2104
2105fn ensure_decommission_target_owner_admission(
2106 meta: &PoolMeta,
2107 owner: DecommissionCapacityOwner,
2108 target_pool_index: usize,
2109 target_physical_bytes: usize,
2110 now: OffsetDateTime,
2111) -> Result<()> {
2112 let reservation = meta
2113 .pools
2114 .get(owner.source_pool_index)
2115 .and_then(|pool| pool.decommission.as_ref())
2116 .and_then(|info| info.capacity_reservation.as_ref())
2117 .filter(|reservation| reservation.admits_owner(owner, now))
2118 .ok_or_else(|| Error::SlowDown)?;
2119 let target = reservation
2120 .targets
2121 .iter()
2122 .find(|target| target.pool_index == target_pool_index)
2123 .ok_or_else(|| Error::SlowDown)?;
2124 let required_peak = target_physical_bytes.saturating_mul(1usize.saturating_add(reservation.temporary_copies));
2125 let remaining = target.remaining_reserved_physical_bytes(reservation.temporary_copies);
2126 if required_peak <= remaining {
2127 return Ok(());
2128 }
2129 Err(decommission_capacity_blocked_error(format!(
2130 "source pool {} target pool {target_pool_index} operation requires {required_peak} physical bytes, but only {remaining} reserved bytes remain",
2131 owner.source_pool_index
2132 )))
2133}
2134
2135fn reserve_decommission_target_pending(
2136 meta: &mut PoolMeta,
2137 source_pool_index: usize,
2138 target_pool_index: usize,
2139 requested_physical_bytes: usize,
2140 mutation_id: uuid::Uuid,
2141 now: OffsetDateTime,
2142) -> Result<usize> {
2143 if requested_physical_bytes == 0 {
2144 return Ok(0);
2145 }
2146 let pool_count = meta.pools.len();
2147 let pool = meta
2148 .pools
2149 .get_mut(source_pool_index)
2150 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2151 let info = pool
2152 .decommission
2153 .as_mut()
2154 .ok_or_else(|| decommission_metadata_not_initialized_error("reserve target capacity mutation"))?;
2155 let reservation = info
2156 .capacity_reservation
2157 .as_mut()
2158 .filter(|reservation| reservation.active())
2159 .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared before target write"))?;
2160 let target = reservation
2161 .targets
2162 .iter_mut()
2163 .find(|target| target.pool_index == target_pool_index)
2164 .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared before target write"))?;
2165 if target.pending_physical_bytes == 0 {
2166 target.pending_mutation_id = None;
2167 }
2168 if target.pending_physical_bytes > 0 {
2169 match target.pending_mutation_id {
2170 Some(pending_mutation_id) if pending_mutation_id == mutation_id => {
2171 if requested_physical_bytes <= target.pending_physical_bytes {
2172 return Ok(0);
2173 }
2174 let budget = target.remaining_reserved_physical_bytes(reservation.temporary_copies)
2175 / 1usize.saturating_add(reservation.temporary_copies);
2176 if requested_physical_bytes > budget {
2177 return Err(decommission_capacity_blocked_error(format!(
2178 "source pool {source_pool_index} target pool {target_pool_index} mutation intent size changed during retry"
2179 )));
2180 }
2181 let additional = requested_physical_bytes.saturating_sub(target.pending_physical_bytes);
2182 target.pending_physical_bytes = requested_physical_bytes;
2183 reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_add(additional);
2184 renew_decommission_capacity_reservation(reservation, now, true);
2185 pool.last_update = pool.last_update.max(now);
2186 return Ok(additional);
2187 }
2188 pending_mutation_id => {
2189 return Err(decommission_capacity_blocked_error(format!(
2190 "source pool {source_pool_index} target pool {target_pool_index} has an unresolved target capacity intent {pending_mutation_id:?} while mutation {mutation_id} is waiting"
2191 )));
2192 }
2193 }
2194 }
2195 let budget = target.remaining_reserved_physical_bytes(reservation.temporary_copies)
2196 / 1usize.saturating_add(reservation.temporary_copies);
2197 if requested_physical_bytes > budget {
2198 return Err(decommission_capacity_blocked_error(format!(
2199 "source pool {source_pool_index} target pool {target_pool_index} mutation requires {requested_physical_bytes} physical bytes, but only {budget} committed-copy bytes remain"
2200 )));
2201 }
2202 let additional = requested_physical_bytes;
2203 target.pending_mutation_id = Some(mutation_id);
2204 target.pending_physical_bytes = target.pending_physical_bytes.saturating_add(additional);
2205 reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_add(additional);
2206 renew_decommission_capacity_reservation(reservation, now, true);
2207 pool.last_update = pool.last_update.max(now);
2208 Ok(additional)
2209}
2210
2211fn resolve_decommission_target_pending(
2212 meta: &mut PoolMeta,
2213 source_pool_index: usize,
2214 target_pool_index: usize,
2215 resolved_physical_bytes: usize,
2216 mutation_id: uuid::Uuid,
2217) -> Result<()> {
2218 if resolved_physical_bytes == 0 {
2219 return Ok(());
2220 }
2221 let pool_count = meta.pools.len();
2222 let pool = meta
2223 .pools
2224 .get_mut(source_pool_index)
2225 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2226 let reservation = pool
2227 .decommission
2228 .as_mut()
2229 .and_then(|info| info.capacity_reservation.as_mut())
2230 .filter(|reservation| reservation.active())
2231 .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared after target write"))?;
2232 let target = reservation
2233 .targets
2234 .iter_mut()
2235 .find(|target| target.pool_index == target_pool_index)
2236 .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared after target write"))?;
2237 if target.pending_physical_bytes > 0 && target.pending_mutation_id != Some(mutation_id) {
2238 return Err(decommission_capacity_blocked_error(format!(
2239 "source pool {source_pool_index} target pool {target_pool_index} pending capacity intent belongs to another mutation"
2240 )));
2241 }
2242 let resolved = target.pending_physical_bytes.min(resolved_physical_bytes);
2243 target.pending_physical_bytes = target.pending_physical_bytes.saturating_sub(resolved);
2244 reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_sub(resolved);
2245 if target.pending_physical_bytes == 0 {
2246 target.pending_mutation_id = None;
2247 }
2248 Ok(())
2249}
2250
2251fn release_decommission_target_temporary_mutation(
2252 target: &mut DecommissionCapacityTarget,
2253 mutation_id: uuid::Uuid,
2254 maximum_physical_bytes: usize,
2255) -> (usize, bool) {
2256 let Some(index) = target
2257 .temporary_mutations
2258 .iter()
2259 .position(|mutation| mutation.mutation_id == mutation_id)
2260 else {
2261 return (0, false);
2262 };
2263 let released = target.temporary_mutations[index].physical_bytes.min(maximum_physical_bytes);
2264 target.temporary_mutations[index].physical_bytes = target.temporary_mutations[index].physical_bytes.saturating_sub(released);
2265 let removed = target.temporary_mutations[index].physical_bytes == 0;
2266 if removed {
2267 target.temporary_mutations.remove(index);
2268 }
2269 target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released);
2270 (released, released > 0 || removed)
2271}
2272
2273fn settle_decommission_target_non_growing_replacement(
2274 meta: &mut PoolMeta,
2275 source_pool_index: usize,
2276 target_pool_index: usize,
2277 mutation_id: uuid::Uuid,
2278 now: OffsetDateTime,
2279) -> Result<bool> {
2280 let pool_count = meta.pools.len();
2281 let pool = meta
2282 .pools
2283 .get_mut(source_pool_index)
2284 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2285 let info = pool
2286 .decommission
2287 .as_mut()
2288 .ok_or_else(|| decommission_metadata_not_initialized_error("settle non-growing target replacement"))?;
2289 let reservation = info
2290 .capacity_reservation
2291 .as_mut()
2292 .filter(|reservation| reservation.active())
2293 .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target replacement"))?;
2294 let target = reservation
2295 .targets
2296 .iter_mut()
2297 .find(|target| target.pool_index == target_pool_index)
2298 .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target replacement"))?;
2299 let (released, changed) = release_decommission_target_temporary_mutation(target, mutation_id, usize::MAX);
2300 reservation.inflight_target_physical_bytes = reservation.inflight_target_physical_bytes.saturating_sub(released);
2301 if changed {
2302 renew_decommission_capacity_reservation(reservation, now, true);
2303 pool.last_update = pool.last_update.max(now);
2304 }
2305 Ok(changed)
2306}
2307
2308struct DecommissionTargetConsumption {
2309 committed_data_bytes: usize,
2310 target_physical_bytes: usize,
2311 observed_physical_bytes: usize,
2312}
2313
2314fn record_decommission_target_consumption(
2315 meta: &mut PoolMeta,
2316 source_pool_index: usize,
2317 target_pool_index: usize,
2318 consumption: DecommissionTargetConsumption,
2319 mutation_id: uuid::Uuid,
2320 now: OffsetDateTime,
2321) -> Result<()> {
2322 let DecommissionTargetConsumption {
2323 committed_data_bytes,
2324 target_physical_bytes,
2325 observed_physical_bytes,
2326 } = consumption;
2327 let pool_count = meta.pools.len();
2328 let pool = meta
2329 .pools
2330 .get_mut(source_pool_index)
2331 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2332 let info = pool
2333 .decommission
2334 .as_mut()
2335 .ok_or_else(|| decommission_metadata_not_initialized_error("record target capacity consumption"))?;
2336 let reservation = info
2337 .capacity_reservation
2338 .as_mut()
2339 .filter(|reservation| reservation.active())
2340 .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target write"))?;
2341 let target = reservation
2342 .targets
2343 .iter_mut()
2344 .find(|target| target.pool_index == target_pool_index)
2345 .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target write"))?;
2346 let remaining_target_bytes = target.remaining_reserved_physical_bytes(reservation.temporary_copies)
2347 / 1usize.saturating_add(reservation.temporary_copies);
2348 let remaining_total_bytes = reservation
2349 .predicted_physical_bytes
2350 .saturating_sub(reservation.consumed_target_physical_bytes);
2351 let consumed = target_physical_bytes.min(remaining_target_bytes).min(remaining_total_bytes);
2352 target.consumed_physical_bytes = target.consumed_physical_bytes.saturating_add(consumed);
2353 target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes);
2354 let has_scoped_temporary_mutations = !target.temporary_mutations.is_empty();
2355 let (released_inflight, _) = release_decommission_target_temporary_mutation(target, mutation_id, usize::MAX);
2356 let released_inflight = if released_inflight == 0 && !has_scoped_temporary_mutations {
2357 let released = target.inflight_physical_bytes.min(consumed);
2358 target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released);
2359 released
2360 } else {
2361 released_inflight
2362 };
2363 reservation.consumed_target_physical_bytes = reservation.consumed_target_physical_bytes.saturating_add(consumed);
2364 let committed = committed_data_bytes.min(
2365 reservation
2366 .source_data_equivalent_bytes
2367 .saturating_sub(reservation.committed_data_bytes),
2368 );
2369 reservation.committed_data_bytes = reservation.committed_data_bytes.saturating_add(committed);
2370 reservation.observed_target_physical_bytes = reservation
2371 .observed_target_physical_bytes
2372 .saturating_add(observed_physical_bytes);
2373 reservation.inflight_target_physical_bytes = reservation.inflight_target_physical_bytes.saturating_sub(released_inflight);
2374 reservation.prediction_error_bytes =
2375 signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes);
2376 renew_decommission_capacity_reservation(reservation, now, true);
2377 info.capacity_blocked_reason = None;
2378 pool.last_update = pool.last_update.max(now);
2379 Ok(())
2380}
2381
2382fn record_decommission_target_inflight(
2383 meta: &mut PoolMeta,
2384 source_pool_index: usize,
2385 target_pool_index: usize,
2386 observed_physical_bytes: usize,
2387 mutation_id: uuid::Uuid,
2388 now: OffsetDateTime,
2389) -> Result<bool> {
2390 let pool_count = meta.pools.len();
2391 let pool = meta
2392 .pools
2393 .get_mut(source_pool_index)
2394 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2395 let info = pool
2396 .decommission
2397 .as_mut()
2398 .ok_or_else(|| decommission_metadata_not_initialized_error("record target capacity inflight bytes"))?;
2399 let reservation = info
2400 .capacity_reservation
2401 .as_mut()
2402 .filter(|reservation| reservation.active())
2403 .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target write"))?;
2404 let target = reservation
2405 .targets
2406 .iter_mut()
2407 .find(|target| target.pool_index == target_pool_index)
2408 .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target write"))?;
2409 let ledger_changed = if let Some(mutation) = target
2410 .temporary_mutations
2411 .iter_mut()
2412 .find(|mutation| mutation.mutation_id == mutation_id)
2413 {
2414 mutation.physical_bytes = mutation.physical_bytes.saturating_add(observed_physical_bytes);
2415 observed_physical_bytes > 0
2416 } else if observed_physical_bytes > 0 || reservation.model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION {
2417 target.temporary_mutations.push(DecommissionCapacityTemporaryMutation {
2421 mutation_id,
2422 physical_bytes: observed_physical_bytes,
2423 });
2424 true
2425 } else {
2426 false
2427 };
2428 target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes);
2429 target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_add(observed_physical_bytes);
2430 reservation.observed_target_physical_bytes = reservation
2431 .observed_target_physical_bytes
2432 .saturating_add(observed_physical_bytes);
2433 reservation.inflight_target_physical_bytes = reservation
2434 .inflight_target_physical_bytes
2435 .saturating_add(observed_physical_bytes);
2436 reservation.prediction_error_bytes =
2437 signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes);
2438 renew_decommission_capacity_reservation(reservation, now, true);
2439 pool.last_update = pool.last_update.max(now);
2440 Ok(ledger_changed)
2441}
2442
2443fn record_decommission_target_observation(
2444 meta: &mut PoolMeta,
2445 source_pool_index: usize,
2446 target_pool_index: usize,
2447 observed_physical_bytes: usize,
2448 now: OffsetDateTime,
2449) -> Result<()> {
2450 let pool_count = meta.pools.len();
2451 let pool = meta
2452 .pools
2453 .get_mut(source_pool_index)
2454 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2455 let info = pool
2456 .decommission
2457 .as_mut()
2458 .ok_or_else(|| decommission_metadata_not_initialized_error("record target capacity observation"))?;
2459 let reservation = info
2460 .capacity_reservation
2461 .as_mut()
2462 .filter(|reservation| reservation.active())
2463 .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target write"))?;
2464 let target = reservation
2465 .targets
2466 .iter_mut()
2467 .find(|target| target.pool_index == target_pool_index)
2468 .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target write"))?;
2469 target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes);
2470 reservation.observed_target_physical_bytes = reservation
2471 .observed_target_physical_bytes
2472 .saturating_add(observed_physical_bytes);
2473 reservation.prediction_error_bytes =
2474 signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes);
2475 renew_decommission_capacity_reservation(reservation, now, true);
2476 pool.last_update = pool.last_update.max(now);
2477 Ok(())
2478}
2479
2480fn release_decommission_target_inflight(
2481 meta: &mut PoolMeta,
2482 source_pool_index: usize,
2483 target_pool_index: usize,
2484 released_physical_bytes: usize,
2485 mutation_id: uuid::Uuid,
2486 proof: DecommissionCapacityReleaseProof,
2487 now: OffsetDateTime,
2488) -> Result<bool> {
2489 let DecommissionCapacityReleaseProof {
2490 confirmed_absent,
2491 clear_pending,
2492 } = proof;
2493 let pool_count = meta.pools.len();
2494 let pool = meta
2495 .pools
2496 .get_mut(source_pool_index)
2497 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2498 let info = pool
2499 .decommission
2500 .as_mut()
2501 .ok_or_else(|| decommission_metadata_not_initialized_error("release target capacity inflight bytes"))?;
2502 let reservation = info
2503 .capacity_reservation
2504 .as_mut()
2505 .filter(|reservation| reservation.active())
2506 .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target cleanup"))?;
2507 let model_version = reservation.model_version;
2508 let target = reservation
2509 .targets
2510 .iter_mut()
2511 .find(|target| target.pool_index == target_pool_index)
2512 .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target cleanup"))?;
2513 let has_temporary_mutations = !target.temporary_mutations.is_empty();
2514 let pending_belongs_to_mutation = target.pending_mutation_id == Some(mutation_id);
2515 let zero_ledger_cleanup = confirmed_absent
2516 && model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
2517 && !has_temporary_mutations
2518 && target.pending_mutation_id.is_none()
2519 && target.pending_physical_bytes == 0;
2520 let (released, temporary_mutation_changed) = release_decommission_target_temporary_mutation(
2521 target,
2522 mutation_id,
2523 if confirmed_absent {
2524 usize::MAX
2525 } else {
2526 released_physical_bytes
2527 },
2528 );
2529 let released = if released == 0 && !has_temporary_mutations && model_version == DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION {
2530 let released = target.inflight_physical_bytes.min(released_physical_bytes);
2531 target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released);
2532 released
2533 } else {
2534 released
2535 };
2536 let pending_matches = clear_pending && confirmed_absent && pending_belongs_to_mutation;
2537 let published_pending_delayed_release = confirmed_absent
2538 && model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
2539 && !clear_pending
2540 && !has_temporary_mutations
2541 && pending_belongs_to_mutation;
2542 if released == 0
2543 && released_physical_bytes > 0
2544 && !zero_ledger_cleanup
2545 && !published_pending_delayed_release
2546 && (has_temporary_mutations || !pending_matches)
2547 {
2548 return Err(decommission_capacity_blocked_error(
2549 "temporary target cleanup released bytes that cannot be attributed to its mutation",
2550 ));
2551 }
2552 reservation.inflight_target_physical_bytes = reservation.inflight_target_physical_bytes.saturating_sub(released);
2553 let cleared_pending = if pending_matches {
2554 let cleared = target.pending_physical_bytes;
2555 target.pending_physical_bytes = 0;
2556 target.pending_mutation_id = None;
2557 reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_sub(cleared);
2558 cleared
2559 } else {
2560 0
2561 };
2562 let changed = released > 0 || temporary_mutation_changed || cleared_pending > 0;
2563 if changed {
2564 renew_decommission_capacity_reservation(reservation, now, true);
2565 pool.last_update = pool.last_update.max(now);
2566 }
2567 Ok(changed)
2568}
2569
2570fn ensure_valid_decommission_pool_index(pool_count: usize, idx: usize) -> Result<()> {
2571 if idx >= pool_count {
2572 return Err(invalid_decommission_pool_index_error(pool_count, idx));
2573 }
2574
2575 Ok(())
2576}
2577
2578fn get_by_index<'a, T>(items: &'a [T], idx: usize, operation: &'static str) -> Result<&'a T> {
2579 items.get(idx).ok_or_else(|| {
2580 Error::other(format!(
2581 "failed to {operation}: invalid decommission pool index {idx} for {pool_count} pools",
2582 pool_count = items.len()
2583 ))
2584 })
2585}
2586
2587fn decommission_metadata_not_initialized_error(operation: &str) -> Error {
2588 Error::other(format!("failed to {operation}: decommission metadata not initialized"))
2589}
2590
2591fn resolve_decommission_bucket_state(meta: &PoolMeta, idx: usize, bucket: &DecomBucketInfo) -> Result<bool> {
2592 let pool_count = meta.pools.len();
2593 ensure_valid_decommission_pool_index(pool_count, idx)?;
2594
2595 let Some(pool) = meta.pools.get(idx) else {
2596 return Err(invalid_decommission_pool_index_error(pool_count, idx));
2597 };
2598 let Some(info) = pool.decommission.as_ref() else {
2599 return Err(decommission_metadata_not_initialized_error("resolve decommission bucket state"));
2600 };
2601
2602 Ok(info.is_bucket_decommissioned(&bucket.to_string()))
2603}
2604
2605fn mark_decommission_bucket_done(meta: &mut PoolMeta, idx: usize, bucket: &DecomBucketInfo) -> Result<bool> {
2606 let pool_count = meta.pools.len();
2607 ensure_valid_decommission_pool_index(pool_count, idx)?;
2608
2609 let Some(pool) = meta.pools.get_mut(idx) else {
2610 return Err(invalid_decommission_pool_index_error(pool_count, idx));
2611 };
2612 let Some(info) = pool.decommission.as_mut() else {
2613 return Err(decommission_metadata_not_initialized_error("mark decommission bucket done"));
2614 };
2615
2616 Ok(info.bucket_pop(&bucket.to_string()))
2617}
2618
2619fn count_decommission_item(meta: &mut PoolMeta, idx: usize, size: usize, failed: bool) -> Result<()> {
2620 let pool_count = meta.pools.len();
2621 ensure_valid_decommission_pool_index(pool_count, idx)?;
2622
2623 let Some(pool) = meta.pools.get_mut(idx) else {
2624 return Err(invalid_decommission_pool_index_error(pool_count, idx));
2625 };
2626 let Some(info) = pool.decommission.as_mut() else {
2627 return Err(decommission_metadata_not_initialized_error("count decommission item"));
2628 };
2629
2630 if failed {
2631 info.items_decommission_failed += 1;
2632 info.bytes_failed += size;
2633 } else {
2634 info.items_decommissioned += 1;
2635 info.bytes_done += size;
2636 }
2637
2638 Ok(())
2639}
2640
2641fn ensure_decommission_generation(meta: &PoolMeta, idx: usize, generation: OffsetDateTime) -> Result<()> {
2642 let Some(pool) = meta.pools.get(idx) else {
2643 return Err(invalid_decommission_pool_index_error(meta.pools.len(), idx));
2644 };
2645 let Some(info) = pool.decommission.as_ref() else {
2646 return Err(decommission_metadata_not_initialized_error("check decommission generation"));
2647 };
2648
2649 if info.start_time == Some(generation) && !info.queued && is_decommission_active(info.complete, info.failed, info.canceled) {
2650 Ok(())
2651 } else {
2652 Err(Error::OperationCanceled)
2653 }
2654}
2655
2656fn record_decommission_unresolved_entry(
2657 meta: &mut PoolMeta,
2658 idx: usize,
2659 generation: OffsetDateTime,
2660 entry: DecommissionUnresolvedEntry,
2661 now: OffsetDateTime,
2662 rebalance_meta: Option<&RebalanceMeta>,
2663) -> Result<bool> {
2664 ensure_decommission_generation(meta, idx, generation)?;
2665 if entry.pool_index != idx || entry.source_generation != generation {
2666 return Err(Error::other("decommission unresolved entry does not match the active pool generation"));
2667 }
2668
2669 let pool_count = meta.pools.len();
2670 let Some(pool) = meta.pools.get(idx) else {
2671 return Err(invalid_decommission_pool_index_error(pool_count, idx));
2672 };
2673 let Some(info) = pool.decommission.as_ref() else {
2674 return Err(decommission_metadata_not_initialized_error("record decommission unresolved entry"));
2675 };
2676 let existing_index = info.unresolved_entries.iter().position(|existing| {
2677 existing.bucket == entry.bucket
2678 && existing.object == entry.object
2679 && existing.pool_index == entry.pool_index
2680 && existing.set_index == entry.set_index
2681 && existing.source_generation == entry.source_generation
2682 });
2683 if existing_index.is_some_and(|index| info.unresolved_entries[index] == entry) {
2684 return Ok(false);
2685 }
2686
2687 let last_update = meta.next_scanner_data_movement_update(now, rebalance_meta);
2688 let Some(pool) = meta.pools.get_mut(idx) else {
2689 return Err(invalid_decommission_pool_index_error(pool_count, idx));
2690 };
2691 let Some(info) = pool.decommission.as_mut() else {
2692 return Err(decommission_metadata_not_initialized_error("record decommission unresolved entry"));
2693 };
2694 if let Some(index) = existing_index {
2695 info.unresolved_entries[index] = entry;
2696 } else {
2697 info.unresolved_entries.push(entry);
2698 }
2699 pool.last_update = last_update;
2700 Ok(true)
2701}
2702
2703type DecommissionUnresolvedEntryIdentity = (usize, String, String);
2704
2705fn decommission_unresolved_entry_identity(entry: &DecommissionUnresolvedEntry) -> DecommissionUnresolvedEntryIdentity {
2706 (entry.set_index, entry.bucket.clone(), entry.object.clone())
2707}
2708
2709fn reconcile_decommission_unresolved_entries_for_completion(
2710 meta: &mut PoolMeta,
2711 idx: usize,
2712 verified_generation: Option<OffsetDateTime>,
2713 verified_entries: Option<&[DecommissionUnresolvedEntry]>,
2714) -> Result<()> {
2715 let pool_count = meta.pools.len();
2716 let Some(pool) = meta.pools.get(idx) else {
2717 return Err(invalid_decommission_pool_index_error(pool_count, idx));
2718 };
2719 let Some(info) = pool.decommission.as_ref() else {
2720 return Err(decommission_metadata_not_initialized_error("reconcile decommission unresolved entries"));
2721 };
2722 if info.unresolved_entries.is_empty() {
2723 return Ok(());
2724 }
2725
2726 let Some(generation) = verified_generation else {
2727 return Err(Error::other(format!(
2728 "failed to complete decommission for pool {idx}: {} unresolved listing entries remain",
2729 info.unresolved_entries.len()
2730 )));
2731 };
2732 ensure_decommission_generation(meta, idx, generation)?;
2733 if info
2734 .unresolved_entries
2735 .iter()
2736 .any(|entry| entry.source_generation != generation)
2737 {
2738 return Err(Error::other(format!(
2739 "failed to complete decommission for pool {idx}: unresolved listing ledger contains a different generation"
2740 )));
2741 }
2742
2743 let verified_entries = verified_entries.unwrap_or_default();
2744 let unverified_count = info
2745 .unresolved_entries
2746 .iter()
2747 .filter(|entry| !verified_entries.contains(entry))
2748 .count();
2749 if unverified_count > 0 {
2750 return Err(Error::other(format!(
2751 "failed to complete decommission for pool {idx}: {unverified_count} unresolved listing entries were not individually verified"
2752 )));
2753 }
2754
2755 let Some(info) = meta.pools.get_mut(idx).and_then(|pool| pool.decommission.as_mut()) else {
2756 return Err(decommission_metadata_not_initialized_error("reconcile decommission unresolved entries"));
2757 };
2758 info.unresolved_entries.clear();
2759 Ok(())
2760}
2761
2762#[cfg(test)]
2763async fn run_decommission_side_effect<T, E, F, Fut>(
2764 rx: &CancellationToken,
2765 operation_gate: &Arc<tokio::sync::RwLock<()>>,
2766 operation: F,
2767) -> std::result::Result<T, E>
2768where
2769 F: FnOnce() -> Fut,
2770 Fut: std::future::Future<Output = std::result::Result<T, E>>,
2771 E: From<Error>,
2772{
2773 let _operation_guard = tokio::select! {
2774 biased;
2775 _ = rx.cancelled() => return Err(Error::OperationCanceled.into()),
2776 guard = operation_gate.read() => guard,
2777 };
2778
2779 if rx.is_cancelled() {
2780 return Err(Error::OperationCanceled.into());
2781 }
2782
2783 let result = operation().await;
2784 if rx.is_cancelled() {
2785 return Err(Error::OperationCanceled.into());
2786 }
2787 result
2788}
2789
2790fn track_decommission_current_object_stage(
2791 meta: &mut PoolMeta,
2792 idx: usize,
2793 bucket: &str,
2794 object: &str,
2795 stage: &str,
2796) -> Result<()> {
2797 let pool_count = meta.pools.len();
2798 ensure_valid_decommission_pool_index(pool_count, idx)?;
2799
2800 let Some(pool) = meta.pools.get_mut(idx) else {
2801 return Err(invalid_decommission_pool_index_error(pool_count, idx));
2802 };
2803 let Some(info) = pool.decommission.as_mut() else {
2804 return Err(decommission_metadata_not_initialized_error("track decommission current object"));
2805 };
2806
2807 info.object = object.to_string();
2808 info.bucket = bucket.to_string();
2809 info.stage = stage.to_string();
2810 Ok(())
2811}
2812
2813fn track_decommission_current_object(meta: &mut PoolMeta, idx: usize, bucket: &str, object: &str) -> Result<()> {
2814 track_decommission_current_object_stage(meta, idx, bucket, object, "")
2815}
2816
2817fn resolve_decommission_update_after_result(result: Result<bool>) -> Result<bool> {
2818 result.map_err(|err| Error::other(format!("decommission metadata update failed: {err}")))
2819}
2820
2821fn resolve_decommission_progress_save_result(result: Result<()>) -> Option<Error> {
2822 result
2823 .err()
2824 .map(|err| Error::other(format!("decommission progress save failed: {err}")))
2825}
2826
2827fn resolve_decommission_preflight_heal_result<T>(bucket: &str, result: Result<T>) -> Result<T> {
2828 result.map_err(|err| Error::other(format!("decommission preflight heal failed for bucket {bucket}: {err}")))
2829}
2830
2831fn resolve_decommission_optional_bucket_config_result<T>(bucket: &str, stage: &str, result: Result<T>) -> Result<Option<T>> {
2832 match result {
2833 Ok(config) => Ok(Some(config)),
2834 Err(Error::ConfigNotFound) => Ok(None),
2835 Err(err) => Err(Error::other(format!(
2836 "decommission {stage} config load failed for bucket {bucket}: {err}"
2837 ))),
2838 }
2839}
2840
2841fn resolve_decommission_entry_cleanup_delete_result<T>(result: Result<T>, bucket: &str, object_name: &str) -> Result<()> {
2842 match result {
2843 Ok(_) => Ok(()),
2844 Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(()),
2845 Err(err) => Err(Error::other(format!(
2846 "decommission cleanup_delete_object failed for {bucket}/{object_name}: {err}"
2847 ))),
2848 }
2849}
2850
2851fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
2852 result.map_err(|err| Error::other(format!("decommission reload_pool_meta failed for {bucket}/{object_name}: {err}")))
2853}
2854
2855fn resolve_decommission_terminal_mark_result(result: Result<()>, stage: &str, pool_label: &str) -> Result<()> {
2856 result.map_err(|err| Error::other(format!("decommission terminal mark {stage} failed for pool {pool_label}: {err}")))
2857}
2858
2859fn resolve_decommission_terminal_mark_after_error_result(result: Result<()>, idx: usize, primary_err: &Error) -> Result<()> {
2860 result.map_err(|err| {
2861 Error::other(format!(
2862 "decommission terminal mark failed after background error on pool {idx}: {primary_err}; mark error: {err}"
2863 ))
2864 })
2865}
2866
2867fn observe_decommission_terminal_reload_result(result: Result<()>, stage: &str) -> Option<Error> {
2868 result
2869 .err()
2870 .map(|err| Error::other(format!("decommission terminal pool meta reload failed during {stage}: {err}")))
2871}
2872
2873fn decommission_item_size<T>(size: T) -> usize
2874where
2875 usize: TryFrom<T>,
2876{
2877 usize::try_from(size).unwrap_or_default()
2878}
2879
2880fn is_decommission_set_local_usage_cache(bucket: &str, object: &str) -> bool {
2881 if bucket != RUSTFS_META_BUCKET {
2882 return false;
2883 }
2884 let Some(path) = object
2885 .strip_prefix(BUCKET_META_PREFIX)
2886 .and_then(|path| path.strip_prefix('/'))
2887 else {
2888 return false;
2889 };
2890 let name = match path.rsplit_once('/') {
2891 Some((bucket, name)) if !bucket.is_empty() && !bucket.contains('/') && bucket != "." && bucket != ".." => name,
2892 Some(_) => return false,
2893 None => path,
2894 };
2895 name.strip_suffix(".bkp").unwrap_or(name) == DATA_USAGE_CACHE_NAME
2896}
2897
2898fn with_decommission_entry_context<E: Display>(stage: &str, bucket: &str, object: &str, err: E) -> Error {
2899 Error::other(format!("decommission entry {stage} failed for bucket {bucket} object {object}: {err}"))
2900}
2901
2902#[cfg(test)]
2903fn load_decommission_entry_versions(entry: &MetaCacheEntry, bucket: &str, stage: &str) -> Result<FileInfoVersions> {
2904 entry
2905 .file_info_versions(bucket)
2906 .map_err(|err| with_decommission_entry_context(stage, bucket, &entry.name, err))
2907}
2908
2909fn empty_decommission_entry_versions(bucket: &str, object: &str) -> FileInfoVersions {
2910 FileInfoVersions {
2911 volume: bucket.to_string(),
2912 name: object.to_string(),
2913 versions: Vec::new(),
2914 ..Default::default()
2915 }
2916}
2917
2918fn resolve_decommission_entry_exact_versions(
2919 result: Result<Option<FileInfoVersions>>,
2920 entry: &MetaCacheEntry,
2921 bucket: &str,
2922 stage: &str,
2923) -> Result<FileInfoVersions> {
2924 match result {
2925 Ok(Some(fivs)) => Ok(fivs),
2926 Ok(None) => Ok(empty_decommission_entry_versions(bucket, &entry.name)),
2927 Err(err) => Err(with_decommission_entry_context(stage, bucket, &entry.name, err)),
2928 }
2929}
2930
2931async fn load_decommission_entry_exact_versions(
2932 set: &SetDisks,
2933 entry: &MetaCacheEntry,
2934 bucket: &str,
2935 stage: &str,
2936) -> Result<FileInfoVersions> {
2937 resolve_decommission_entry_exact_versions(set.load_file_info_versions_exact(bucket, &entry.name).await, entry, bucket, stage)
2938}
2939
2940fn resolve_decommission_check_after_list_result(list_result: Result<()>, entry_error: Option<Error>) -> Result<()> {
2941 match list_result {
2942 Ok(()) => entry_error.map_or(Ok(()), Err),
2943 Err(list_err) => resolve_decommission_listing_error(Some(list_err), entry_error).map_or(Ok(()), Err),
2944 }
2945}
2946
2947fn resolve_decommission_listing_error(listing_error: Option<Error>, entry_error: Option<Error>) -> Option<Error> {
2948 match (listing_error, entry_error) {
2949 (Some(listing_error), Some(entry_error)) if is_err_operation_canceled(&listing_error) => Some(entry_error),
2950 (Some(listing_error), Some(entry_error)) if is_err_operation_canceled(&entry_error) => Some(listing_error),
2951 (Some(listing_error), _) => Some(listing_error),
2952 (None, entry_error) => entry_error,
2953 }
2954}
2955
2956fn decommission_unresolved_listing_error(entry: &DecommissionUnresolvedEntry) -> Error {
2957 Error::other(format!(
2958 "decommission listing could not resolve metadata for {bucket}/{object} on pool {pool_index} set {set_index} ({candidate_count} candidate(s), {disk_error_count} disk error(s))",
2959 bucket = entry.bucket,
2960 object = entry.object,
2961 pool_index = entry.pool_index,
2962 set_index = entry.set_index,
2963 candidate_count = entry.candidate_count,
2964 disk_error_count = entry.disk_error_count,
2965 ))
2966}
2967
2968#[allow(clippy::result_large_err)]
2971#[allow(clippy::too_many_arguments)]
2972fn resolve_decommission_partial_listing_entry(
2973 entries: MetaCacheEntries,
2974 resolver: MetadataResolutionParams,
2975 bucket: &str,
2976 prefix: &str,
2977 disk_error_count: usize,
2978 pool_index: usize,
2979 set_index: usize,
2980 source_generation: OffsetDateTime,
2981) -> std::result::Result<MetaCacheEntry, DecommissionUnresolvedEntry> {
2982 let candidate_count = entries.as_ref().iter().flatten().count();
2983 if let Some(entry) = entries.resolve(resolver) {
2984 return Ok(entry);
2985 }
2986
2987 let object = entries
2988 .as_ref()
2989 .iter()
2990 .flatten()
2991 .map(|entry| entry.name.as_str())
2992 .next()
2993 .unwrap_or(prefix)
2994 .to_string();
2995 Err(DecommissionUnresolvedEntry {
2996 bucket: bucket.to_string(),
2997 object,
2998 candidate_count,
2999 disk_error_count,
3000 pool_index,
3001 set_index,
3002 source_generation,
3003 observed_at: OffsetDateTime::now_utc(),
3004 reason: "metadata_resolution_failed".to_string(),
3005 })
3006}
3007
3008fn validate_decommission_durable_ilm_copy(
3009 path: &str,
3010 source_record: &ValidatedDurableIlmRecord,
3011 target: &[u8],
3012) -> Result<ValidatedDurableIlmRecord> {
3013 let target_record = validate_durable_ilm_record(path, target).map_err(|err| {
3014 Error::other(format!(
3015 "target durable ILM record is invalid at path `{path}` {}: {err}",
3016 source_record.context()
3017 ))
3018 })?;
3019 source_record
3020 .checkpoint
3021 .validate_successor(&target_record.checkpoint)
3022 .map_err(|err| {
3023 Error::other(format!(
3024 "target durable ILM record generation mismatch at path `{path}` {}: {err}",
3025 source_record.context()
3026 ))
3027 })?;
3028 Ok(target_record)
3029}
3030
3031#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3032#[serde(deny_unknown_fields)]
3033struct DecommissionDurableIlmReceipt {
3034 source_path: String,
3035 namespace: String,
3036 id_kind: String,
3037 id: String,
3038 checkpoint: DurableIlmRecordCheckpoint,
3039 terminal_checkpoint: Option<DurableIlmRecordCheckpoint>,
3040 #[serde(default, skip_serializing_if = "Option::is_none")]
3041 fleet_topology_generation: Option<String>,
3042}
3043
3044impl DecommissionDurableIlmReceipt {
3045 fn new(path: &str, record: &ValidatedDurableIlmRecord, fleet_topology_generation: Option<String>) -> Self {
3046 Self {
3047 source_path: path.to_string(),
3048 namespace: record.namespace.to_string(),
3049 id_kind: record.id_kind.to_string(),
3050 id: record.id.clone(),
3051 checkpoint: record.checkpoint.clone(),
3052 terminal_checkpoint: None,
3053 fleet_topology_generation,
3054 }
3055 }
3056
3057 fn context(&self) -> String {
3058 format!("namespace `{}` {} `{}`", self.namespace, self.id_kind, self.id)
3059 }
3060
3061 fn validate(&self) -> Result<()> {
3062 let namespace = classify_durable_ilm_record(&self.source_path)?
3063 .ok_or_else(|| Error::other(format!("receipt source path `{}` is not a durable ILM record", self.source_path)))?;
3064 if namespace.name != self.namespace {
3065 return Err(Error::other(format!(
3066 "receipt namespace `{}` does not match source path `{}`",
3067 self.namespace, self.source_path
3068 )));
3069 }
3070 if self.id_kind.is_empty() || self.id.is_empty() {
3071 return Err(Error::other(format!(
3072 "receipt identity is missing for source path `{}`",
3073 self.source_path
3074 )));
3075 }
3076 if !is_sha256_checksum(self.checkpoint.content_sha256()) {
3077 return Err(Error::other(format!(
3078 "receipt target checksum is invalid for source path `{}` {}",
3079 self.source_path,
3080 self.context()
3081 )));
3082 }
3083 if let Some(terminal_checkpoint) = &self.terminal_checkpoint {
3084 self.checkpoint.validate_successor(terminal_checkpoint).map_err(|err| {
3085 Error::other(format!(
3086 "receipt terminal checkpoint is invalid for source path `{}` {}: {err}",
3087 self.source_path,
3088 self.context()
3089 ))
3090 })?;
3091 }
3092 if self
3093 .fleet_topology_generation
3094 .as_deref()
3095 .is_some_and(|generation| !is_sha256_checksum(generation))
3096 {
3097 return Err(Error::other_with_context(
3098 "receipt fleet topology generation is invalid",
3099 format!("source path `{}` {}", self.source_path, self.context()),
3100 ));
3101 }
3102 Ok(())
3103 }
3104
3105 fn encode(&self) -> Result<Vec<u8>> {
3106 let mut receipt = self.clone();
3107 receipt.checkpoint = receipt.checkpoint.compacted()?;
3108 receipt.terminal_checkpoint = receipt
3109 .terminal_checkpoint
3110 .as_ref()
3111 .map(DurableIlmRecordCheckpoint::compacted)
3112 .transpose()?;
3113 receipt.validate()?;
3114 let receipt_bytes = serde_json::to_vec(&receipt)?;
3115 let persisted = PersistedDecommissionDurableIlmReceipt {
3116 schema: DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA.to_string(),
3117 content_sha256: hex_sha256(&receipt_bytes, ToOwned::to_owned),
3118 receipt,
3119 };
3120 let encoded = serde_json::to_vec(&persisted)?;
3121 if encoded.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE {
3122 return Err(Error::other(format!(
3123 "durable ILM receipt exceeds maximum size for source path `{}` {}",
3124 self.source_path,
3125 self.context()
3126 )));
3127 }
3128 Ok(encoded)
3129 }
3130
3131 fn decode(data: &[u8]) -> Result<Self> {
3132 if data.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE {
3133 return Err(Error::other("durable ILM receipt exceeds maximum size"));
3134 }
3135 let persisted: PersistedDecommissionDurableIlmReceipt = serde_json::from_slice(data)?;
3136 if persisted.schema != DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA {
3137 return Err(Error::other(format!("unsupported durable ILM receipt schema `{}`", persisted.schema)));
3138 }
3139 if !is_sha256_checksum(&persisted.content_sha256) {
3140 return Err(Error::other("durable ILM receipt checksum is invalid"));
3141 }
3142 let receipt_bytes = serde_json::to_vec(&persisted.receipt)?;
3143 let actual_checksum = hex_sha256(&receipt_bytes, ToOwned::to_owned);
3144 if persisted.content_sha256 != actual_checksum {
3145 return Err(Error::other("durable ILM receipt checksum mismatch"));
3146 }
3147 persisted.receipt.validate()?;
3148 Ok(persisted.receipt)
3149 }
3150}
3151
3152fn merge_decommission_durable_ilm_receipts(
3153 existing: &DecommissionDurableIlmReceipt,
3154 incoming: &DecommissionDurableIlmReceipt,
3155) -> Result<DecommissionDurableIlmReceipt> {
3156 if existing.source_path != incoming.source_path
3157 || existing.namespace != incoming.namespace
3158 || existing.id_kind != incoming.id_kind
3159 || existing.id != incoming.id
3160 {
3161 return Err(Error::other(format!(
3162 "durable ILM receipt identity conflict for source path `{}` {}; incoming {}",
3163 existing.source_path,
3164 existing.context(),
3165 incoming.context()
3166 )));
3167 }
3168
3169 let checkpoint =
3170 if existing.checkpoint == incoming.checkpoint || incoming.checkpoint.validate_successor(&existing.checkpoint).is_ok() {
3171 existing.checkpoint.clone()
3172 } else {
3173 existing.checkpoint.validate_successor(&incoming.checkpoint).map_err(|err| {
3174 Error::other(format!(
3175 "durable ILM receipt checkpoint conflict for source path `{}` {}: {err}",
3176 existing.source_path,
3177 existing.context()
3178 ))
3179 })?;
3180 incoming.checkpoint.clone()
3181 };
3182 let terminal_checkpoint = match (&existing.terminal_checkpoint, &incoming.terminal_checkpoint) {
3183 (Some(existing_terminal), Some(incoming_terminal)) if existing_terminal == incoming_terminal => {
3184 Some(existing_terminal.clone())
3185 }
3186 (Some(existing_terminal), Some(incoming_terminal)) if incoming_terminal.validate_successor(existing_terminal).is_ok() => {
3187 Some(existing_terminal.clone())
3188 }
3189 (Some(existing_terminal), Some(incoming_terminal)) => {
3190 existing_terminal.validate_successor(incoming_terminal).map_err(|err| {
3191 Error::other(format!(
3192 "durable ILM receipt terminal checkpoint conflict for source path `{}` {}: {err}",
3193 existing.source_path,
3194 existing.context()
3195 ))
3196 })?;
3197 Some(incoming_terminal.clone())
3198 }
3199 (Some(existing_terminal), None) => Some(existing_terminal.clone()),
3200 (None, Some(incoming_terminal)) => Some(incoming_terminal.clone()),
3201 (None, None) => None,
3202 };
3203 let merged = DecommissionDurableIlmReceipt {
3204 source_path: existing.source_path.clone(),
3205 namespace: existing.namespace.clone(),
3206 id_kind: existing.id_kind.clone(),
3207 id: existing.id.clone(),
3208 checkpoint,
3209 terminal_checkpoint,
3210 fleet_topology_generation: match (
3211 existing.fleet_topology_generation.as_ref(),
3212 incoming.fleet_topology_generation.as_ref(),
3213 ) {
3214 (Some(existing_generation), Some(incoming_generation)) if existing_generation == incoming_generation => {
3215 Some(existing_generation.clone())
3216 }
3217 (None, None) => None,
3218 _ => {
3219 return Err(Error::other_with_context(
3220 "durable ILM receipt fleet topology conflict",
3221 format!("source path `{}` {}", existing.source_path, existing.context()),
3222 ));
3223 }
3224 },
3225 };
3226 merged.validate()?;
3227 Ok(merged)
3228}
3229
3230#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3231#[serde(deny_unknown_fields)]
3232struct PersistedDecommissionDurableIlmReceipt {
3233 schema: String,
3234 content_sha256: String,
3235 receipt: DecommissionDurableIlmReceipt,
3236}
3237
3238#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3239#[serde(deny_unknown_fields)]
3240struct DecommissionDurableIlmManifest {
3241 schema: String,
3242 run_token: String,
3243 receipt_count: u64,
3244 receipt_paths_sha256: String,
3245}
3246
3247impl DecommissionDurableIlmManifest {
3248 fn new(run_token: &str, receipt_paths: &[String]) -> Result<Self> {
3249 let manifest = Self {
3250 schema: DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA.to_string(),
3251 run_token: run_token.to_string(),
3252 receipt_count: u64::try_from(receipt_paths.len())
3253 .map_err(|_| Error::other("durable ILM expected manifest receipt count exceeds u64"))?,
3254 receipt_paths_sha256: decommission_durable_ilm_manifest_paths_sha256(receipt_paths)?,
3255 };
3256 manifest.validate(run_token, receipt_paths)?;
3257 Ok(manifest)
3258 }
3259
3260 fn validate(&self, run_token: &str, receipt_paths: &[String]) -> Result<()> {
3261 if self.schema != DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA {
3262 return Err(Error::other(format!(
3263 "unsupported durable ILM expected manifest schema `{}`",
3264 self.schema
3265 )));
3266 }
3267 if self.run_token != run_token || !is_sha256_checksum(&self.run_token) {
3268 return Err(Error::other("durable ILM expected manifest run token is invalid"));
3269 }
3270 let receipt_count = u64::try_from(receipt_paths.len())
3271 .map_err(|_| Error::other("durable ILM expected manifest receipt count exceeds u64"))?;
3272 if self.receipt_count != receipt_count {
3273 return Err(Error::other(format!(
3274 "durable ILM expected manifest receipt count mismatch: expected {}, found {receipt_count}",
3275 self.receipt_count
3276 )));
3277 }
3278 if !is_sha256_checksum(&self.receipt_paths_sha256)
3279 || self.receipt_paths_sha256 != decommission_durable_ilm_manifest_paths_sha256(receipt_paths)?
3280 {
3281 return Err(Error::other("durable ILM expected manifest receipt paths checksum mismatch"));
3282 }
3283 Ok(())
3284 }
3285
3286 fn encode(&self) -> Result<Vec<u8>> {
3287 let encoded = serde_json::to_vec(self)?;
3288 if encoded.len() > DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE {
3289 return Err(Error::other("durable ILM expected manifest exceeds maximum size"));
3290 }
3291 Ok(encoded)
3292 }
3293
3294 fn decode(data: &[u8], run_token: &str, receipt_paths: &[String]) -> Result<Self> {
3295 if data.len() > DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE {
3296 return Err(Error::other("durable ILM expected manifest exceeds maximum size"));
3297 }
3298 let manifest: Self = serde_json::from_slice(data)?;
3299 manifest.validate(run_token, receipt_paths)?;
3300 Ok(manifest)
3301 }
3302}
3303
3304#[derive(Debug, Clone, PartialEq, Eq)]
3305struct DecommissionDurableIlmReceiptLocator {
3306 run_token: String,
3307 source_path: String,
3308 id_kind: String,
3309 id: String,
3310}
3311
3312impl DecommissionDurableIlmReceiptLocator {
3313 fn context(&self) -> String {
3314 format!("source path `{}` {} `{}`", self.source_path, self.id_kind, self.id)
3315 }
3316}
3317
3318fn decommission_durable_ilm_receipt_run_token(cmd_line: &str, start_time: OffsetDateTime) -> String {
3319 let identity = format!("{cmd_line}\0{}", start_time.unix_timestamp_nanos());
3320 hex_sha256(identity.as_bytes(), ToOwned::to_owned)
3321}
3322
3323fn decommission_durable_ilm_receipt_run_prefix(run_token: &str) -> String {
3324 format!("{DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT}/{run_token}/")
3325}
3326
3327fn decommission_durable_ilm_receipt_path(run_token: &str, source_path: &str, id_kind: &str, id: &str) -> String {
3328 format!(
3329 "{}{}/{}/{}.json",
3330 decommission_durable_ilm_receipt_run_prefix(run_token),
3331 source_path,
3332 id_kind,
3333 id
3334 )
3335}
3336
3337fn decommission_durable_ilm_manifest_path(run_token: &str) -> String {
3338 format!("{DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT}/{run_token}.json")
3339}
3340
3341fn decommission_durable_ilm_manifest_paths_sha256(receipt_paths: &[String]) -> Result<String> {
3342 let mut sorted_paths = receipt_paths.iter().map(String::as_str).collect::<Vec<_>>();
3343 sorted_paths.sort_unstable();
3344 let encoded = serde_json::to_vec(&sorted_paths)?;
3345 Ok(hex_sha256(&encoded, ToOwned::to_owned))
3346}
3347
3348fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result<DecommissionDurableIlmReceiptLocator> {
3349 let prefix = format!("{DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT}/");
3350 let suffix = path
3351 .strip_prefix(&prefix)
3352 .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has the wrong root")))?;
3353 let (run_token, record_path) = suffix
3354 .split_once('/')
3355 .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record path")))?;
3356 let mut parts = record_path.rsplitn(3, '/');
3357 let id = parts
3358 .next()
3359 .and_then(|file| file.strip_suffix(".json"))
3360 .filter(|id| !id.is_empty())
3361 .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record id")))?;
3362 let id_kind = parts
3363 .next()
3364 .filter(|id_kind| matches!(*id_kind, "operation_id" | "transaction_id" | "job_id" | "control_id"))
3365 .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has an invalid id kind")))?;
3366 let source_path = parts
3367 .next()
3368 .filter(|source_path| !source_path.is_empty())
3369 .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its source path")))?;
3370 if !is_sha256_checksum(run_token) {
3371 return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid run token")));
3372 }
3373 match id_kind {
3374 "operation_id" | "control_id" if !is_sha256_checksum(id) => {
3375 let id_label = id_kind.trim_end_matches("_id");
3376 return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid {id_label} id")));
3377 }
3378 "transaction_id" | "job_id" if uuid::Uuid::parse_str(id).is_err() => {
3379 return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid UUID")));
3380 }
3381 _ => {}
3382 }
3383 classify_durable_ilm_record(source_path)?
3384 .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` does not identify a durable ILM source path")))?;
3385 Ok(DecommissionDurableIlmReceiptLocator {
3386 run_token: run_token.to_string(),
3387 source_path: source_path.to_string(),
3388 id_kind: id_kind.to_string(),
3389 id: id.to_string(),
3390 })
3391}
3392
3393fn resolve_decommission_pool_meta_reload_result(result: Result<()>, stage: &str) -> Result<()> {
3394 result.map_err(|err| Error::other(format!("decommission pool meta reload failed during {stage}: {err}")))
3395}
3396
3397fn apply_decommission_status_space_info(mut pool_info: PoolStatus, space_info: PoolSpaceInfo) -> PoolStatus {
3398 match pool_info.decommission.as_mut() {
3399 Some(d) => {
3400 d.total_size = space_info.total;
3401 d.current_size = space_info.free;
3402 }
3403 None => {
3404 pool_info.decommission = Some(PoolDecommissionInfo {
3405 total_size: space_info.total,
3406 current_size: space_info.free,
3407 ..Default::default()
3408 });
3409 }
3410 }
3411
3412 pool_info
3413}
3414
3415fn should_replace_pool_status_for_status_refresh(
3416 current: Option<&PoolStatus>,
3417 persisted: &PoolStatus,
3418 has_active_worker: bool,
3419) -> bool {
3420 let Some(current) = current else {
3421 return true;
3422 };
3423
3424 !has_active_worker && persisted.last_update > current.last_update
3425}
3426
3427fn pool_decommission_movement_snapshot(
3428 info: Option<&PoolDecommissionInfo>,
3429) -> (bool, bool, bool, bool, bool, Option<OffsetDateTime>) {
3430 info.map(|info| {
3431 (
3432 info.has_decommission_state(),
3433 info.complete,
3434 info.failed,
3435 info.canceled,
3436 info.queued,
3437 info.start_time,
3438 )
3439 })
3440 .unwrap_or_default()
3441}
3442
3443pub(crate) fn pool_meta_movement_snapshot_changed(before: &PoolMeta, after: &PoolMeta) -> bool {
3444 before.pools.len() != after.pools.len()
3445 || before.pools.iter().zip(after.pools.iter()).any(|(before, after)| {
3446 pool_decommission_movement_snapshot(before.decommission.as_ref())
3447 != pool_decommission_movement_snapshot(after.decommission.as_ref())
3448 })
3449}
3450
3451pub(crate) fn merge_pool_status_refresh(current: &mut PoolMeta, persisted: PoolMeta, active_workers: &[bool]) -> bool {
3457 let observed_version = current.version.max(persisted.version);
3458 if persisted.pools.is_empty() {
3459 current.version = observed_version;
3460 return false;
3461 }
3462
3463 if current.pools.is_empty() {
3464 *current = persisted;
3465 current.version = observed_version;
3466 return true;
3467 }
3468
3469 current.version = observed_version;
3470
3471 let mut merged_newer = false;
3472 for (idx, persisted_pool) in persisted.pools.into_iter().enumerate() {
3473 if persisted_pool.id != idx {
3474 continue;
3475 }
3476
3477 let has_active_worker = active_workers.get(idx).copied().unwrap_or(false);
3478 if idx < current.pools.len() {
3479 if should_replace_pool_status_for_status_refresh(current.pools.get(idx), &persisted_pool, has_active_worker) {
3480 current.pools[idx] = persisted_pool;
3481 merged_newer = true;
3482 }
3483 } else if idx == current.pools.len() && !has_active_worker {
3484 current.pools.push(persisted_pool);
3485 merged_newer = true;
3486 }
3487 }
3488 merged_newer
3489}
3490
3491fn merge_pool_meta_updates_for_save(
3492 persisted: &mut PoolMeta,
3493 current: &PoolMeta,
3494 indices: &[usize],
3495 operation: &str,
3496) -> Result<()> {
3497 if persisted.pools.is_empty() {
3498 *persisted = current.clone();
3499 return Ok(());
3500 }
3501 if persisted.version != current.version {
3502 return Err(Error::other(format!(
3503 "{operation}: pool metadata version changed from {} to {}",
3504 current.version, persisted.version
3505 )));
3506 }
3507
3508 for &idx in indices {
3509 let current_pool = current
3510 .pools
3511 .get(idx)
3512 .ok_or_else(|| invalid_decommission_pool_index_error(current.pools.len(), idx))?;
3513 let persisted_count = persisted.pools.len();
3514 let persisted_pool = persisted
3515 .pools
3516 .get_mut(idx)
3517 .ok_or_else(|| invalid_decommission_pool_index_error(persisted_count, idx))?;
3518 if current_pool.id != idx || persisted_pool.id != idx || current_pool.cmd_line != persisted_pool.cmd_line {
3519 return Err(Error::other(format!("{operation}: pool metadata layout changed for pool {idx}")));
3520 }
3521 let current_clears_decommission = current_pool
3522 .decommission
3523 .as_ref()
3524 .is_none_or(|info| !info.has_decommission_state());
3525 if current_clears_decommission
3526 && persisted_pool.decommission.as_ref().is_some_and(|info| {
3527 info.has_decommission_state() && is_decommission_active(info.complete, info.failed, info.canceled)
3528 })
3529 {
3530 record_pool_meta_stale_write_rejection("nonterminal_decommission_clear");
3531 return Err(Error::other(format!(
3532 "{operation}: stale pool metadata update rejected for pool {idx}; persisted active or queued decommission cannot be cleared"
3533 )));
3534 }
3535 if current_clears_decommission
3536 && persisted_pool
3537 .decommission
3538 .as_ref()
3539 .is_some_and(|info| info.complete || !info.unresolved_entries.is_empty())
3540 {
3541 record_pool_meta_stale_write_rejection("unsafe_terminal_decommission_clear");
3542 return Err(Error::StalePoolMetadataUpdate {
3543 operation: operation.to_string(),
3544 pool_index: idx,
3545 reason: "completed or unresolved decommission state cannot be cleared",
3546 });
3547 }
3548 if current_pool.last_update < persisted_pool.last_update {
3549 record_pool_meta_stale_write_rejection("older_pool_revision");
3550 return Err(Error::other(format!(
3551 "{operation}: stale pool metadata update rejected for pool {idx}; persisted update is newer"
3552 )));
3553 }
3554 if let (Some(persisted_info), Some(current_info)) =
3555 (persisted_pool.decommission.as_ref(), current_pool.decommission.as_ref())
3556 && current_info.has_decommission_state()
3557 {
3558 let persisted_terminal = (persisted_info.complete, persisted_info.failed, persisted_info.canceled);
3559 let current_terminal = (current_info.complete, current_info.failed, current_info.canceled);
3560 if persisted_terminal != (false, false, false) && persisted_terminal != current_terminal {
3561 record_pool_meta_stale_write_rejection("terminal_state_regression");
3562 return Err(Error::other(format!(
3563 "{operation}: stale pool metadata update rejected for pool {idx}; terminal state cannot be replaced"
3564 )));
3565 }
3566 if current_info.items_decommissioned < persisted_info.items_decommissioned
3567 || current_info.items_decommission_failed < persisted_info.items_decommission_failed
3568 || current_info.bytes_done < persisted_info.bytes_done
3569 || current_info.bytes_failed < persisted_info.bytes_failed
3570 {
3571 record_pool_meta_stale_write_rejection("progress_regression");
3572 return Err(Error::other(format!(
3573 "{operation}: stale pool metadata update rejected for pool {idx}; durable progress cannot decrease"
3574 )));
3575 }
3576 }
3577 *persisted_pool = current_pool.clone();
3578 }
3579
3580 Ok(())
3581}
3582
3583fn publish_pool_meta_updates(current: &mut PoolMeta, saved: &PoolMeta, indices: &[usize]) {
3584 current.version = current.version.max(saved.version);
3585 for &idx in indices {
3586 let Some(saved_pool) = saved.pools.get(idx) else {
3587 continue;
3588 };
3589 let Some(current_pool) = current.pools.get_mut(idx) else {
3590 continue;
3591 };
3592 if current_pool.id == idx && saved_pool.id == idx && current_pool.cmd_line == saved_pool.cmd_line {
3593 *current_pool = saved_pool.clone();
3594 }
3595 }
3596}
3597
3598fn publish_decommission_capacity_update(current: &mut PoolMeta, saved: &PoolMeta, idx: usize) -> Result<()> {
3601 let stale = |reason| Error::StalePoolMetadataUpdate {
3602 operation: "publish decommission capacity".to_string(),
3603 pool_index: idx,
3604 reason,
3605 };
3606 let saved_pool = saved
3607 .pools
3608 .get(idx)
3609 .ok_or_else(|| invalid_decommission_pool_index_error(saved.pools.len(), idx))?;
3610 let current_count = current.pools.len();
3611 let pool = current
3612 .pools
3613 .get_mut(idx)
3614 .ok_or_else(|| invalid_decommission_pool_index_error(current_count, idx))?;
3615 if pool.id != idx || saved_pool.id != idx || pool.cmd_line != saved_pool.cmd_line || saved.version < current.version {
3616 return Err(stale("pool identity changed before capacity publication"));
3617 }
3618 let info = pool
3619 .decommission
3620 .as_mut()
3621 .ok_or_else(|| stale("decommission cleared before capacity publication"))?;
3622 let saved_info = saved_pool
3623 .decommission
3624 .as_ref()
3625 .ok_or_else(|| stale("persisted decommission is missing during capacity publication"))?;
3626 if (info.start_time, info.queued, info.complete, info.failed, info.canceled)
3627 != (
3628 saved_info.start_time,
3629 saved_info.queued,
3630 saved_info.complete,
3631 saved_info.failed,
3632 saved_info.canceled,
3633 )
3634 {
3635 return Err(stale("decommission generation or terminal state changed before capacity publication"));
3636 }
3637 let identity = |reservation: &DecommissionCapacityReservation| {
3638 (
3639 reservation.source_pool_index,
3640 reservation.operation_id,
3641 reservation.generation,
3642 reservation.owner_nonce,
3643 reservation.model_version,
3644 )
3645 };
3646 if info.capacity_reservation.as_ref().map(identity) != saved_info.capacity_reservation.as_ref().map(identity) {
3647 return Err(stale("decommission capacity owner changed before publication"));
3648 }
3649
3650 info.progress_save_last_at.get_or_insert(pool.last_update);
3653 info.capacity_reservation = saved_info.capacity_reservation.clone();
3654 info.capacity_blocked_reason = saved_info.capacity_blocked_reason.clone();
3655 pool.last_update = pool.last_update.max(saved_pool.last_update);
3656 current.version = current.version.max(saved.version);
3657 Ok(())
3658}
3659
3660fn resolve_start_decommission_pool_meta_reload_result(result: Result<()>) -> Result<()> {
3661 resolve_decommission_pool_meta_reload_result(result, "start_decommission")
3662}
3663
3664fn activation_rebalance_meta_lock_error(err: rustfs_lock::LockError) -> Error {
3665 match err {
3666 rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
3667 mode: "write",
3668 bucket: RUSTFS_META_BUCKET.to_string(),
3669 object: REBAL_META_NAME.to_string(),
3670 required,
3671 achieved,
3672 },
3673 other => Error::other(format!(
3674 "failed to acquire rebalance activation lock on {RUSTFS_META_BUCKET}/{REBAL_META_NAME}: {other}"
3675 )),
3676 }
3677}
3678
3679fn activation_pool_meta_lock_error(err: rustfs_lock::LockError) -> Error {
3680 match err {
3681 rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
3682 mode: "write",
3683 bucket: RUSTFS_META_BUCKET.to_string(),
3684 object: POOL_META_NAME.to_string(),
3685 required,
3686 achieved,
3687 },
3688 other => Error::other(format!(
3689 "failed to acquire pool activation lock on {RUSTFS_META_BUCKET}/{POOL_META_NAME}: {other}"
3690 )),
3691 }
3692}
3693
3694pub(crate) struct PoolRebalanceActivationFence {
3695 pool_meta_guard: rustfs_lock::NamespaceLockGuard,
3696 rebalance_meta_guard: rustfs_lock::NamespaceLockGuard,
3697 fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
3698 decommission_target_fence_proof: Option<crate::services::notification_sys::DecommissionTargetFenceFleetProofToken>,
3699 #[cfg(test)]
3700 forced_lost: Arc<AtomicBool>,
3701}
3702
3703impl PoolRebalanceActivationFence {
3704 pub(crate) fn set_fleet_proof(
3705 &mut self,
3706 fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
3707 ) {
3708 self.fleet_proof = fleet_proof;
3709 }
3710
3711 fn set_decommission_target_fence_proof(
3712 &mut self,
3713 fleet_proof: Option<crate::services::notification_sys::DecommissionTargetFenceFleetProofToken>,
3714 ) {
3715 self.decommission_target_fence_proof = fleet_proof;
3716 }
3717
3718 pub(crate) fn ensure_held(&self) -> Result<()> {
3719 #[cfg(test)]
3720 let forced_lost = self.forced_lost.load(Ordering::Acquire);
3721 #[cfg(not(test))]
3722 let forced_lost = false;
3723 if forced_lost || self.pool_meta_guard.is_lock_lost() || self.rebalance_meta_guard.is_lock_lost() {
3724 return Err(Error::other("activation lock lost before metadata commit or worker admission"));
3725 }
3726 if self
3727 .fleet_proof
3728 .as_ref()
3729 .is_some_and(|proof| !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof))
3730 {
3731 return Err(Error::other(POOL_ACTIVATION_FLEET_PROOF_EXPIRED));
3732 }
3733 if self
3734 .decommission_target_fence_proof
3735 .as_ref()
3736 .is_some_and(|proof| !crate::services::notification_sys::decommission_target_fence_fleet_proof_matches(proof))
3737 {
3738 return Err(Error::other(DECOMMISSION_TARGET_FLEET_PROOF_EXPIRED));
3739 }
3740
3741 Ok(())
3742 }
3743
3744 pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
3745 opts.add_namespace_lock_guard(&self.pool_meta_guard);
3746 opts.add_namespace_lock_guard(&self.rebalance_meta_guard);
3747 }
3748
3749 #[cfg(test)]
3750 fn force_lost_for_test(&self) {
3751 self.forced_lost.store(true, Ordering::Release);
3752 }
3753}
3754
3755#[cfg(test)]
3756tokio::task_local! {
3757 pub(crate) static REBALANCE_ACTIVATION_LOCK_ATTEMPT: Arc<tokio::sync::Notify>;
3758}
3759
3760pub(crate) async fn acquire_pool_rebalance_activation_locks<S>(
3761 pool: Arc<S>,
3762 fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
3763) -> Result<PoolRebalanceActivationFence>
3764where
3765 S: crate::storage_api_contracts::namespace::NamespaceLocking<
3766 Error = Error,
3767 NamespaceLock = rustfs_lock::NamespaceLockWrapper,
3768 >,
3769{
3770 let rebalance_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
3774 #[cfg(test)]
3775 let _ = REBALANCE_ACTIVATION_LOCK_ATTEMPT.try_with(|attempted| attempted.notify_one());
3776 let rebalance_meta_guard = rebalance_meta_lock
3777 .get_write_lock(get_lock_acquire_timeout())
3778 .await
3779 .map_err(activation_rebalance_meta_lock_error)?;
3780 let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
3781 let pool_meta_guard = pool_meta_lock
3782 .get_write_lock(get_lock_acquire_timeout())
3783 .await
3784 .map_err(activation_pool_meta_lock_error)?;
3785
3786 Ok(PoolRebalanceActivationFence {
3787 pool_meta_guard,
3788 rebalance_meta_guard,
3789 fleet_proof,
3790 decommission_target_fence_proof: None,
3791 #[cfg(test)]
3792 forced_lost: Arc::new(AtomicBool::new(false)),
3793 })
3794}
3795
3796pub(crate) async fn acquire_pool_activation_fleet_proof(
3797 ctx: &crate::runtime::instance::InstanceContext,
3798) -> Result<Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>> {
3799 if !ctx.is_dist_erasure().await {
3800 return Ok(None);
3801 }
3802 crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof()
3803 .map(Some)
3804 .ok_or_else(|| Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED))
3805}
3806
3807pub fn is_pool_activation_fleet_proof_error(err: &Error) -> bool {
3808 matches!(err, Error::Io(io_error) if io_error.kind() == std::io::ErrorKind::Other && {
3812 let message = io_error.to_string();
3813 message.contains(POOL_ACTIVATION_FLEET_PROOF_REQUIRED) || message.contains(POOL_ACTIVATION_FLEET_PROOF_EXPIRED)
3814 })
3815}
3816
3817#[cfg(test)]
3818#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3819pub(crate) enum PoolActivationStartKind {
3820 Rebalance,
3821 Decommission,
3822}
3823
3824#[cfg(test)]
3825struct PoolActivationDurableSaveBarrierState {
3826 pool_key: usize,
3827 arrived: tokio::sync::Notify,
3828 release: tokio::sync::Notify,
3829 force_fence_loss: AtomicBool,
3830}
3831
3832#[cfg(test)]
3833static POOL_ACTIVATION_DURABLE_SAVE_BARRIER: std::sync::OnceLock<
3834 std::sync::Mutex<Option<Arc<PoolActivationDurableSaveBarrierState>>>,
3835> = std::sync::OnceLock::new();
3836
3837#[cfg(test)]
3838pub(crate) struct PoolActivationDurableSaveBarrier {
3839 state: Arc<PoolActivationDurableSaveBarrierState>,
3840}
3841
3842#[cfg(test)]
3843fn pool_activation_test_pool_key<S>(pool: &Arc<S>) -> usize {
3844 Arc::as_ptr(pool).cast::<()>() as usize
3845}
3846
3847#[cfg(test)]
3848impl PoolActivationDurableSaveBarrier {
3849 pub(crate) fn install<S>(pool: &Arc<S>) -> Self {
3850 let state = Arc::new(PoolActivationDurableSaveBarrierState {
3851 pool_key: pool_activation_test_pool_key(pool),
3852 arrived: tokio::sync::Notify::new(),
3853 release: tokio::sync::Notify::new(),
3854 force_fence_loss: AtomicBool::new(false),
3855 });
3856 let mut barrier = POOL_ACTIVATION_DURABLE_SAVE_BARRIER
3857 .get_or_init(|| std::sync::Mutex::new(None))
3858 .lock()
3859 .expect("pool activation durable save barrier should not be poisoned");
3860 assert!(barrier.is_none(), "pool activation durable save barrier must be unique");
3861 *barrier = Some(Arc::clone(&state));
3862 Self { state }
3863 }
3864
3865 pub(crate) async fn wait_until_paused(&self) {
3866 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.arrived.notified())
3867 .await
3868 .expect("activation should reach the post-durable-save barrier");
3869 }
3870
3871 pub(crate) fn release_after_fence_loss(&self) {
3872 self.state.force_fence_loss.store(true, Ordering::Release);
3873 self.state.release.notify_one();
3874 }
3875
3876 pub(crate) fn release_without_fence_loss(&self) {
3877 self.state.release.notify_one();
3878 }
3879}
3880
3881#[cfg(test)]
3882impl Drop for PoolActivationDurableSaveBarrier {
3883 fn drop(&mut self) {
3884 self.state.release.notify_one();
3885 let mut barrier = POOL_ACTIVATION_DURABLE_SAVE_BARRIER
3886 .get_or_init(|| std::sync::Mutex::new(None))
3887 .lock()
3888 .expect("pool activation durable save barrier should not be poisoned");
3889 if barrier.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
3890 *barrier = None;
3891 }
3892 }
3893}
3894
3895#[cfg(test)]
3896pub(crate) async fn pause_pool_activation_after_durable_save<S>(pool: &Arc<S>, fence: &PoolRebalanceActivationFence) {
3897 let pool_key = pool_activation_test_pool_key(pool);
3898 let barrier = {
3899 let mut barrier = POOL_ACTIVATION_DURABLE_SAVE_BARRIER
3900 .get_or_init(|| std::sync::Mutex::new(None))
3901 .lock()
3902 .expect("pool activation durable save barrier should not be poisoned");
3903 if barrier.as_ref().is_some_and(|state| state.pool_key == pool_key) {
3904 barrier.take()
3905 } else {
3906 None
3907 }
3908 };
3909 if let Some(barrier) = barrier {
3910 barrier.arrived.notify_one();
3911 barrier.release.notified().await;
3912 if barrier.force_fence_loss.load(Ordering::Acquire) {
3913 fence.force_lost_for_test();
3914 }
3915 }
3916}
3917
3918#[cfg(test)]
3919struct PoolActivationStartProbeState {
3920 kind: PoolActivationStartKind,
3921 preflight_side_effect_attempted: std::sync::atomic::AtomicBool,
3922 attempted: std::sync::atomic::AtomicBool,
3923 notify: tokio::sync::Notify,
3924}
3925
3926#[cfg(test)]
3927static POOL_ACTIVATION_START_PROBES: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<PoolActivationStartProbeState>>>> =
3928 std::sync::OnceLock::new();
3929
3930#[cfg(test)]
3931pub(crate) struct PoolActivationStartProbe {
3932 state: Arc<PoolActivationStartProbeState>,
3933}
3934
3935#[cfg(test)]
3936impl PoolActivationStartProbe {
3937 pub(crate) fn install(kind: PoolActivationStartKind) -> Self {
3938 let state = Arc::new(PoolActivationStartProbeState {
3939 kind,
3940 preflight_side_effect_attempted: std::sync::atomic::AtomicBool::new(false),
3941 attempted: std::sync::atomic::AtomicBool::new(false),
3942 notify: tokio::sync::Notify::new(),
3943 });
3944 POOL_ACTIVATION_START_PROBES
3945 .get_or_init(|| std::sync::Mutex::new(Vec::new()))
3946 .lock()
3947 .expect("pool activation start probe should not be poisoned")
3948 .push(Arc::clone(&state));
3949 Self { state }
3950 }
3951
3952 pub(crate) async fn wait_until_attempted(&self) {
3953 while !self.state.attempted.load(Ordering::Acquire) {
3954 self.state.notify.notified().await;
3955 }
3956 }
3957
3958 pub(crate) fn preflight_side_effect_was_attempted(&self) -> bool {
3959 self.state.preflight_side_effect_attempted.load(Ordering::Acquire)
3960 }
3961
3962 pub(crate) fn activation_was_attempted(&self) -> bool {
3963 self.state.attempted.load(Ordering::Acquire)
3964 }
3965}
3966
3967#[cfg(test)]
3968impl Drop for PoolActivationStartProbe {
3969 fn drop(&mut self) {
3970 let mut probes = POOL_ACTIVATION_START_PROBES
3971 .get_or_init(|| std::sync::Mutex::new(Vec::new()))
3972 .lock()
3973 .expect("pool activation start probe should not be poisoned");
3974 probes.retain(|state| !Arc::ptr_eq(state, &self.state));
3975 }
3976}
3977
3978#[cfg(test)]
3979pub(crate) fn observe_pool_activation_start_attempt(kind: PoolActivationStartKind) {
3980 let probes = POOL_ACTIVATION_START_PROBES
3981 .get_or_init(|| std::sync::Mutex::new(Vec::new()))
3982 .lock()
3983 .expect("pool activation start probe should not be poisoned")
3984 .iter()
3985 .filter(|state| state.kind == kind)
3986 .cloned()
3987 .collect::<Vec<_>>();
3988 for state in probes {
3989 state.attempted.store(true, Ordering::Release);
3990 state.notify.notify_one();
3991 }
3992}
3993
3994#[cfg(test)]
3995fn observe_pool_activation_preflight_side_effect_attempt(kind: PoolActivationStartKind) {
3996 let probes = POOL_ACTIVATION_START_PROBES
3997 .get_or_init(|| std::sync::Mutex::new(Vec::new()))
3998 .lock()
3999 .expect("pool activation start probe should not be poisoned")
4000 .iter()
4001 .filter(|state| state.kind == kind)
4002 .cloned()
4003 .collect::<Vec<_>>();
4004 for state in probes {
4005 state.preflight_side_effect_attempted.store(true, Ordering::Release);
4006 }
4007}
4008
4009fn rollback_decommission_pool_meta(pool_meta: &mut PoolMeta, previous_pool_meta: &PoolMeta, indices: &[usize]) {
4010 publish_pool_meta_updates(pool_meta, previous_pool_meta, indices);
4011}
4012
4013#[derive(Debug)]
4014struct DecommissionCancelCommit {
4015 previous_start_time: Option<OffsetDateTime>,
4016 previous_queued: bool,
4017 previous_last_update: OffsetDateTime,
4018 canceled_pool: PoolStatus,
4019}
4020
4021fn commit_decommission_cancel(pool_meta: &mut PoolMeta, idx: usize, commit: DecommissionCancelCommit) -> Result<()> {
4022 let pool_count = pool_meta.pools.len();
4023 let Some(current) = pool_meta.pools.get(idx) else {
4024 return Err(invalid_decommission_pool_index_error(pool_count, idx));
4025 };
4026 let cancel_already_published = commit
4028 .canceled_pool
4029 .decommission
4030 .as_ref()
4031 .is_some_and(|info| info.canceled && !info.complete && !info.failed && info.start_time.is_none())
4032 && PersistedPoolStatus::from(current) == PersistedPoolStatus::from(&commit.canceled_pool);
4033 if cancel_already_published {
4034 return Ok(());
4035 }
4036
4037 let matches_generation = current.id == commit.canceled_pool.id
4038 && current.cmd_line == commit.canceled_pool.cmd_line
4039 && current.decommission.as_ref().is_some_and(|info| {
4040 info.start_time == commit.previous_start_time
4041 && info.queued == commit.previous_queued
4042 && is_decommission_active(info.complete, info.failed, info.canceled)
4043 && (commit.previous_start_time.is_some() || current.last_update == commit.previous_last_update)
4044 });
4045 if !matches_generation {
4046 return Err(Error::other(format!(
4047 "failed to publish decommission cancel for pool {idx}: operation generation changed"
4048 )));
4049 }
4050
4051 pool_meta.pools[idx] = commit.canceled_pool;
4052 Ok(())
4053}
4054
4055fn rollback_start_decommission_pool_meta(pool_meta: &mut PoolMeta, previous_pool_meta: &PoolMeta, indices: &[usize]) {
4056 let active_updates = indices
4057 .iter()
4058 .filter_map(|&idx| pool_meta.pools.get(idx).map(|pool| (idx, pool.last_update)))
4059 .collect::<Vec<_>>();
4060 rollback_decommission_pool_meta(pool_meta, previous_pool_meta, indices);
4061 let rollback_at = OffsetDateTime::now_utc();
4062 for (idx, active_update) in active_updates {
4063 if let Some(pool) = pool_meta.pools.get_mut(idx) {
4064 pool.last_update = std::cmp::max(rollback_at, active_update + Duration::nanoseconds(1));
4065 }
4066 }
4067}
4068
4069fn ensure_pool_meta_write_fence(guard: &rustfs_lock::NamespaceLockGuard, operation: &str) -> Result<()> {
4070 if guard.is_lock_lost() {
4071 return Err(Error::other(format!("{operation}: pool metadata distributed fence was lost")));
4072 }
4073 Ok(())
4074}
4075
4076fn ensure_pool_not_left_in_cmdline_after_decommission(position: usize, cmd_line: &str, completed: bool) -> Result<()> {
4077 if completed {
4078 return Err(Error::other(format!(
4079 "pool({}) = {} is decommissioned, please remove from server command line",
4080 position + 1,
4081 cmd_line
4082 )));
4083 }
4084
4085 Ok(())
4086}
4087
4088fn resolve_decommission_listing_worker_result(
4089 set_idx: usize,
4090 worker_result: std::result::Result<Result<()>, tokio::task::JoinError>,
4091) -> Result<()> {
4092 worker_result.map_err(|err| Error::other(format!("decommission listing worker {set_idx} task join error: {err}")))?
4093}
4094
4095fn ensure_decommission_deferred_drained(bucket: &str, set_idx: usize, deferred_remaining: usize) -> Result<()> {
4102 if deferred_remaining == 0 {
4103 return Ok(());
4104 }
4105 Err(decommission_capacity_blocked_error(format!(
4106 "decommission bucket {bucket} set {set_idx} finished with {deferred_remaining} deferred entries remaining"
4107 )))
4108}
4109
4110fn decommission_entry_budget_acquire_error(err: impl Display) -> Error {
4111 Error::other(format!("decommission entry budget permit acquire failed: {err}"))
4112}
4113
4114#[cfg(test)]
4119fn resolve_decommission_entry_test_outcome(outcome: Result<DecommissionEntryOutcome>) -> Result<()> {
4120 match outcome? {
4121 DecommissionEntryOutcome::Complete => Ok(()),
4122 DecommissionEntryOutcome::Deferred(err) => Err(err),
4123 }
4124}
4125
4126fn should_retry_decommission_listing(err: &Error, attempt: usize, max_attempts: usize) -> bool {
4127 !is_err_bucket_not_found(err) && attempt + 1 < max_attempts
4128}
4129
4130async fn wait_decommission_retry_backoff(rx: &CancellationToken, delay: std::time::Duration) -> bool {
4131 tokio::select! {
4132 _ = rx.cancelled() => true,
4133 _ = tokio::time::sleep(delay) => false,
4134 }
4135}
4136
4137fn decommission_retry_backoff_ceiling(base: std::time::Duration, attempt: usize) -> std::time::Duration {
4140 let exponent = u32::try_from(attempt.saturating_sub(1)).unwrap_or(u32::MAX).min(31);
4141 base.saturating_mul(1u32 << exponent).min(DECOMMISSION_RETRY_BACKOFF_CAP)
4142}
4143
4144fn decommission_retry_backoff_delay(base: std::time::Duration, attempt: usize) -> std::time::Duration {
4150 let ceiling = decommission_retry_backoff_ceiling(base, attempt);
4151 if ceiling.is_zero() {
4152 return ceiling;
4153 }
4154 let ceiling_millis = u64::try_from(ceiling.as_millis()).unwrap_or(u64::MAX);
4155 std::time::Duration::from_millis(rand::rng().random_range(0..=ceiling_millis))
4156}
4157
4158pub(crate) fn decommission_capacity_gate_retry_delay(attempt: usize) -> std::time::Duration {
4160 let delay = decommission_retry_backoff_delay(DECOMMISSION_CAPACITY_GATE_RETRY_BASE, attempt);
4161 record_decommission_capacity_gate_wait(delay);
4162 delay
4163}
4164
4165#[cfg(test)]
4166async fn run_decommission_listing_with_retry<List, ListFuture>(
4167 rx: CancellationToken,
4168 bucket: String,
4169 cb: ListCallback,
4170 pool_idx: usize,
4171 set_idx: usize,
4172 max_attempts: usize,
4173 list: List,
4174) -> Result<()>
4175where
4176 List: FnMut(ListCallback) -> ListFuture,
4177 ListFuture: std::future::Future<Output = Result<()>>,
4178{
4179 run_decommission_listing_with_retry_and_drain(rx, bucket, cb, pool_idx, set_idx, max_attempts, list, || async { false }).await
4180}
4181
4182#[allow(clippy::too_many_arguments)]
4183async fn run_decommission_listing_with_retry_and_drain<List, ListFuture, Drain, DrainFuture>(
4184 rx: CancellationToken,
4185 bucket: String,
4186 cb: ListCallback,
4187 pool_idx: usize,
4188 set_idx: usize,
4189 max_attempts: usize,
4190 mut list: List,
4191 mut drain: Drain,
4192) -> Result<()>
4193where
4194 List: FnMut(ListCallback) -> ListFuture,
4195 ListFuture: std::future::Future<Output = Result<()>>,
4196 Drain: FnMut() -> DrainFuture,
4197 DrainFuture: std::future::Future<Output = bool>,
4198{
4199 let max_attempts = max_attempts.max(1);
4200
4201 for attempt in 0..max_attempts {
4202 if rx.is_cancelled() {
4203 debug!(
4204 event = EVENT_DECOMMISSION_BUCKET,
4205 component = LOG_COMPONENT_ECSTORE,
4206 subsystem = LOG_SUBSYSTEM_POOLS,
4207 pool_index = pool_idx,
4208 set_index = set_idx,
4209 bucket = %bucket,
4210 state = "listing_worker_cancelled",
4211 "Decommission listing worker cancelled"
4212 );
4213 return Ok(());
4214 }
4215
4216 debug!(
4217 event = EVENT_DECOMMISSION_BUCKET,
4218 component = LOG_COMPONENT_ECSTORE,
4219 subsystem = LOG_SUBSYSTEM_POOLS,
4220 pool_index = pool_idx,
4221 set_index = set_idx,
4222 bucket = %bucket,
4223 attempt = attempt + 1,
4224 max_attempts,
4225 state = "listing_started",
4226 "Decommission listing started"
4227 );
4228
4229 let list_result = list(cb.clone()).await;
4230 if drain().await {
4231 return Ok(());
4232 }
4233
4234 match list_result {
4235 Ok(()) => {
4236 debug!(
4237 event = EVENT_DECOMMISSION_BUCKET,
4238 component = LOG_COMPONENT_ECSTORE,
4239 subsystem = LOG_SUBSYSTEM_POOLS,
4240 pool_index = pool_idx,
4241 set_index = set_idx,
4242 bucket = %bucket,
4243 attempt = attempt + 1,
4244 max_attempts,
4245 state = "listing_completed",
4246 "Decommission listing completed"
4247 );
4248 return Ok(());
4249 }
4250 Err(err) if is_err_bucket_not_found(&err) => {
4251 warn!(
4252 event = EVENT_DECOMMISSION_BUCKET,
4253 component = LOG_COMPONENT_ECSTORE,
4254 subsystem = LOG_SUBSYSTEM_POOLS,
4255 pool_index = pool_idx,
4256 set_index = set_idx,
4257 bucket = %bucket,
4258 attempt = attempt + 1,
4259 max_attempts,
4260 state = "listing_bucket_missing",
4261 "Decommission listing bucket missing"
4262 );
4263 return Ok(());
4264 }
4265 Err(err) if should_retry_decommission_listing(&err, attempt, max_attempts) => {
4266 error!(
4267 event = EVENT_DECOMMISSION_BUCKET,
4268 component = LOG_COMPONENT_ECSTORE,
4269 subsystem = LOG_SUBSYSTEM_POOLS,
4270 pool_index = pool_idx,
4271 set_index = set_idx,
4272 bucket = %bucket,
4273 attempt = attempt + 1,
4274 max_attempts,
4275 retry_delay_ms = DECOMMISSION_LISTING_RETRY_DELAY.as_millis(),
4276 state = "listing_failed_retrying",
4277 error = ?err,
4278 "Decommission listing failed; retrying"
4279 );
4280 if wait_decommission_retry_backoff(&rx, DECOMMISSION_LISTING_RETRY_DELAY).await {
4281 debug!(
4282 event = EVENT_DECOMMISSION_BUCKET,
4283 component = LOG_COMPONENT_ECSTORE,
4284 subsystem = LOG_SUBSYSTEM_POOLS,
4285 pool_index = pool_idx,
4286 set_index = set_idx,
4287 bucket = %bucket,
4288 state = "listing_worker_cancelled",
4289 "Decommission listing worker cancelled during retry wait"
4290 );
4291 return Ok(());
4292 }
4293 }
4294 Err(err) => {
4295 error!(
4296 event = EVENT_DECOMMISSION_BUCKET,
4297 component = LOG_COMPONENT_ECSTORE,
4298 subsystem = LOG_SUBSYSTEM_POOLS,
4299 pool_index = pool_idx,
4300 set_index = set_idx,
4301 bucket = %bucket,
4302 attempt = attempt + 1,
4303 max_attempts,
4304 state = "listing_failed",
4305 error = ?err,
4306 "Decommission listing failed"
4307 );
4308 return Err(Error::other(format!(
4309 "decommission listing failed for bucket {bucket} pool {pool_idx} set {set_idx} attempt {}/{}: {err}",
4310 attempt + 1,
4311 max_attempts
4312 )));
4313 }
4314 }
4315 }
4316
4317 Ok(())
4318}
4319
4320fn should_count_decommission_version_complete(ignore: bool, cleanup_ignored: bool, failure: bool) -> bool {
4321 cleanup_ignored || (!ignore && !failure)
4322}
4323
4324fn is_decommission_copy_cleanup_safe_error(err: &Error) -> bool {
4325 if is_err_object_not_found(err) || is_err_version_not_found(err) {
4328 return true;
4329 }
4330
4331 data_movement::data_movement_stage_source(err).is_some_and(is_decommission_copy_cleanup_safe_error)
4334}
4335
4336fn is_decommission_target_capacity_error(err: &Error) -> bool {
4337 if matches!(err, Error::DiskFull | Error::StorageFull) {
4338 return true;
4339 }
4340
4341 if let Some(source) = data_movement::data_movement_stage_source(err) {
4345 return is_decommission_target_capacity_error(source);
4346 }
4347
4348 let message = err.to_string();
4349 let disk_full = Error::DiskFull.to_string();
4350 let storage_full = Error::StorageFull.to_string();
4351 message.contains(&disk_full) || message.contains(&storage_full)
4352}
4353
4354fn should_cleanup_decommission_source_entry(decommissioned: usize, total_versions: usize, expired: usize) -> bool {
4355 decommissioned.saturating_add(expired) == total_versions
4356}
4357
4358fn should_fail_decommission_pool_after_exhausted_source_changed(exhausted_entries: usize) -> bool {
4359 exhausted_entries > DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT
4360}
4361
4362#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4363enum DecommissionEntryAttemptOutcome {
4364 Complete,
4365 SourceChanged,
4366}
4367
4368#[derive(Debug)]
4370struct DeferredDecommissionEntry {
4371 name: String,
4372 attempts: usize,
4373}
4374
4375#[derive(Debug)]
4377enum DecommissionEntryOutcome {
4378 Complete,
4379 Deferred(Error),
4385}
4386
4387fn resolve_decommission_entry_failure(err: Error) -> Result<DecommissionEntryOutcome> {
4396 match classify_decommission_capacity_failure(&err) {
4397 DecommissionCapacityFailure::TargetGateBusy { .. } | DecommissionCapacityFailure::IntentConflict => {
4398 Ok(DecommissionEntryOutcome::Deferred(err))
4399 }
4400 DecommissionCapacityFailure::BenignContention => Ok(DecommissionEntryOutcome::Complete),
4401 DecommissionCapacityFailure::Fatal => Err(err),
4402 }
4403}
4404
4405#[cfg(test)]
4406pub(crate) type DecommissionTestFaultDecision = Arc<dyn Fn(&'static str, &str, &str, usize, bool) -> bool + Send + Sync>;
4407
4408#[cfg(test)]
4409static DECOMMISSION_TEST_FAULT_HOOK: std::sync::OnceLock<std::sync::Mutex<Option<DecommissionTestFaultDecision>>> =
4410 std::sync::OnceLock::new();
4411
4412#[cfg(test)]
4413pub(crate) struct DecommissionTestFaultGuard(DecommissionTestFaultDecision);
4414
4415#[cfg(test)]
4416impl DecommissionTestFaultGuard {
4417 pub(crate) fn install(decision: DecommissionTestFaultDecision) -> Self {
4418 let mut slot = DECOMMISSION_TEST_FAULT_HOOK
4419 .get_or_init(|| std::sync::Mutex::new(None))
4420 .lock()
4421 .expect("decommission test fault hook mutex should not poison");
4422 assert!(slot.is_none(), "decommission test fault hook must be unique");
4423 let stored = Arc::clone(&decision);
4424 *slot = Some(decision);
4425 Self(stored)
4426 }
4427}
4428
4429#[cfg(test)]
4430impl Drop for DecommissionTestFaultGuard {
4431 fn drop(&mut self) {
4432 let mut slot = DECOMMISSION_TEST_FAULT_HOOK
4433 .get_or_init(|| std::sync::Mutex::new(None))
4434 .lock()
4435 .expect("decommission test fault hook mutex should not poison");
4436 if slot.as_ref().is_some_and(|decision| Arc::ptr_eq(decision, &self.0)) {
4437 *slot = None;
4438 }
4439 }
4440}
4441
4442#[cfg(test)]
4443fn decommission_test_wrap_result<T>(
4444 stage: &'static str,
4445 bucket: &str,
4446 object: &str,
4447 attempt: usize,
4448 result: Result<T>,
4449) -> Result<T> {
4450 let decision = DECOMMISSION_TEST_FAULT_HOOK
4451 .get_or_init(|| std::sync::Mutex::new(None))
4452 .lock()
4453 .expect("decommission test fault hook mutex should not poison")
4454 .clone();
4455 let inject = decision.is_some_and(|decision| decision(stage, bucket, object, attempt, result.is_ok()));
4456 if result.is_ok() && inject {
4457 return Err(Error::other(format!(
4458 "injected decommission test fault at {stage} attempt {attempt} for {bucket}/{object}"
4459 )));
4460 }
4461 result
4462}
4463
4464#[cfg(test)]
4465pub(crate) type DecommissionCleanupMutationHook = Arc<dyn Fn(&str, &str, usize) -> BoxFuture<'static, ()> + Send + Sync>;
4466
4467#[cfg(test)]
4468static DECOMMISSION_CLEANUP_MUTATION_HOOK: std::sync::OnceLock<std::sync::Mutex<Option<DecommissionCleanupMutationHook>>> =
4469 std::sync::OnceLock::new();
4470
4471#[cfg(test)]
4472pub(crate) struct DecommissionCleanupMutationGuard(DecommissionCleanupMutationHook);
4473
4474#[cfg(test)]
4475impl DecommissionCleanupMutationGuard {
4476 pub(crate) fn install(hook: DecommissionCleanupMutationHook) -> Self {
4477 let mut slot = DECOMMISSION_CLEANUP_MUTATION_HOOK
4478 .get_or_init(|| std::sync::Mutex::new(None))
4479 .lock()
4480 .expect("decommission cleanup mutation hook mutex should not poison");
4481 assert!(slot.is_none(), "decommission cleanup mutation hook must be unique");
4482 let stored = Arc::clone(&hook);
4483 *slot = Some(hook);
4484 Self(stored)
4485 }
4486}
4487
4488#[cfg(test)]
4489impl Drop for DecommissionCleanupMutationGuard {
4490 fn drop(&mut self) {
4491 let mut slot = DECOMMISSION_CLEANUP_MUTATION_HOOK
4492 .get_or_init(|| std::sync::Mutex::new(None))
4493 .lock()
4494 .expect("decommission cleanup mutation hook mutex should not poison");
4495 if slot.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.0)) {
4496 *slot = None;
4497 }
4498 }
4499}
4500
4501#[cfg(test)]
4502async fn run_decommission_cleanup_mutation_hook(bucket: &str, object: &str, attempt: usize) {
4503 let hook = DECOMMISSION_CLEANUP_MUTATION_HOOK
4504 .get_or_init(|| std::sync::Mutex::new(None))
4505 .lock()
4506 .expect("decommission cleanup mutation hook mutex should not poison")
4507 .clone();
4508 if let Some(hook) = hook {
4509 hook(bucket, object, attempt).await;
4510 }
4511}
4512
4513const DECOMMISSION_FREE_VERSION_MIGRATED_REASON: &str = "tier_free_version_migrated";
4514const DECOMMISSION_FREE_VERSION_CONSUMED_REASON: &str = "tier_free_version_already_consumed";
4515const DECOMMISSION_FREE_VERSION_RETAINED_REASON: &str = "tier_free_version_migration_failed";
4516const DECOMMISSION_FREE_VERSION_SWEEP_REASON: &str = "tier_free_version_unresolved_after_decommission";
4517const DECOMMISSION_FREE_VERSION_DISPOSITION_REASON: &str = "tier_free_version_disposition_recorded";
4518
4519#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
4520struct DecommissionFreeVersionDisposition {
4521 migrated: usize,
4522 consumed: usize,
4523 retained: usize,
4524}
4525
4526impl DecommissionFreeVersionDisposition {
4527 fn record_migrated(&mut self) {
4528 self.migrated += 1;
4529 }
4530
4531 fn record_consumed(&mut self) {
4532 self.consumed += 1;
4533 }
4534
4535 fn record_retained(&mut self) {
4536 self.retained += 1;
4537 }
4538
4539 fn total(self) -> usize {
4540 self.migrated.saturating_add(self.consumed).saturating_add(self.retained)
4541 }
4542}
4543
4544enum DecommissionFreeVersionAttempt {
4545 Migrated,
4546 Consumed,
4547 CapacityFailure(Error),
4548 Retry(Error),
4549}
4550
4551fn classify_decommission_free_version_attempt(result: Result<()>) -> DecommissionFreeVersionAttempt {
4552 match result {
4553 Ok(()) => DecommissionFreeVersionAttempt::Migrated,
4554 Err(err) if is_decommission_copy_cleanup_safe_error(&err) => DecommissionFreeVersionAttempt::Consumed,
4555 Err(err) if is_decommission_target_capacity_error(&err) => DecommissionFreeVersionAttempt::CapacityFailure(err),
4556 Err(err) => DecommissionFreeVersionAttempt::Retry(err),
4557 }
4558}
4559
4560#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4561#[allow(
4562 dead_code,
4563 reason = "terminal-state classification asserted by this file's tests (backlog#1823)"
4564)]
4565enum DecommissionTerminalState {
4566 Completed,
4567 Failed,
4568}
4569
4570#[allow(
4571 dead_code,
4572 reason = "terminal-state classification asserted by this file's tests (backlog#1823)"
4573)]
4574fn classify_decommission_terminal_state(failed_items_present: bool) -> DecommissionTerminalState {
4575 if failed_items_present {
4576 DecommissionTerminalState::Failed
4577 } else {
4578 DecommissionTerminalState::Completed
4579 }
4580}
4581
4582fn should_preserve_decommission_canceled_state(meta_canceled: bool, _cancel_signal: bool) -> bool {
4583 meta_canceled
4584}
4585
4586fn should_continue_decommission_queue(meta: &PoolMeta, idx: usize) -> bool {
4587 meta.pools
4588 .get(idx)
4589 .and_then(|pool| pool.decommission.as_ref())
4590 .is_some_and(|info| info.complete && !info.failed && !info.canceled)
4591}
4592
4593fn decommission_cancel_signal_result(cancel_signal: bool) -> Result<()> {
4594 if cancel_signal {
4595 Err(StorageError::OperationCanceled)
4596 } else {
4597 Ok(())
4598 }
4599}
4600
4601fn is_decommission_cancel_requested(cancel_signal: bool, pool: Option<&PoolStatus>) -> bool {
4602 cancel_signal
4603 || pool
4604 .and_then(|pool| pool.decommission.as_ref())
4605 .is_some_and(|info| info.canceled)
4606}
4607
4608fn should_skip_canceled_decommission_routine(cancel_signal: bool, pool: Option<&PoolStatus>) -> bool {
4609 cancel_signal
4610 && pool
4611 .and_then(|pool| pool.decommission.as_ref())
4612 .is_some_and(|info| info.canceled)
4613}
4614
4615async fn run_decommission_buckets_bounded<F>(
4616 rx: CancellationToken,
4617 buckets: Vec<DecomBucketInfo>,
4618 limit: usize,
4619 mut start_bucket: F,
4620) -> Result<()>
4621where
4622 F: FnMut(DecomBucketInfo, CancellationToken) -> BoxFuture<'static, Result<()>>,
4623{
4624 let mut pending = buckets.into_iter();
4625 let mut active: FuturesUnordered<BoxFuture<'static, Result<()>>> = FuturesUnordered::new();
4626 let mut first_err = None;
4627 let limit = limit.max(1);
4628
4629 for _ in 0..limit {
4630 let Some(bucket) = pending.next() else {
4631 break;
4632 };
4633
4634 active.push(start_bucket(bucket, rx.clone()));
4635 }
4636
4637 while let Some(result) = active.next().await {
4638 if let Err(err) = result {
4639 rx.cancel();
4640 if first_err.is_none() {
4641 first_err = Some(err);
4642 }
4643 continue;
4644 }
4645
4646 if first_err.is_some() || rx.is_cancelled() {
4647 continue;
4648 }
4649
4650 let Some(bucket) = pending.next() else {
4651 continue;
4652 };
4653
4654 active.push(start_bucket(bucket, rx.clone()));
4655 }
4656
4657 if first_err.is_none() && rx.is_cancelled() && pending.len() > 0 {
4658 return decommission_cancel_signal_result(true);
4659 }
4660
4661 if let Some(err) = first_err {
4662 return Err(err);
4663 }
4664
4665 Ok(())
4666}
4667
4668async fn run_decommission_phases<F>(
4669 rx: CancellationToken,
4670 regular_buckets: Vec<DecomBucketInfo>,
4671 meta_buckets: Vec<DecomBucketInfo>,
4672 bucket_concurrency: usize,
4673 mut start_bucket: F,
4674) -> Result<()>
4675where
4676 F: FnMut(DecomBucketInfo, CancellationToken) -> BoxFuture<'static, Result<()>>,
4677{
4678 decommission_cancel_signal_result(rx.is_cancelled())?;
4679
4680 for bucket in meta_buckets {
4681 decommission_cancel_signal_result(rx.is_cancelled())?;
4682 start_bucket(bucket, rx.clone()).await?;
4683 }
4684
4685 decommission_cancel_signal_result(rx.is_cancelled())?;
4686
4687 if bucket_concurrency <= 1 {
4688 for bucket in regular_buckets {
4689 decommission_cancel_signal_result(rx.is_cancelled())?;
4690 start_bucket(bucket, rx.clone()).await?;
4691 }
4692 return Ok(());
4693 }
4694
4695 run_decommission_buckets_bounded(rx, regular_buckets, bucket_concurrency, start_bucket).await
4696}
4697
4698#[cfg(test)]
4699async fn wait_decommission_worker_drain(workers: &Semaphore, limit: usize) -> Result<()> {
4700 let permits = u32::try_from(limit)
4701 .map_err(|_| Error::other(format!("decommission worker limit {limit} exceeds semaphore drain capacity")))?;
4702 let _drain = workers
4703 .acquire_many(permits)
4704 .await
4705 .map_err(|err| Error::other(format!("decommission worker drain failed: {err}")))?;
4706 Ok(())
4707}
4708
4709fn should_reject_decommission_cancel_as_terminal(complete: bool, failed: bool) -> bool {
4710 complete || failed
4711}
4712
4713fn should_retry_decommission_cancel_reload(changed: bool, already_canceled: bool) -> bool {
4714 changed || already_canceled
4715}
4716
4717fn ensure_decommission_cancel_allowed(pool_present: bool, decommission_present: bool, terminal: bool) -> Result<()> {
4718 if !pool_present {
4719 return Err(invalid_decommission_request("failed to cancel decommission: target pool was not found"));
4720 }
4721
4722 if !decommission_present || terminal {
4723 return Err(StorageError::DecommissionNotStarted);
4724 }
4725
4726 Ok(())
4727}
4728
4729fn ensure_decommission_clear_allowed(
4730 pool_present: bool,
4731 decommission_present: bool,
4732 complete: bool,
4733 failed: bool,
4734 canceled: bool,
4735 unresolved_entries: usize,
4736) -> Result<()> {
4737 if !pool_present {
4738 return Err(invalid_decommission_request("failed to clear decommission: target pool was not found"));
4739 }
4740
4741 if !decommission_present {
4742 return Err(StorageError::DecommissionNotStarted);
4743 }
4744
4745 if complete {
4746 return Err(StorageError::DecommissionNotStarted);
4747 }
4748
4749 if !failed && !canceled {
4750 return Err(StorageError::DecommissionAlreadyRunning);
4751 }
4752
4753 if unresolved_entries > 0 {
4754 return Err(invalid_decommission_request(format!(
4755 "failed to clear decommission: {unresolved_entries} unresolved listing entries must be reconciled by retrying decommission"
4756 )));
4757 }
4758
4759 Ok(())
4760}
4761
4762fn ensure_decommission_terminal_operation_supported(single_pool: bool, operation: &str) -> Result<()> {
4763 if single_pool {
4764 return Err(invalid_decommission_request(format!(
4765 "failed to {operation}: single pool deployments do not support decommission"
4766 )));
4767 }
4768
4769 Ok(())
4770}
4771
4772fn validate_start_decommission_request(indices: &[usize], single_pool: bool) -> Result<()> {
4773 if indices.is_empty() {
4774 return Err(invalid_decommission_request(
4775 "failed to start decommission: no target pools were provided",
4776 ));
4777 }
4778
4779 ensure_decommission_terminal_operation_supported(single_pool, "start decommission")
4780}
4781
4782fn require_decommission_store<T>(store: Option<T>, operation: &str) -> Result<T> {
4783 store.ok_or_else(|| Error::other(format!("failed to {operation}: store not initialized")))
4784}
4785
4786fn ensure_decommission_listing_disks_available(has_disks: bool, bucket: &str) -> Result<()> {
4787 if !has_disks {
4788 return Err(Error::other(format!(
4789 "failed to list objects to decommission for bucket {bucket}: no disks available"
4790 )));
4791 }
4792
4793 Ok(())
4794}
4795
4796#[derive(Debug, Clone, Serialize, Deserialize)]
4797pub struct PoolStatus {
4798 #[serde(rename = "id")]
4799 pub id: usize,
4800 #[serde(rename = "cmdline")]
4801 pub cmd_line: String,
4802 #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
4803 pub last_update: OffsetDateTime,
4804 #[serde(rename = "decommissionInfo")]
4805 pub decommission: Option<PoolDecommissionInfo>,
4806}
4807
4808#[derive(Debug, Clone, Default, Serialize, Deserialize)]
4809pub struct PoolMeta {
4810 pub version: u16,
4811 pub pools: Vec<PoolStatus>,
4812 pub dont_save: bool,
4813}
4814
4815#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4816struct PoolMetaRevision {
4817 version: u16,
4818 cluster_id: Option<uuid::Uuid>,
4819 epoch: u64,
4820 generation: u64,
4821 transaction_id: Option<uuid::Uuid>,
4822}
4823
4824impl PoolMetaRevision {
4825 fn legacy(version: u16) -> Self {
4826 Self {
4827 version,
4828 cluster_id: None,
4829 epoch: 0,
4830 generation: 0,
4831 transaction_id: None,
4832 }
4833 }
4834
4835 fn is_generation_protocol(self) -> bool {
4836 self.version == POOL_META_GENERATION_VERSION
4837 }
4838}
4839
4840#[derive(Debug, Clone)]
4841struct PoolMetaCommittedCandidate {
4842 canonical: Vec<u8>,
4843 meta: PoolMeta,
4844 revision: PoolMetaRevision,
4845}
4846
4847#[derive(Debug)]
4848enum PoolMetaReplica {
4849 Missing,
4850 Valid {
4851 raw: Vec<u8>,
4852 canonical: Vec<u8>,
4853 meta: PoolMeta,
4854 revision: PoolMetaRevision,
4855 committed: bool,
4856 previous: Option<Box<PoolMetaCommittedCandidate>>,
4857 },
4858 Corrupt(String),
4859 Incompatible(String),
4860 Unreadable(Arc<Error>),
4861}
4862
4863#[derive(Debug, Clone)]
4864enum PoolMetaCasToken {
4865 Missing,
4866 Existing(String),
4867 Unsafe,
4868}
4869
4870#[derive(Debug)]
4871struct PoolMetaReplicaRead {
4872 replica: PoolMetaReplica,
4873 cas: PoolMetaCasToken,
4874}
4875
4876impl From<PoolMetaReplica> for PoolMetaReplicaRead {
4877 fn from(replica: PoolMetaReplica) -> Self {
4878 Self {
4879 replica,
4880 cas: PoolMetaCasToken::Unsafe,
4881 }
4882 }
4883}
4884
4885#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4886pub(crate) struct PoolMetaReplicaState {
4887 pub(crate) needs_repair: bool,
4888 pub(crate) repair_write_safe: bool,
4889}
4890
4891impl PoolMetaReplicaState {
4892 pub(crate) fn ensure_write_safe(self, operation: &str) -> Result<()> {
4893 if self.repair_write_safe {
4894 return Ok(());
4895 }
4896 Err(Error::other(format!(
4897 "{operation}: pool metadata update cannot overwrite an unreadable replica"
4898 )))
4899 }
4900}
4901
4902#[derive(Debug, Clone, Default)]
4903pub(crate) struct PoolMetaWriteState {
4904 write_blocked: bool,
4905 aborted_transaction: Arc<AtomicBool>,
4906 block_context: Option<crate::error::PoolMetadataError>,
4907 transaction_failure: Arc<std::sync::Mutex<Option<crate::error::PoolMetadataError>>>,
4908 active_transactions: Arc<AtomicUsize>,
4909 recovery_failure: Option<Arc<Error>>,
4910 block_started_at: Arc<std::sync::Mutex<Option<OffsetDateTime>>>,
4911 expected_cluster_id: Option<uuid::Uuid>,
4912 cluster_epoch: Option<u64>,
4913 pool_meta_absent: bool,
4914 bootstrap_authority: PoolMetaBootstrapAuthority,
4915 pool_bootstrap_authorities: Vec<PoolMetaBootstrapAuthority>,
4920 elected_bootstrap_writer: Option<bool>,
4925 identity_initialized: Option<bool>,
4926 identity_fresh_bootstrap_nonce: Option<uuid::Uuid>,
4927 identity_needs_repair: bool,
4928 identity_replicas_missing: bool,
4930 identity_replicas_invalid: bool,
4932}
4933
4934enum MissingMetadataRejection {
4936 BootstrapPending(Error),
4939 RecoveryRequired(Error),
4942}
4943
4944impl MissingMetadataRejection {
4945 fn into_error(self) -> Error {
4946 match self {
4947 Self::BootstrapPending(err) | Self::RecoveryRequired(err) => err,
4948 }
4949 }
4950}
4951
4952#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
4953pub(crate) enum PoolMetaBootstrapAuthority {
4954 #[default]
4955 None,
4956 Fresh,
4957 LegacyAdoption,
4958}
4959
4960impl PoolMetaBootstrapAuthority {
4961 pub(crate) fn combine_across_pools(self, other: Self) -> Self {
4962 if self == other { self } else { Self::None }
4963 }
4964
4965 fn is_proven(self) -> bool {
4966 !matches!(self, Self::None)
4967 }
4968}
4969
4970impl PoolMetaWriteState {
4971 #[cfg(test)]
4972 pub(crate) fn for_startup(cluster_id: uuid::Uuid, fresh_bootstrap_proven: bool) -> Self {
4973 let bootstrap_authority = if fresh_bootstrap_proven {
4974 PoolMetaBootstrapAuthority::Fresh
4975 } else {
4976 PoolMetaBootstrapAuthority::None
4977 };
4978 Self::for_startup_with_bootstrap_authority(cluster_id, bootstrap_authority)
4979 }
4980
4981 #[cfg(test)]
4982 pub(crate) fn for_startup_with_bootstrap_authority(
4983 cluster_id: uuid::Uuid,
4984 bootstrap_authority: PoolMetaBootstrapAuthority,
4985 ) -> Self {
4986 Self {
4987 expected_cluster_id: Some(cluster_id),
4988 bootstrap_authority,
4989 ..Default::default()
4990 }
4991 }
4992
4993 pub(crate) fn for_startup_with_pool_bootstrap_authorities(
4998 cluster_id: uuid::Uuid,
4999 pool_bootstrap_authorities: Vec<PoolMetaBootstrapAuthority>,
5000 elected_bootstrap_writer: bool,
5001 ) -> Self {
5002 let bootstrap_authority = pool_bootstrap_authorities
5003 .iter()
5004 .copied()
5005 .reduce(PoolMetaBootstrapAuthority::combine_across_pools)
5006 .unwrap_or_default();
5007 Self {
5008 expected_cluster_id: Some(cluster_id),
5009 bootstrap_authority,
5010 pool_bootstrap_authorities,
5011 elected_bootstrap_writer: Some(elected_bootstrap_writer),
5012 ..Default::default()
5013 }
5014 }
5015
5016 pub(crate) fn bootstrap_identity_proven(&self) -> bool {
5017 self.bootstrap_authority.is_proven()
5018 }
5019
5020 fn pool_bootstrap_authority_proven(&self, pool_idx: usize) -> bool {
5021 self.pool_bootstrap_authorities
5022 .get(pool_idx)
5023 .is_some_and(|authority| authority.is_proven())
5024 }
5025
5026 pub(crate) fn attested_pool_indices(&self) -> Vec<usize> {
5028 self.pool_bootstrap_authorities
5029 .iter()
5030 .enumerate()
5031 .filter(|(_, authority)| authority.is_proven())
5032 .map(|(pool_idx, _)| pool_idx)
5033 .collect()
5034 }
5035
5036 fn pending_identity_attested_by_every_pool(&self) -> bool {
5045 self.elected_bootstrap_writer == Some(true)
5046 && self.pool_bootstrap_authority_proven(0)
5047 && self.identity_initialized == Some(false)
5048 && !self.identity_needs_repair
5049 && self.identity_fresh_bootstrap_nonce.is_some()
5050 }
5051
5052 fn awaiting_creator_attestation(&self) -> bool {
5056 self.elected_bootstrap_writer == Some(true)
5057 && self.pool_bootstrap_authority_proven(0)
5058 && self.identity_initialized == Some(false)
5059 && self.identity_fresh_bootstrap_nonce.is_some()
5060 && self.identity_needs_repair
5061 && self.identity_replicas_missing
5062 && !self.identity_replicas_invalid
5063 }
5064
5065 fn is_non_elected_bootstrap_observer(&self) -> bool {
5066 self.elected_bootstrap_writer == Some(false)
5067 }
5068
5069 pub(crate) fn identity_is_pending(&self) -> bool {
5070 self.identity_initialized == Some(false)
5071 }
5072
5073 #[cfg(test)]
5074 pub(crate) fn aborted_transaction_latch_for_test(&self) -> Arc<AtomicBool> {
5075 Arc::clone(&self.aborted_transaction)
5076 }
5077
5078 #[cfg(test)]
5079 pub(crate) fn independent_clone_for_test(&self) -> Self {
5080 let mut cloned = self.clone();
5081 cloned.aborted_transaction = Arc::new(AtomicBool::new(self.aborted_transaction.load(Ordering::Acquire)));
5082 cloned.transaction_failure = Arc::new(std::sync::Mutex::new(
5083 self.transaction_failure
5084 .lock()
5085 .unwrap_or_else(std::sync::PoisonError::into_inner)
5086 .clone(),
5087 ));
5088 cloned.active_transactions = Arc::new(AtomicUsize::new(0));
5089 cloned.block_started_at = Arc::new(std::sync::Mutex::new(
5090 *self
5091 .block_started_at
5092 .lock()
5093 .unwrap_or_else(std::sync::PoisonError::into_inner),
5094 ));
5095 cloned
5096 }
5097
5098 #[cfg(any(test, feature = "test-util"))]
5099 fn for_test_bootstrap() -> Self {
5100 Self {
5101 bootstrap_authority: PoolMetaBootstrapAuthority::Fresh,
5102 identity_initialized: Some(false),
5103 identity_fresh_bootstrap_nonce: Some(uuid::Uuid::new_v4()),
5104 ..Default::default()
5105 }
5106 }
5107
5108 pub(crate) fn observe_replicas(&mut self, replica_state: PoolMetaReplicaState) {
5109 if !replica_state.repair_write_safe {
5110 self.block_writes();
5111 }
5112 }
5113
5114 fn block_writes(&mut self) {
5115 self.block_with_context(pool_metadata_error(
5116 crate::error::PoolMetadataFailure::RecoveryRequired,
5117 "validation",
5118 None,
5119 ));
5120 }
5121
5122 fn block_with_context(&mut self, mut context: crate::error::PoolMetadataError) {
5123 if !self.write_blocked {
5124 record_pool_meta_block_once(&self.block_started_at, &mut context);
5125 self.block_context = Some(context);
5126 } else if let Some(previous) = &self.block_context
5127 && previous.source.is_none()
5128 && context.source.is_some()
5129 {
5130 context.since = previous.since;
5131 self.block_context = Some(context);
5132 }
5133 self.write_blocked = true;
5134 }
5135
5136 pub(crate) fn block_writes_after_fence_loss(&mut self) {
5137 self.block_with_context(pool_metadata_error(crate::error::PoolMetadataFailure::FenceLost, "format_heal", None));
5138 }
5139
5140 fn arm_transaction(&self) -> PoolMetaTransactionArm {
5141 self.active_transactions.fetch_add(1, Ordering::SeqCst);
5142 PoolMetaTransactionArm {
5143 aborted_transaction: Arc::clone(&self.aborted_transaction),
5144 failure: Arc::clone(&self.transaction_failure),
5145 active_transactions: Arc::clone(&self.active_transactions),
5146 block_started_at: Arc::clone(&self.block_started_at),
5147 phase: None,
5148 source: None,
5149 }
5150 }
5151
5152 fn observe_selection(&mut self, selection: &PoolMetaSelection) -> Result<()> {
5153 self.pool_meta_absent = selection.absent;
5154 self.validate_selection(selection)?;
5155 if self.cluster_epoch.is_none()
5156 && let Some((_, metadata_epoch)) = selection.generation_identity
5157 {
5158 self.cluster_epoch = Some(metadata_epoch);
5159 }
5160 Ok(())
5161 }
5162
5163 fn validate_selection(&self, selection: &PoolMetaSelection) -> Result<()> {
5164 if let Some(expected_cluster_id) = self.expected_cluster_id
5165 && let Some((cluster_id, _)) = selection.generation_identity
5166 && cluster_id != expected_cluster_id
5167 {
5168 return Err(Error::other(format!(
5169 "pool metadata incompatible: cluster identity {cluster_id} does not match deployment {expected_cluster_id}"
5170 )));
5171 }
5172 if let Some(identity_epoch) = self.cluster_epoch
5173 && let Some((_, metadata_epoch)) = selection.generation_identity
5174 && metadata_epoch != identity_epoch
5175 {
5176 return Err(Error::other(format!(
5177 "pool metadata recovery required: committed epoch {} does not match cluster identity epoch {identity_epoch}",
5178 metadata_epoch
5179 )));
5180 }
5181 Ok(())
5182 }
5183
5184 fn observe_identity(&mut self, selection: &PoolMetaIdentitySelection) -> Result<()> {
5185 self.identity_needs_repair = selection.needs_repair;
5186 self.identity_initialized = selection.identity.map(|identity| identity.initialized);
5187 self.identity_fresh_bootstrap_nonce = selection.identity.and_then(|identity| identity.fresh_bootstrap_nonce);
5188 self.identity_replicas_missing = selection
5189 .cas_tokens
5190 .iter()
5191 .any(|token| matches!(token, PoolMetaCasToken::Missing));
5192 self.identity_replicas_invalid = selection
5193 .valid_replicas
5194 .iter()
5195 .zip(&selection.cas_tokens)
5196 .any(|(valid, token)| !valid && !matches!(token, PoolMetaCasToken::Missing));
5197 if let Some(identity) = selection.identity {
5198 if identity.initialized {
5199 self.bootstrap_authority = PoolMetaBootstrapAuthority::None;
5200 self.pool_bootstrap_authorities.clear();
5201 }
5202 if let Some(metadata_epoch) = self.cluster_epoch
5203 && metadata_epoch != identity.epoch
5204 {
5205 self.block_writes();
5206 return Err(Error::other(format!(
5207 "pool metadata recovery required: metadata epoch {metadata_epoch} does not match cluster identity epoch {}",
5208 identity.epoch
5209 )));
5210 }
5211 self.cluster_epoch = Some(identity.epoch);
5212 }
5213 if !selection.repair_write_safe {
5214 self.block_writes();
5215 }
5216 Ok(())
5217 }
5218
5219 pub(crate) fn ensure_missing_metadata_can_initialize(&mut self) -> Result<()> {
5220 if !self.pool_meta_absent {
5221 return Ok(());
5222 }
5223 match self.validate_missing_metadata_can_initialize() {
5224 Ok(()) => Ok(()),
5225 Err(MissingMetadataRejection::BootstrapPending(err)) => Err(err),
5226 Err(MissingMetadataRejection::RecoveryRequired(err)) => {
5227 Err(block_pool_meta_validation(self, err, "metadata_absence"))
5228 }
5229 }
5230 }
5231
5232 fn validate_missing_metadata_can_initialize(&self) -> std::result::Result<(), MissingMetadataRejection> {
5233 use MissingMetadataRejection::{BootstrapPending, RecoveryRequired};
5234 match self.identity_initialized {
5235 Some(false)
5236 if self.identity_fresh_bootstrap_nonce.is_some()
5237 && (self.bootstrap_identity_proven() || self.pending_identity_attested_by_every_pool()) =>
5238 {
5239 Ok(())
5240 }
5241 Some(false) if self.awaiting_creator_attestation() => Err(BootstrapPending(Error::other(
5242 "pool metadata bootstrap pending: waiting for every pool creator to attest the pending cluster identity",
5243 ))),
5244 Some(false) if self.is_non_elected_bootstrap_observer() && self.identity_fresh_bootstrap_nonce.is_some() => {
5245 Err(BootstrapPending(Error::other(
5246 "pool metadata bootstrap pending: waiting for the elected writer to publish the initial pool.bin",
5247 )))
5248 }
5249 Some(false) => Err(RecoveryRequired(Error::other(
5250 "pool metadata recovery required: pending cluster identity exists but this startup has no verified fresh-bootstrap proof, legacy-adoption proof, or complete per-pool creator attestation",
5251 ))),
5252 Some(true) => Err(RecoveryRequired(Error::other(
5253 "pool metadata recovery required: initialized cluster identity exists but every pool.bin replica is missing",
5254 ))),
5255 None if self.is_non_elected_bootstrap_observer() => Err(BootstrapPending(Error::other(
5256 "pool metadata bootstrap pending: waiting for the elected writer to establish the cluster identity",
5257 ))),
5258 None => Err(RecoveryRequired(Error::other(
5259 "pool metadata recovery required: no durable bootstrap identity or pool.bin replica is available",
5260 ))),
5261 }
5262 }
5263
5264 pub(crate) fn identity_requires_repair(&self) -> bool {
5265 self.expected_cluster_id.is_some() && (self.identity_needs_repair || self.identity_initialized != Some(true))
5266 }
5267
5268 pub(crate) fn ensure_write_safe(&self, operation: &str) -> Result<()> {
5269 if !self.write_blocked && !self.aborted_transaction.load(Ordering::SeqCst) {
5270 return Ok(());
5271 }
5272 let mut context = self
5273 .block_context
5274 .clone()
5275 .or_else(|| {
5276 self.transaction_failure
5277 .lock()
5278 .unwrap_or_else(std::sync::PoisonError::into_inner)
5279 .clone()
5280 })
5281 .unwrap_or_else(|| pool_metadata_error(crate::error::PoolMetadataFailure::FenceLost, "external_fence", None));
5282 context.operation = operation.to_owned();
5283 Err(Error::other(context))
5284 }
5285}
5286
5287fn pool_metadata_error(
5288 kind: crate::error::PoolMetadataFailure,
5289 phase: &'static str,
5290 source: Option<Arc<Error>>,
5291) -> crate::error::PoolMetadataError {
5292 crate::error::PoolMetadataError {
5293 kind,
5294 operation: phase.to_owned(),
5295 phase,
5296 since: OffsetDateTime::now_utc(),
5297 source,
5298 }
5299}
5300
5301fn record_pool_meta_block_once(
5302 started_at: &std::sync::Mutex<Option<OffsetDateTime>>,
5303 context: &mut crate::error::PoolMetadataError,
5304) {
5305 let mut started_at = started_at.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
5306 if let Some(since) = *started_at {
5307 context.since = since;
5308 return;
5309 }
5310 *started_at = Some(context.since);
5311 error!(
5312 event = EVENT_DECOMMISSION_STATE,
5313 component = LOG_COMPONENT_ECSTORE,
5314 subsystem = LOG_SUBSYSTEM_POOLS,
5315 state = "pool_metadata_blocked",
5316 reason = context.kind.as_str(),
5317 phase = context.phase,
5318 blocked_since = %context.since,
5319 "Pool metadata writes blocked"
5320 );
5321 metrics::counter!("rustfs_pool_metadata_blocks_total", "reason" => context.kind.as_str()).increment(1);
5322}
5323
5324#[derive(Debug)]
5325struct PoolMetaTransactionArm {
5326 aborted_transaction: Arc<AtomicBool>,
5327 failure: Arc<std::sync::Mutex<Option<crate::error::PoolMetadataError>>>,
5328 active_transactions: Arc<AtomicUsize>,
5329 block_started_at: Arc<std::sync::Mutex<Option<OffsetDateTime>>>,
5330 phase: Option<&'static str>,
5331 source: Option<Arc<Error>>,
5332}
5333
5334impl PoolMetaTransactionArm {
5335 fn failure(&mut self, err: Error) -> Error {
5336 let Some(phase) = self.phase else {
5337 return err;
5338 };
5339 let source = Arc::new(err);
5340 self.source = Some(Arc::clone(&source));
5341 Error::other(pool_metadata_error(
5342 crate::error::PoolMetadataFailure::TransactionUnknown,
5343 phase,
5344 Some(source),
5345 ))
5346 }
5347
5348 fn disarm(&mut self) {
5349 self.phase = None;
5350 }
5351}
5352
5353impl Drop for PoolMetaTransactionArm {
5354 fn drop(&mut self) {
5355 if let Some(phase) = self.phase {
5356 let mut context =
5357 pool_metadata_error(crate::error::PoolMetadataFailure::TransactionUnknown, phase, self.source.take());
5358 let mut failure = self.failure.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
5359 if !self.aborted_transaction.swap(true, Ordering::SeqCst) {
5360 record_pool_meta_block_once(&self.block_started_at, &mut context);
5361 *failure = Some(context);
5362 }
5363 }
5364 self.active_transactions.fetch_sub(1, Ordering::SeqCst);
5365 }
5366}
5367
5368#[derive(Debug)]
5369struct PoolMetaSelection {
5370 meta: PoolMeta,
5371 revision: PoolMetaRevision,
5372 canonical: Option<Vec<u8>>,
5373 replica_state: PoolMetaReplicaState,
5374 cas_tokens: Vec<PoolMetaCasToken>,
5375 absent: bool,
5376 generation_protocol_observed: bool,
5377 generation_identity: Option<(uuid::Uuid, u64)>,
5378}
5379
5380fn classify_pool_meta_tuple_decode_error(kind: &str, err: rmp_serde::decode::Error) -> PoolMetaReplica {
5381 let truncated = matches!(
5382 &err,
5383 rmp_serde::decode::Error::InvalidMarkerRead(source)
5384 | rmp_serde::decode::Error::InvalidDataRead(source)
5385 if source.kind() == std::io::ErrorKind::UnexpectedEof
5386 );
5387 if truncated {
5388 PoolMetaReplica::Corrupt(format!("{kind} tuple payload is truncated: {err}"))
5389 } else {
5390 PoolMetaReplica::Incompatible(format!("{kind} tuple payload is not decodable: {err}"))
5391 }
5392}
5393
5394fn parse_pool_meta_uuid(value: &str, field: &str) -> Result<uuid::Uuid> {
5395 let parsed = uuid::Uuid::parse_str(value)
5396 .map_err(|err| Error::other(format!("pool metadata corrupt: invalid {field} `{value}`: {err}")))?;
5397 if parsed.is_nil() || parsed == uuid::Uuid::max() {
5398 return Err(Error::other(format!("pool metadata corrupt: {field} must be a non-reserved UUID")));
5399 }
5400 Ok(parsed)
5401}
5402
5403fn pool_meta_generation_revision(
5404 cluster_id: &str,
5405 epoch: u64,
5406 generation: u64,
5407 transaction_id: &str,
5408) -> Result<PoolMetaRevision> {
5409 if epoch == 0 || generation == 0 {
5410 return Err(Error::other(
5411 "pool metadata corrupt: version 3 epoch and generation must both be non-zero",
5412 ));
5413 }
5414 Ok(PoolMetaRevision {
5415 version: POOL_META_GENERATION_VERSION,
5416 cluster_id: Some(parse_pool_meta_uuid(cluster_id, "cluster identity")?),
5417 epoch,
5418 generation,
5419 transaction_id: Some(parse_pool_meta_uuid(transaction_id, "transaction id")?),
5420 })
5421}
5422
5423fn pool_meta_from_v3_statuses(version: u16, pools: Vec<PersistedPoolStatus>) -> Result<PoolMeta> {
5424 if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) {
5425 return Err(Error::other(format!(
5426 "pool metadata corrupt: version 3 previous snapshot has unsupported source version {version}"
5427 )));
5428 }
5429 let meta = PoolMeta {
5430 version,
5431 pools: pools.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?,
5432 dont_save: false,
5433 };
5434 validate_decommission_capacity_model_cohort(&meta)?;
5435 Ok(meta)
5436}
5437
5438fn pool_meta_previous_candidate(value: PersistedPoolMetaV3Previous) -> Result<PoolMetaCommittedCandidate> {
5439 let revision = match value.version {
5440 POOL_META_V1_VERSION | POOL_META_VERSION => {
5441 if value.cluster_id.is_some() || value.epoch != 0 || value.generation != 0 || value.transaction_id.is_some() {
5442 return Err(Error::other(
5443 "pool metadata corrupt: legacy previous snapshot carries version 3 revision fields",
5444 ));
5445 }
5446 PoolMetaRevision::legacy(value.version)
5447 }
5448 POOL_META_GENERATION_VERSION => pool_meta_generation_revision(
5449 value
5450 .cluster_id
5451 .as_deref()
5452 .ok_or_else(|| Error::other("pool metadata corrupt: version 3 previous snapshot has no cluster identity"))?,
5453 value.epoch,
5454 value.generation,
5455 value
5456 .transaction_id
5457 .as_deref()
5458 .ok_or_else(|| Error::other("pool metadata corrupt: version 3 previous snapshot has no transaction id"))?,
5459 )?,
5460 version => {
5461 return Err(Error::other(format!(
5462 "pool metadata corrupt: previous snapshot has unsupported version {version}"
5463 )));
5464 }
5465 };
5466 let meta = pool_meta_from_v3_statuses(value.version, value.pools)?;
5467 let canonical = if revision.is_generation_protocol() {
5468 encode_pool_meta_v3_envelope(&meta, revision, true, None)?
5469 } else {
5470 meta.encode_config_data_for_v2_gate(true)?
5471 };
5472 Ok(PoolMetaCommittedCandidate {
5473 canonical,
5474 meta,
5475 revision,
5476 })
5477}
5478
5479fn decode_pool_meta_v3(data: Vec<u8>) -> PoolMetaReplica {
5480 let persisted = match rmp_serde::from_slice::<PersistedPoolMetaV3>(&data[4..]) {
5481 Ok(persisted) => persisted,
5482 Err(err) => return classify_pool_meta_tuple_decode_error("v3", err),
5483 };
5484 if persisted.version != POOL_META_GENERATION_VERSION {
5485 return PoolMetaReplica::Corrupt(format!(
5486 "v3 payload has version {}, expected {POOL_META_GENERATION_VERSION}",
5487 persisted.version
5488 ));
5489 }
5490 let revision = match pool_meta_generation_revision(
5491 &persisted.cluster_id,
5492 persisted.epoch,
5493 persisted.generation,
5494 &persisted.transaction_id,
5495 ) {
5496 Ok(revision) => revision,
5497 Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5498 };
5499 let meta = match pool_meta_from_v3_statuses(POOL_META_GENERATION_VERSION, persisted.pools) {
5500 Ok(meta) => meta,
5501 Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5502 };
5503 let previous = match persisted.previous.map(pool_meta_previous_candidate).transpose() {
5504 Ok(previous) => previous,
5505 Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5506 };
5507 if persisted.committed && previous.is_some() {
5508 return PoolMetaReplica::Corrupt("committed v3 payload unexpectedly retains a previous snapshot".to_string());
5509 }
5510 if !persisted.committed {
5511 match previous.as_ref() {
5512 Some(previous) if previous.revision.is_generation_protocol() => {
5513 if previous.revision.cluster_id != revision.cluster_id
5514 || previous.revision.epoch != revision.epoch
5515 || previous.revision.generation.checked_add(1) != Some(revision.generation)
5516 {
5517 return PoolMetaReplica::Corrupt(
5518 "pending v3 payload does not advance exactly one generation from its previous snapshot".to_string(),
5519 );
5520 }
5521 }
5522 Some(_) if revision.generation != 1 => {
5523 return PoolMetaReplica::Corrupt(
5524 "first v3 generation must be generation 1 when migrating a legacy snapshot".to_string(),
5525 );
5526 }
5527 None if revision.generation != 1 => {
5528 return PoolMetaReplica::Corrupt(
5529 "pending v3 payload without a previous snapshot must be the initial generation".to_string(),
5530 );
5531 }
5532 _ => {}
5533 }
5534 }
5535 let canonical = match encode_pool_meta_v3_envelope(&meta, revision, true, None) {
5536 Ok(canonical) => canonical,
5537 Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5538 };
5539 PoolMetaReplica::Valid {
5540 raw: data,
5541 canonical,
5542 meta,
5543 revision,
5544 committed: persisted.committed,
5545 previous: previous.map(Box::new),
5546 }
5547}
5548
5549fn decode_pool_meta_replica(data: Vec<u8>) -> PoolMetaReplica {
5550 if data.len() <= 4 {
5551 return PoolMetaReplica::Corrupt("metadata payload is empty or truncated".to_string());
5552 }
5553
5554 let format = LittleEndian::read_u16(&data[0..2]);
5555 if format != POOL_META_FORMAT {
5556 return PoolMetaReplica::Incompatible(format!("unsupported format {format}"));
5557 }
5558 let version = LittleEndian::read_u16(&data[2..4]);
5559 if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) {
5560 return PoolMetaReplica::Incompatible(format!("unsupported version {version}"));
5561 }
5562 if version == POOL_META_GENERATION_VERSION {
5563 return decode_pool_meta_v3(data);
5564 }
5565
5566 let payload = &data[4..];
5567 let meta = match (version, rmp::decode::read_array_len(&mut &payload[..])) {
5568 (POOL_META_VERSION, Ok(2)) => match rmp_serde::from_slice::<PersistedPoolMeta>(payload) {
5569 Ok(meta) => match PoolMeta::try_from(meta) {
5570 Ok(meta) => meta,
5571 Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5572 },
5573 Err(err) => return classify_pool_meta_tuple_decode_error("current", err),
5574 },
5575 (POOL_META_V1_VERSION, Ok(2)) => match rmp_serde::from_slice::<PersistedPoolMetaV1>(payload) {
5576 Ok(meta) => match PoolMeta::try_from(meta) {
5577 Ok(meta) => meta,
5578 Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5579 },
5580 Err(err) => return classify_pool_meta_tuple_decode_error("v1", err),
5581 },
5582 (POOL_META_V1_VERSION, Ok(3)) => match rmp_serde::from_slice::<LegacyPoolMeta>(payload) {
5584 Ok(meta) => match PoolMeta::try_from(meta) {
5585 Ok(meta) => meta,
5586 Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5587 },
5588 Err(err) => return classify_pool_meta_tuple_decode_error("legacy v1", err),
5589 },
5590 (_, Ok(field_count)) if field_count < 2 => {
5591 return PoolMetaReplica::Corrupt(format!("pool metadata tuple has only {field_count} fields"));
5592 }
5593 (_, Ok(field_count)) => {
5594 return PoolMetaReplica::Incompatible(format!(
5595 "pool metadata version {version} tuple has unsupported field count {field_count}"
5596 ));
5597 }
5598 (_, Err(_)) => {
5599 let mut meta = PoolMeta::default();
5600 if let Err(err) = meta.load_from_config_data(data.clone()) {
5601 let reason = err.to_string();
5602 if reason.contains("unknown field") {
5603 return PoolMetaReplica::Incompatible(format!("current-version payload uses unsupported fields: {reason}"));
5604 }
5605 return PoolMetaReplica::Corrupt(reason);
5606 }
5607 meta
5608 }
5609 };
5610
5611 match meta.encode_config_data_for_v2_gate(true) {
5612 Ok(canonical) => PoolMetaReplica::Valid {
5613 raw: data,
5614 canonical,
5615 meta,
5616 revision: PoolMetaRevision::legacy(version),
5617 committed: true,
5618 previous: None,
5619 },
5620 Err(err) => PoolMetaReplica::Corrupt(err.to_string()),
5621 }
5622}
5623
5624#[cfg(test)]
5625pub(crate) fn pool_meta_v3_commit_state_for_test(data: Vec<u8>) -> Result<(u64, bool)> {
5626 match decode_pool_meta_replica(data) {
5627 PoolMetaReplica::Valid { revision, committed, .. } if revision.is_generation_protocol() => {
5628 Ok((revision.generation, committed))
5629 }
5630 _ => Err(Error::other("test pool metadata is not a valid V3 replica")),
5631 }
5632}
5633
5634async fn read_pool_meta_replica<S>(pool: Arc<S>, no_lock: bool) -> PoolMetaReplicaRead
5635where
5636 S: EcstoreObjectIO,
5637{
5638 let result = if no_lock {
5639 read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME)
5640 .await
5641 .map(|(data, object_info)| (data, object_info.etag))
5642 } else {
5643 read_config_preserve_empty(pool, POOL_META_NAME)
5644 .await
5645 .map(|data| (data, None))
5646 };
5647 match result {
5648 Ok((data, etag)) => PoolMetaReplicaRead {
5649 replica: decode_pool_meta_replica(data),
5650 cas: etag
5651 .filter(|etag| !etag.trim().is_empty())
5652 .map(PoolMetaCasToken::Existing)
5653 .unwrap_or(PoolMetaCasToken::Unsafe),
5654 },
5655 Err(Error::ConfigNotFound) => PoolMetaReplicaRead {
5656 replica: PoolMetaReplica::Missing,
5657 cas: PoolMetaCasToken::Missing,
5658 },
5659 Err(err) => PoolMetaReplicaRead {
5660 replica: PoolMetaReplica::Unreadable(Arc::new(err)),
5661 cas: PoolMetaCasToken::Unsafe,
5662 },
5663 }
5664}
5665
5666fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<PoolMetaSelection> {
5667 if reads.is_empty() {
5668 return Err(Error::other("pool metadata recovery required: no storage pools available"));
5669 }
5670
5671 let cas_tokens = reads.iter().map(|read| read.cas.clone()).collect();
5672 let mut committed = Vec::<(usize, Vec<u8>, PoolMetaCommittedCandidate)>::new();
5673 let mut needs_repair = false;
5674 let mut repair_write_safe = true;
5675 let mut missing = 0usize;
5676 let mut unusable = Vec::new();
5677 let mut observed_version = POOL_META_V1_VERSION;
5678 let mut generation_protocol_observed = false;
5679 let mut generation_identity = None;
5680
5681 for (idx, read) in reads.into_iter().enumerate() {
5682 match read.replica {
5683 PoolMetaReplica::Missing => {
5684 missing += 1;
5685 needs_repair = true;
5686 }
5687 PoolMetaReplica::Corrupt(reason) => {
5688 needs_repair = true;
5689 unusable.push(format!("pool {idx} is corrupt: {reason}"));
5690 }
5691 PoolMetaReplica::Unreadable(reason) => {
5692 needs_repair = true;
5693 repair_write_safe = false;
5694 unusable.push(format!("pool {idx} is unreadable: {reason}"));
5695 }
5696 PoolMetaReplica::Incompatible(reason) => {
5697 return Err(Error::other(format!(
5698 "pool metadata recovery required: pool {idx} is incompatible ({reason}); upgrade or restore a compatible replica without overwriting it"
5699 )));
5700 }
5701 PoolMetaReplica::Valid {
5702 raw,
5703 canonical,
5704 meta,
5705 revision,
5706 committed: is_committed,
5707 previous,
5708 } => {
5709 generation_protocol_observed |= revision.is_generation_protocol();
5710 if revision.is_generation_protocol() {
5711 let identity = (
5712 revision
5713 .cluster_id
5714 .expect("validated V3 revision should carry a cluster identity"),
5715 revision.epoch,
5716 );
5717 if generation_identity.is_some_and(|current| current != identity) {
5718 return Err(Error::other(
5719 "pool metadata recovery required: V3 replicas disagree on cluster identity or epoch",
5720 ));
5721 }
5722 generation_identity = Some(identity);
5723 }
5724 if is_committed {
5725 observed_version = observed_version.max(meta.version);
5726 committed.push((
5727 idx,
5728 raw,
5729 PoolMetaCommittedCandidate {
5730 canonical,
5731 meta,
5732 revision,
5733 },
5734 ));
5735 } else if let Some(previous) = previous {
5736 observed_version = observed_version.max(previous.meta.version);
5737 needs_repair = true;
5738 committed.push((idx, raw, *previous));
5739 } else {
5740 needs_repair = true;
5741 unusable.push(format!("pool {idx} contains an uncommitted initial generation"));
5742 }
5743 }
5744 }
5745 }
5746
5747 if committed.is_empty() && missing > 0 && unusable.is_empty() {
5748 return Ok(PoolMetaSelection {
5749 meta: PoolMeta::default(),
5750 revision: PoolMetaRevision::legacy(0),
5751 canonical: None,
5752 replica_state: PoolMetaReplicaState {
5753 needs_repair: false,
5754 repair_write_safe: true,
5755 },
5756 cas_tokens,
5757 absent: true,
5758 generation_protocol_observed,
5759 generation_identity,
5760 });
5761 }
5762 if committed.is_empty() {
5763 return Err(Error::other(format!(
5764 "pool metadata recovery required: no valid committed replica is available ({})",
5765 unusable.join("; ")
5766 )));
5767 }
5768
5769 if committed
5770 .iter()
5771 .any(|(_, _, candidate)| candidate.revision.is_generation_protocol())
5772 {
5773 let highest = committed
5774 .iter()
5775 .filter(|(_, _, candidate)| candidate.revision.is_generation_protocol())
5776 .map(|(_, _, candidate)| candidate.revision.generation)
5777 .max()
5778 .ok_or_else(|| Error::other("pool metadata recovery required: no committed V3 generation is available"))?;
5779 let mut selected: Option<(usize, &PoolMetaCommittedCandidate)> = None;
5780 for (idx, _, candidate) in &committed {
5781 if !candidate.revision.is_generation_protocol() || candidate.revision.generation != highest {
5782 needs_repair = true;
5783 continue;
5784 }
5785 if let Some((selected_idx, selected_candidate)) = selected {
5786 if selected_candidate.canonical != candidate.canonical
5787 || selected_candidate.revision.transaction_id != candidate.revision.transaction_id
5788 {
5789 return Err(Error::other(format!(
5790 "pool metadata recovery required: committed generation {highest} diverges between pools {selected_idx} and {idx}"
5791 )));
5792 }
5793 } else {
5794 selected = Some((*idx, candidate));
5795 }
5796 }
5797 let (_, selected) = selected
5798 .ok_or_else(|| Error::other("pool metadata recovery required: highest V3 generation has no valid snapshot"))?;
5799 for (_, raw, candidate) in &committed {
5800 needs_repair |= candidate.canonical != selected.canonical || raw != &selected.canonical;
5801 }
5802 return Ok(PoolMetaSelection {
5803 meta: selected.meta.clone(),
5804 revision: selected.revision,
5805 canonical: Some(selected.canonical.clone()),
5806 replica_state: PoolMetaReplicaState {
5807 needs_repair,
5808 repair_write_safe,
5809 },
5810 cas_tokens,
5811 absent: false,
5812 generation_protocol_observed,
5813 generation_identity,
5814 });
5815 }
5816
5817 let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
5820 for (idx, raw, candidate) in committed {
5821 if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
5822 if selected_candidate.canonical != candidate.canonical {
5823 if selected_candidate.meta.version == candidate.meta.version && *selected_idx == 0 {
5824 needs_repair = true;
5825 } else {
5826 return Err(Error::other(format!(
5827 "pool metadata recovery required: valid replicas in pools {selected_idx} and {idx} diverge; restore one matching pool.bin snapshot before restart"
5828 )));
5829 }
5830 } else {
5831 needs_repair |= selected_raw != &raw;
5832 }
5833 } else {
5834 selected = Some((idx, raw, candidate));
5835 }
5836 }
5837 let (_, _, mut selected) =
5838 selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
5839 selected.meta.version = observed_version;
5840 selected.revision.version = observed_version;
5841 Ok(PoolMetaSelection {
5842 meta: selected.meta,
5843 revision: selected.revision,
5844 canonical: Some(selected.canonical),
5845 replica_state: PoolMetaReplicaState {
5846 needs_repair,
5847 repair_write_safe,
5848 },
5849 cas_tokens,
5850 absent: false,
5851 generation_protocol_observed,
5852 generation_identity,
5853 })
5854}
5855
5856#[cfg(test)]
5857fn select_pool_meta_replica(replicas: Vec<PoolMetaReplica>) -> Result<PoolMetaSelection> {
5858 select_pool_meta_replica_reads(
5859 replicas
5860 .into_iter()
5861 .map(|replica| PoolMetaReplicaRead {
5862 replica,
5863 cas: PoolMetaCasToken::Unsafe,
5864 })
5865 .collect(),
5866 )
5867}
5868
5869async fn read_pool_meta_replicas<S>(pools: Vec<Arc<S>>, no_lock: bool) -> Vec<PoolMetaReplicaRead>
5870where
5871 S: EcstoreObjectIO,
5872{
5873 let reads = join_all(pools.into_iter().map(|pool| read_pool_meta_replica(pool, no_lock))).await;
5874 #[cfg(feature = "e2e-test-hooks")]
5875 if STARTUP_CAS_OBSERVATION.try_with(|_| ()).is_ok() {
5876 let batch = uuid::Uuid::new_v4();
5877 for (pool, read) in reads.iter().enumerate() {
5878 let mut observation = serde_json::json!({
5879 "kind": "replica-read", "object": POOL_META_NAME, "batch": batch, "pool": pool,
5880 "cas": match &read.cas {
5881 PoolMetaCasToken::Missing => "missing",
5882 PoolMetaCasToken::Existing(_) => "existing",
5883 PoolMetaCasToken::Unsafe => "unsafe",
5884 },
5885 "etag": match &read.cas { PoolMetaCasToken::Existing(etag) => Some(etag), _ => None },
5886 });
5887 match &read.replica {
5888 PoolMetaReplica::Valid {
5889 raw,
5890 canonical,
5891 meta,
5892 revision,
5893 committed,
5894 previous,
5895 } => {
5896 observation["state"] = serde_json::json!("valid");
5897 observation["committed"] = serde_json::json!(committed);
5898 observation["version"] = serde_json::json!(revision.version);
5899 observation["cluster_id"] = serde_json::json!(revision.cluster_id);
5900 observation["epoch"] = serde_json::json!(revision.epoch);
5901 observation["generation"] = serde_json::json!(revision.generation);
5902 observation["transaction_id"] = serde_json::json!(revision.transaction_id);
5903 observation["pool_count"] = serde_json::json!(meta.pools.len());
5904 observation["payload_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(canonical)));
5905 observation["raw_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(raw)));
5906 match pool_meta_test_pools_sha256(meta) {
5907 Ok(digest) => observation["persisted_pools_sha256"] = serde_json::json!(digest),
5908 Err(err) => observation["observation_error"] = serde_json::json!(err.to_string()),
5909 }
5910 if let Some(previous) = previous {
5911 match pool_meta_test_snapshot(&previous.meta, previous.revision, &previous.canonical) {
5912 Ok(summary) => observation["previous"] = summary,
5913 Err(err) => observation["observation_error"] = serde_json::json!(err.to_string()),
5914 }
5915 }
5916 }
5917 PoolMetaReplica::Missing => observation["state"] = serde_json::json!("missing"),
5918 PoolMetaReplica::Corrupt(_) => observation["state"] = serde_json::json!("corrupt"),
5919 PoolMetaReplica::Incompatible(_) => observation["state"] = serde_json::json!("incompatible"),
5920 PoolMetaReplica::Unreadable(_) => observation["state"] = serde_json::json!("unreadable"),
5921 }
5922 startup_cas_test_observe(observation);
5923 }
5924 }
5925 reads
5926}
5927
5928fn select_pool_meta_replicas_observing<R>(write_state: &mut PoolMetaWriteState, replicas: Vec<R>) -> Result<PoolMetaSelection>
5929where
5930 R: Into<PoolMetaReplicaRead>,
5931{
5932 let replicas = replicas.into_iter().map(Into::into).collect::<Vec<_>>();
5933 if let Some(err) = pool_meta_replica_read_failure(&replicas, "pool_read") {
5934 return Err(err);
5935 }
5936 match select_pool_meta_replica_reads(replicas) {
5937 Ok(selection) => {
5938 if let Err(err) = write_state.observe_selection(&selection) {
5939 return Err(block_pool_meta_validation(write_state, err, "pool_selection"));
5940 }
5941 Ok(selection)
5942 }
5943 Err(err) => Err(block_pool_meta_validation(write_state, err, "pool_selection")),
5944 }
5945}
5946
5947fn block_pool_meta_validation(write_state: &mut PoolMetaWriteState, err: Error, phase: &'static str) -> Error {
5948 let context = pool_metadata_error(crate::error::PoolMetadataFailure::RecoveryRequired, phase, Some(Arc::new(err)));
5949 write_state.block_with_context(context.clone());
5950 Error::other(context)
5951}
5952
5953fn pool_meta_replica_read_failure(replicas: &[PoolMetaReplicaRead], operation: &str) -> Option<Error> {
5954 replicas.iter().find_map(|read| match &read.replica {
5955 PoolMetaReplica::Unreadable(source) => {
5956 let mut context =
5957 pool_metadata_error(crate::error::PoolMetadataFailure::ReadUnavailable, "pool_read", Some(Arc::clone(source)));
5958 context.operation = operation.to_owned();
5959 Some(Error::other(context))
5960 }
5961 _ => None,
5962 })
5963}
5964
5965fn select_pool_meta_replicas_for_read_probe<R>(
5966 write_state: &PoolMetaWriteState,
5967 replicas: Vec<R>,
5968 operation: &str,
5969) -> Result<PoolMetaSelection>
5970where
5971 R: Into<PoolMetaReplicaRead>,
5972{
5973 let replicas = replicas.into_iter().map(Into::into).collect::<Vec<_>>();
5974 if let Some(err) = pool_meta_replica_read_failure(&replicas, operation) {
5975 return Err(err);
5976 }
5977 let selection = select_pool_meta_replica_reads(replicas)?;
5978 write_state.validate_selection(&selection)?;
5979 selection.replica_state.ensure_write_safe(operation)?;
5980 if selection.absent && (write_state.expected_cluster_id.is_some() || write_state.identity_initialized.is_some()) {
5981 write_state
5982 .validate_missing_metadata_can_initialize()
5983 .map_err(MissingMetadataRejection::into_error)?;
5984 }
5985 Ok(selection)
5986}
5987
5988async fn load_pool_meta_replicas<S>(pools: Vec<Arc<S>>, no_lock: bool) -> Result<PoolMetaSelection>
5989where
5990 S: EcstoreObjectIO,
5991{
5992 select_pool_meta_replica_reads(read_pool_meta_replicas(pools, no_lock).await)
5993}
5994
5995async fn load_pool_meta_replicas_observing<S>(
5996 pools: Vec<Arc<S>>,
5997 no_lock: bool,
5998 write_state: &mut PoolMetaWriteState,
5999) -> Result<PoolMetaSelection>
6000where
6001 S: EcstoreObjectIO,
6002{
6003 let replicas = read_pool_meta_replicas(pools, no_lock).await;
6004 select_pool_meta_replicas_observing(write_state, replicas)
6005}
6006
6007async fn load_pool_meta_replicas_for_read_probe<S>(
6008 pools: Vec<Arc<S>>,
6009 no_lock: bool,
6010 write_state: &PoolMetaWriteState,
6011 operation: &str,
6012) -> Result<PoolMetaSelection>
6013where
6014 S: EcstoreObjectIO,
6015{
6016 let replicas = read_pool_meta_replicas(pools, no_lock).await;
6017 select_pool_meta_replicas_for_read_probe(write_state, replicas, operation).map_err(|err| {
6018 if err.pool_metadata_failure().is_some() {
6019 err
6020 } else {
6021 Error::other(pool_metadata_error(
6022 crate::error::PoolMetadataFailure::RecoveryRequired,
6023 "read_probe",
6024 Some(Arc::new(err)),
6025 ))
6026 }
6027 })
6028}
6029
6030#[derive(Debug, Clone, Serialize, Deserialize)]
6031#[serde(deny_unknown_fields)]
6032struct PersistedPoolMetaV3 {
6033 version: u16,
6034 cluster_id: String,
6035 epoch: u64,
6036 generation: u64,
6037 transaction_id: String,
6038 committed: bool,
6039 pools: Vec<PersistedPoolStatus>,
6040 previous: Option<PersistedPoolMetaV3Previous>,
6041}
6042
6043#[derive(Debug, Clone, Serialize, Deserialize)]
6044#[serde(deny_unknown_fields)]
6045struct PersistedPoolMetaV3Previous {
6046 version: u16,
6047 cluster_id: Option<String>,
6048 epoch: u64,
6049 generation: u64,
6050 transaction_id: Option<String>,
6051 pools: Vec<PersistedPoolStatus>,
6052}
6053
6054#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
6055#[serde(deny_unknown_fields)]
6056struct PersistedPoolMetaIdentity {
6057 version: u16,
6058 cluster_id: uuid::Uuid,
6059 epoch: u64,
6060 initialized: bool,
6061 fresh_bootstrap_nonce: Option<uuid::Uuid>,
6062}
6063
6064#[derive(Debug)]
6065enum PoolMetaIdentityReplica {
6066 Missing,
6067 Valid(PersistedPoolMetaIdentity),
6068 Corrupt(String),
6069 Incompatible(String),
6070 Unreadable(Arc<Error>),
6071}
6072
6073#[derive(Debug)]
6074struct PoolMetaIdentityRead {
6075 replica: PoolMetaIdentityReplica,
6076 cas: PoolMetaCasToken,
6077}
6078
6079#[derive(Debug)]
6080struct PoolMetaIdentitySelection {
6081 identity: Option<PersistedPoolMetaIdentity>,
6082 needs_repair: bool,
6083 repair_write_safe: bool,
6084 cas_tokens: Vec<PoolMetaCasToken>,
6085 valid_replicas: Vec<bool>,
6087}
6088
6089fn encode_pool_meta_identity(identity: PersistedPoolMetaIdentity) -> Result<Vec<u8>> {
6090 let mut data = Vec::new();
6091 data.write_u16::<LittleEndian>(POOL_META_IDENTITY_FORMAT)?;
6092 data.write_u16::<LittleEndian>(POOL_META_IDENTITY_VERSION)?;
6093 identity.serialize(&mut Serializer::new(&mut data))?;
6094 Ok(data)
6095}
6096
6097fn decode_pool_meta_identity(data: &[u8]) -> PoolMetaIdentityReplica {
6098 if data.len() <= 4 {
6099 return PoolMetaIdentityReplica::Corrupt("identity payload is empty or truncated".to_string());
6100 }
6101 let format = LittleEndian::read_u16(&data[0..2]);
6102 let version = LittleEndian::read_u16(&data[2..4]);
6103 if format != POOL_META_IDENTITY_FORMAT || version != POOL_META_IDENTITY_VERSION {
6104 return PoolMetaIdentityReplica::Incompatible(format!("unsupported identity format {format} version {version}"));
6105 }
6106 let identity = match rmp_serde::from_slice::<PersistedPoolMetaIdentity>(&data[4..]) {
6107 Ok(identity) => identity,
6108 Err(err) => return PoolMetaIdentityReplica::Corrupt(format!("identity payload is not decodable: {err}")),
6109 };
6110 let invalid_bootstrap_nonce = identity
6111 .fresh_bootstrap_nonce
6112 .is_some_and(|nonce| nonce.is_nil() || nonce == uuid::Uuid::max());
6113 if identity.version != POOL_META_IDENTITY_VERSION
6114 || identity.cluster_id.is_nil()
6115 || identity.cluster_id == uuid::Uuid::max()
6116 || identity.epoch == 0
6117 || invalid_bootstrap_nonce
6118 || identity.initialized == identity.fresh_bootstrap_nonce.is_some()
6119 {
6120 return PoolMetaIdentityReplica::Corrupt(
6121 "identity payload contains an invalid version, UUID, epoch, or fresh-bootstrap proof".to_string(),
6122 );
6123 }
6124 PoolMetaIdentityReplica::Valid(identity)
6125}
6126
6127#[cfg(test)]
6128pub(crate) fn pool_meta_identity_initialized_for_test(data: &[u8]) -> Result<bool> {
6129 match decode_pool_meta_identity(data) {
6130 PoolMetaIdentityReplica::Valid(identity) => Ok(identity.initialized),
6131 _ => Err(Error::other("test pool metadata identity is not valid")),
6132 }
6133}
6134
6135#[cfg(test)]
6136pub(crate) fn pending_pool_meta_identity_for_test(cluster_id: uuid::Uuid, epoch: u64, nonce: uuid::Uuid) -> Result<Vec<u8>> {
6137 encode_pool_meta_identity(PersistedPoolMetaIdentity {
6138 version: POOL_META_IDENTITY_VERSION,
6139 cluster_id,
6140 epoch,
6141 initialized: false,
6142 fresh_bootstrap_nonce: Some(nonce),
6143 })
6144}
6145
6146#[cfg(test)]
6147pub(crate) fn initialized_pool_meta_identity_for_test(cluster_id: uuid::Uuid, epoch: u64) -> Result<Vec<u8>> {
6148 encode_pool_meta_identity(PersistedPoolMetaIdentity {
6149 version: POOL_META_IDENTITY_VERSION,
6150 cluster_id,
6151 epoch,
6152 initialized: true,
6153 fresh_bootstrap_nonce: None,
6154 })
6155}
6156
6157async fn read_pool_meta_identity_replica<S>(pool: Arc<S>) -> PoolMetaIdentityRead
6158where
6159 S: EcstoreObjectIO,
6160{
6161 match read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_IDENTITY_NAME).await {
6162 Ok((data, object_info)) => PoolMetaIdentityRead {
6163 replica: decode_pool_meta_identity(&data),
6164 cas: object_info
6165 .etag
6166 .filter(|etag| !etag.trim().is_empty())
6167 .map(PoolMetaCasToken::Existing)
6168 .unwrap_or(PoolMetaCasToken::Unsafe),
6169 },
6170 Err(Error::ConfigNotFound) => PoolMetaIdentityRead {
6171 replica: PoolMetaIdentityReplica::Missing,
6172 cas: PoolMetaCasToken::Missing,
6173 },
6174 Err(err) => PoolMetaIdentityRead {
6175 replica: PoolMetaIdentityReplica::Unreadable(Arc::new(err)),
6176 cas: PoolMetaCasToken::Unsafe,
6177 },
6178 }
6179}
6180
6181fn select_pool_meta_identity(
6182 reads: Vec<PoolMetaIdentityRead>,
6183 expected_cluster_id: uuid::Uuid,
6184) -> Result<PoolMetaIdentitySelection> {
6185 let cas_tokens = reads.iter().map(|read| read.cas.clone()).collect();
6186 let valid_replicas = reads
6187 .iter()
6188 .map(|read| matches!(read.replica, PoolMetaIdentityReplica::Valid(_)))
6189 .collect();
6190 let mut selected: Option<PersistedPoolMetaIdentity> = None;
6191 let mut needs_repair = false;
6192 let mut repair_write_safe = true;
6193 let mut unusable = Vec::new();
6194 for (idx, read) in reads.into_iter().enumerate() {
6195 match read.replica {
6196 PoolMetaIdentityReplica::Missing => needs_repair = true,
6197 PoolMetaIdentityReplica::Corrupt(reason) => {
6198 needs_repair = true;
6199 unusable.push(format!("pool {idx} identity is corrupt: {reason}"));
6200 }
6201 PoolMetaIdentityReplica::Unreadable(reason) => {
6202 needs_repair = true;
6203 repair_write_safe = false;
6204 unusable.push(format!("pool {idx} identity is unreadable: {reason}"));
6205 }
6206 PoolMetaIdentityReplica::Incompatible(reason) => {
6207 return Err(Error::other(format!("pool metadata incompatible: pool {idx} identity uses {reason}")));
6208 }
6209 PoolMetaIdentityReplica::Valid(identity) => {
6210 if identity.cluster_id != expected_cluster_id {
6211 return Err(Error::other(format!(
6212 "pool metadata incompatible: pool {idx} identity {} does not match deployment {expected_cluster_id}",
6213 identity.cluster_id
6214 )));
6215 }
6216 if let Some(current) = selected {
6217 if current.cluster_id != identity.cluster_id || current.epoch != identity.epoch {
6218 return Err(Error::other(
6219 "pool metadata recovery required: identity replicas disagree on cluster identity or epoch",
6220 ));
6221 }
6222 if current.initialized != identity.initialized {
6223 needs_repair = true;
6224 selected = Some(if current.initialized { current } else { identity });
6225 } else if !current.initialized && current.fresh_bootstrap_nonce != identity.fresh_bootstrap_nonce {
6226 return Err(Error::other(
6227 "pool metadata recovery required: pending identity replicas disagree on fresh-bootstrap proof",
6228 ));
6229 }
6230 } else {
6231 selected = Some(identity);
6232 }
6233 }
6234 }
6235 }
6236 if selected.is_none() && !unusable.is_empty() {
6237 return Err(Error::other(format!(
6238 "pool metadata recovery required: no valid cluster identity replica is available ({})",
6239 unusable.join("; ")
6240 )));
6241 }
6242 Ok(PoolMetaIdentitySelection {
6243 identity: selected,
6244 needs_repair,
6245 repair_write_safe,
6246 cas_tokens,
6247 valid_replicas,
6248 })
6249}
6250
6251async fn load_pool_meta_identity<S>(pools: Vec<Arc<S>>, expected_cluster_id: uuid::Uuid) -> Result<PoolMetaIdentitySelection>
6252where
6253 S: EcstoreObjectIO,
6254{
6255 let reads = join_all(pools.into_iter().map(read_pool_meta_identity_replica)).await;
6256 if let Some(source) = reads.iter().find_map(|read| match &read.replica {
6257 PoolMetaIdentityReplica::Unreadable(source) => Some(Arc::clone(source)),
6258 _ => None,
6259 }) {
6260 return Err(Error::other(pool_metadata_error(
6261 crate::error::PoolMetadataFailure::ReadUnavailable,
6262 "identity_read",
6263 Some(source),
6264 )));
6265 }
6266 select_pool_meta_identity(reads, expected_cluster_id)
6267}
6268
6269async fn load_pool_meta_identity_selection_observing<S>(
6270 pools: Vec<Arc<S>>,
6271 write_state: &mut PoolMetaWriteState,
6272 expected_cluster_id: uuid::Uuid,
6273) -> Result<PoolMetaIdentitySelection>
6274where
6275 S: EcstoreObjectIO,
6276{
6277 let selection = match load_pool_meta_identity(pools, expected_cluster_id).await {
6278 Ok(selection) => selection,
6279 Err(err) => {
6280 if err
6281 .pool_metadata_failure()
6282 .is_some_and(|context| context.kind == crate::error::PoolMetadataFailure::ReadUnavailable)
6283 {
6284 return Err(err);
6285 }
6286 return Err(block_pool_meta_validation(write_state, err, "identity_selection"));
6287 }
6288 };
6289 if let Err(err) = write_state.observe_identity(&selection) {
6290 return Err(block_pool_meta_validation(write_state, err, "identity_selection"));
6291 }
6292 Ok(selection)
6293}
6294
6295fn persisted_pool_meta_v3_previous(candidate: &PoolMetaCommittedCandidate) -> PersistedPoolMetaV3Previous {
6296 PersistedPoolMetaV3Previous {
6297 version: candidate.revision.version,
6298 cluster_id: candidate.revision.cluster_id.map(|id| id.to_string()),
6299 epoch: candidate.revision.epoch,
6300 generation: candidate.revision.generation,
6301 transaction_id: candidate.revision.transaction_id.map(|id| id.to_string()),
6302 pools: candidate.meta.pools.iter().map(Into::into).collect(),
6303 }
6304}
6305
6306fn encode_pool_meta_v3_envelope(
6307 meta: &PoolMeta,
6308 revision: PoolMetaRevision,
6309 committed: bool,
6310 previous: Option<&PoolMetaCommittedCandidate>,
6311) -> Result<Vec<u8>> {
6312 if meta.dont_save {
6313 return Ok(Vec::new());
6314 }
6315 let cluster_id = revision
6316 .cluster_id
6317 .ok_or_else(|| Error::other("pool metadata V3 save failed: cluster identity is not initialized"))?;
6318 let transaction_id = revision
6319 .transaction_id
6320 .ok_or_else(|| Error::other("pool metadata V3 save failed: transaction id is not initialized"))?;
6321 if revision.version != POOL_META_GENERATION_VERSION || revision.epoch == 0 || revision.generation == 0 {
6322 return Err(Error::other("pool metadata V3 save failed: invalid durable revision"));
6323 }
6324 let persisted = PersistedPoolMetaV3 {
6325 version: POOL_META_GENERATION_VERSION,
6326 cluster_id: cluster_id.to_string(),
6327 epoch: revision.epoch,
6328 generation: revision.generation,
6329 transaction_id: transaction_id.to_string(),
6330 committed,
6331 pools: meta.pools.iter().map(Into::into).collect(),
6332 previous: previous.map(persisted_pool_meta_v3_previous),
6333 };
6334 let mut data = Vec::new();
6335 data.write_u16::<LittleEndian>(POOL_META_FORMAT)?;
6336 data.write_u16::<LittleEndian>(POOL_META_GENERATION_VERSION)?;
6337 persisted.serialize(&mut Serializer::new(&mut data))?;
6338 Ok(data)
6339}
6340
6341enum PoolMetaPersistenceFence<'a> {
6342 Distributed(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>),
6343 Activation(&'a PoolRebalanceActivationFence),
6344}
6345
6346impl PoolMetaPersistenceFence<'_> {
6347 fn ensure_held(&self) -> Result<()> {
6348 match self {
6349 Self::Distributed(Some(signal)) if signal.is_lost() => {
6350 Err(Error::other("pool metadata distributed fence was lost before a replica write"))
6351 }
6352 Self::Activation(fence) => fence.ensure_held(),
6353 _ => Ok(()),
6354 }
6355 }
6356
6357 fn add_to_options(&self, opts: &mut ObjectOptions) {
6358 match self {
6359 Self::Distributed(Some(signal)) => opts.add_namespace_lock_lost_signal(Arc::clone(signal)),
6360 Self::Activation(fence) => fence.add_namespace_lock_fence(opts),
6361 Self::Distributed(None) => {}
6362 }
6363 }
6364
6365 fn is_activation(&self) -> bool {
6366 matches!(self, Self::Activation(_))
6367 }
6368}
6369
6370fn pool_meta_cas_preconditions(token: &PoolMetaCasToken, object: &str) -> Result<HTTPPreconditions> {
6371 match token {
6372 PoolMetaCasToken::Missing => Ok(HTTPPreconditions {
6373 if_none_match: Some("*".to_string()),
6374 ..Default::default()
6375 }),
6376 PoolMetaCasToken::Existing(etag) => Ok(HTTPPreconditions {
6377 if_match: Some(etag.clone()),
6378 ..Default::default()
6379 }),
6380 PoolMetaCasToken::Unsafe => Err(Error::other(format!(
6381 "pool metadata recovery required: {object} replica has no safe conditional-write revision"
6382 ))),
6383 }
6384}
6385
6386#[cfg(feature = "e2e-test-hooks")]
6387struct StartupCasObservation {
6388 attempt: uuid::Uuid,
6389 phase: &'static str,
6390 pools: Vec<usize>,
6391}
6392
6393#[cfg(feature = "e2e-test-hooks")]
6394tokio::task_local! {
6395 static STARTUP_CAS_OBSERVATION: StartupCasObservation;
6396}
6397
6398#[cfg(feature = "e2e-test-hooks")]
6401pub(crate) async fn startup_cas_test_scope<S, F: std::future::Future>(
6402 attempt: uuid::Uuid,
6403 phase: &'static str,
6404 pools: &[Arc<S>],
6405 future: F,
6406) -> F::Output {
6407 STARTUP_CAS_OBSERVATION
6408 .scope(
6409 StartupCasObservation {
6410 attempt,
6411 phase,
6412 pools: pools.iter().map(|pool| Arc::as_ptr(pool) as usize).collect(),
6415 },
6416 future,
6417 )
6418 .await
6419}
6420
6421#[cfg(feature = "e2e-test-hooks")]
6423pub(crate) fn startup_cas_test_observe(mut observation: serde_json::Value) {
6424 let Some(nonce) = std::env::var("RUSTFS_E2E_STARTUP_CAS_NONCE")
6425 .ok()
6426 .and_then(|value| uuid::Uuid::parse_str(&value).ok())
6427 else {
6428 return;
6429 };
6430 observation["nonce"] = serde_json::json!(nonce);
6431 observation["pid"] = serde_json::json!(std::process::id());
6432 let _ = STARTUP_CAS_OBSERVATION.try_with(|scope| {
6433 observation["attempt"] = serde_json::json!(scope.attempt);
6434 observation["startup_phase"] = serde_json::json!(scope.phase);
6435 });
6436 let line = format!("RUSTFS_E2E_STARTUP_CAS {observation}\n");
6437 let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes());
6438}
6439
6440#[cfg(feature = "e2e-test-hooks")]
6441fn pool_meta_test_pools_sha256(meta: &PoolMeta) -> Result<String> {
6442 let pools = meta.pools.iter().map(PersistedPoolStatus::from).collect::<Vec<_>>();
6443 Ok(rustfs_utils::crypto::hex(Sha256::digest(serde_json::to_vec(&pools)?)))
6444}
6445
6446#[cfg(feature = "e2e-test-hooks")]
6447fn pool_meta_test_snapshot(meta: &PoolMeta, revision: PoolMetaRevision, canonical: &[u8]) -> Result<serde_json::Value> {
6448 Ok(serde_json::json!({
6449 "version": revision.version, "cluster_id": revision.cluster_id, "epoch": revision.epoch,
6450 "generation": revision.generation, "transaction_id": revision.transaction_id,
6451 "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(canonical)),
6452 "persisted_pools_sha256": pool_meta_test_pools_sha256(meta)?,
6453 }))
6454}
6455
6456#[cfg(feature = "e2e-test-hooks")]
6457#[derive(Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
6458#[serde(rename_all = "snake_case")]
6459enum PoolMetaPhaseBarrierCase {
6460 PrepareSubset,
6461 PreparedAll,
6462 CommitOne,
6463 BeforePublish,
6464}
6465
6466#[cfg(feature = "e2e-test-hooks")]
6467#[derive(Deserialize)]
6468#[serde(deny_unknown_fields)]
6469struct PoolMetaPhaseBarrierArm {
6470 nonce: uuid::Uuid,
6471 case: PoolMetaPhaseBarrierCase,
6472}
6473
6474#[cfg(feature = "e2e-test-hooks")]
6475struct PoolMetaPhaseBarrier {
6476 directory: std::path::PathBuf,
6477 arm: PoolMetaPhaseBarrierArm,
6478 transaction_id: uuid::Uuid,
6479 event_gate: tokio::sync::Mutex<()>,
6480 blocked_pools: AtomicUsize,
6481 completed_writes: AtomicUsize,
6482 ready_emitted: AtomicBool,
6483}
6484
6485#[cfg(feature = "e2e-test-hooks")]
6486impl PoolMetaPhaseBarrier {
6487 async fn bind(
6488 previous: &PoolMetaCommittedCandidate,
6489 candidate: &PoolMeta,
6490 revision: PoolMetaRevision,
6491 durable: &[u8],
6492 ) -> Result<Option<Self>> {
6493 let Some(directory) = std::env::var_os("RUSTFS_E2E_POOL_META_BARRIER_DIR").map(std::path::PathBuf::from) else {
6494 return Ok(None);
6495 };
6496 Self::bind_in_directory(directory, previous, candidate, revision, durable).await
6497 }
6498
6499 async fn bind_in_directory(
6500 directory: std::path::PathBuf,
6501 previous: &PoolMetaCommittedCandidate,
6502 candidate: &PoolMeta,
6503 revision: PoolMetaRevision,
6504 durable: &[u8],
6505 ) -> Result<Option<Self>> {
6506 let data = match tokio::fs::read(directory.join("arm.json")).await {
6507 Ok(data) => data,
6508 Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
6509 Err(err) => return Err(err.into()),
6510 };
6511 let arm: PoolMetaPhaseBarrierArm = serde_json::from_slice(&data)?;
6512 if arm.nonce.is_nil() || !directory.is_absolute() {
6513 return Err(Error::other(
6514 "pool metadata test barrier requires an absolute directory and non-nil nonce",
6515 ));
6516 }
6517 let transaction_id = revision
6518 .transaction_id
6519 .ok_or_else(|| Error::other("pool metadata test barrier requires a V3 transaction"))?;
6520 if pool_meta_test_pools_sha256(&previous.meta)? == pool_meta_test_pools_sha256(candidate)? {
6523 return Ok(None);
6524 }
6525 let claim = tokio::fs::OpenOptions::new()
6527 .write(true)
6528 .create_new(true)
6529 .open(directory.join("claimed"))
6530 .await;
6531 match claim {
6532 Ok(_) => {}
6533 Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => return Ok(None),
6534 Err(err) => return Err(err.into()),
6535 }
6536 let barrier = Self {
6537 directory,
6538 arm,
6539 transaction_id,
6540 event_gate: tokio::sync::Mutex::new(()),
6541 blocked_pools: AtomicUsize::new(0),
6542 completed_writes: AtomicUsize::new(0),
6543 ready_emitted: AtomicBool::new(false),
6544 };
6545 barrier
6546 .observe(serde_json::json!({
6547 "kind": "armed",
6548 "previous": pool_meta_test_snapshot(&previous.meta, previous.revision, &previous.canonical)?,
6549 "candidate": pool_meta_test_snapshot(candidate, revision, durable)?,
6550 }))
6551 .await?;
6552 Ok(Some(barrier))
6553 }
6554
6555 async fn observe(&self, mut event: serde_json::Value) -> Result<()> {
6556 use tokio::io::AsyncWriteExt as _;
6557 event["nonce"] = serde_json::json!(self.arm.nonce);
6558 event["pid"] = serde_json::json!(std::process::id());
6559 event["transaction_id"] = serde_json::json!(self.transaction_id);
6560 event["case"] = serde_json::json!(self.arm.case);
6561 let mut line = serde_json::to_vec(&event)?;
6562 line.push(b'\n');
6563 let _event_guard = self.event_gate.lock().await;
6564 let mut file = tokio::fs::OpenOptions::new()
6565 .create(true)
6566 .append(true)
6567 .open(self.directory.join("events.jsonl"))
6568 .await?;
6569 file.write_all(&line).await?;
6570 file.flush().await?;
6571 Ok(())
6572 }
6573
6574 async fn wait_for_release(&self) -> Result<()> {
6575 tokio::time::timeout(std::time::Duration::from_secs(120), async {
6576 loop {
6577 match tokio::fs::metadata(self.directory.join("release")).await {
6578 Ok(_) => return Ok(()),
6579 Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
6580 Err(err) => return Err(Error::from(err)),
6581 }
6582 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
6583 }
6584 })
6585 .await
6586 .map_err(|_| Error::other("pool metadata test barrier release timed out"))?
6587 }
6588
6589 async fn ready(&self) -> Result<()> {
6590 self.observe(serde_json::json!({"kind": "ready", "cutpoint": self.arm.case}))
6591 .await?;
6592 self.wait_for_release().await
6593 }
6594
6595 async fn ready_after_all_participants(&self) -> Result<()> {
6596 let (blocked, completed) = match self.arm.case {
6597 PoolMetaPhaseBarrierCase::PrepareSubset => (0b0101, 0b1010),
6598 PoolMetaPhaseBarrierCase::CommitOne => (0b0111, 0b1000),
6599 _ => return Ok(()),
6600 };
6601 if self.blocked_pools.load(Ordering::SeqCst) == blocked
6602 && self.completed_writes.load(Ordering::SeqCst) == completed
6603 && !self.ready_emitted.swap(true, Ordering::SeqCst)
6604 {
6605 self.ready().await?;
6606 }
6607 Ok(())
6608 }
6609
6610 async fn before_write(&self, phase: &'static str, pool: usize, data: &[u8]) -> Result<()> {
6611 let blocked = match (self.arm.case, phase) {
6612 (PoolMetaPhaseBarrierCase::PrepareSubset, "prepare_cas") => ![1, 3].contains(&pool),
6613 (PoolMetaPhaseBarrierCase::CommitOne, "commit_cas") => pool != 3,
6614 _ => false,
6615 };
6616 if blocked {
6617 self.observe(serde_json::json!({
6618 "kind": "blocked", "phase": phase, "pool": pool,
6619 "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(data)),
6620 }))
6621 .await?;
6622 self.blocked_pools.fetch_or(1 << pool, Ordering::SeqCst);
6623 self.ready_after_all_participants().await?;
6624 self.wait_for_release().await?;
6625 }
6626 self.observe(serde_json::json!({
6627 "kind": "before-dispatch", "phase": phase, "pool": pool,
6628 "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(data)),
6629 }))
6630 .await
6631 }
6632
6633 async fn after_write(&self, phase: &'static str, pool: usize, data: &[u8], outcome: &PoolMetaCasWriteOutcome) -> Result<()> {
6634 self.observe(serde_json::json!({
6635 "kind": "after-cas", "phase": phase, "pool": pool,
6636 "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(data)),
6637 "ok": outcome.result.is_ok(), "tail_drained": outcome.result.is_ok(),
6638 "etag": outcome.result.as_ref().ok().and_then(|info| info.etag.as_deref()),
6639 "may_have_mutated": outcome.may_have_mutated,
6640 }))
6641 .await?;
6642 if outcome.result.is_ok() {
6643 match (self.arm.case, phase, pool) {
6644 (PoolMetaPhaseBarrierCase::PrepareSubset, "prepare_cas", 1 | 3)
6645 | (PoolMetaPhaseBarrierCase::CommitOne, "commit_cas", 3) => {
6646 self.completed_writes.fetch_or(1 << pool, Ordering::SeqCst);
6647 self.ready_after_all_participants().await?;
6648 }
6649 _ => {}
6650 }
6651 }
6652 Ok(())
6653 }
6654}
6655
6656fn pool_meta_cas_options(object: &str, token: &PoolMetaCasToken, fence: &PoolMetaPersistenceFence<'_>) -> Result<ObjectOptions> {
6657 fence.ensure_held()?;
6658 let mut opts = ObjectOptions {
6659 max_parity: true,
6660 write_completion: crate::object_api::WriteCompletion::TailDrained,
6661 no_lock: true,
6662 http_preconditions: Some(pool_meta_cas_preconditions(token, object)?),
6663 ..Default::default()
6664 };
6665 fence.add_to_options(&mut opts);
6666 Ok(opts)
6667}
6668
6669struct PoolMetaCasWriteOutcome {
6670 result: Result<crate::object_api::ObjectInfo>,
6671 source: Option<Arc<Error>>,
6672 may_have_mutated: bool,
6673}
6674
6675impl PoolMetaCasWriteOutcome {
6676 fn not_dispatched(err: Error) -> Self {
6677 Self {
6678 result: Err(err),
6679 source: None,
6680 may_have_mutated: false,
6681 }
6682 }
6683}
6684
6685async fn execute_pool_meta_object_cas<S>(
6686 pool: Arc<S>,
6687 object: &str,
6688 data: Vec<u8>,
6689 opts: ObjectOptions,
6690 fence: &PoolMetaPersistenceFence<'_>,
6691 phase: &'static str,
6692) -> PoolMetaCasWriteOutcome
6693where
6694 S: EcstoreObjectIO,
6695{
6696 if let Err(err) = fence.ensure_held() {
6698 return PoolMetaCasWriteOutcome::not_dispatched(err);
6699 }
6700 #[cfg(feature = "e2e-test-hooks")]
6701 let observation = std::env::var_os("RUSTFS_E2E_STARTUP_CAS_NONCE").map(|_| {
6702 serde_json::json!({
6703 "kind": "cas", "object": object, "phase": phase,
6704 "pool": STARTUP_CAS_OBSERVATION.try_with(|scope| {
6705 scope.pools.iter().position(|identity| *identity == Arc::as_ptr(&pool) as usize)
6706 }).ok().flatten(),
6707 "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&data)),
6708 "if_match": opts.http_preconditions.as_ref().and_then(|p| p.if_match.as_deref()),
6709 "if_none_match": opts.http_preconditions.as_ref().and_then(|p| p.if_none_match.as_deref()),
6710 "tail_drained": opts.write_completion == crate::object_api::WriteCompletion::TailDrained,
6711 "no_lock": opts.no_lock,
6712 })
6713 });
6714 let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await;
6715 let may_have_mutated = !matches!(&result, Err(Error::PreconditionFailed));
6716 if !may_have_mutated {
6717 record_pool_meta_stale_write_rejection(phase);
6718 }
6719 let mut source = None;
6720 let result = match result {
6721 Ok(object_info) => fence.ensure_held().map(|()| object_info),
6722 Err(err) => {
6723 let original = Arc::new(err);
6724 source = Some(Arc::clone(&original));
6725 if matches!(original.as_ref(), Error::PreconditionFailed) {
6726 Err(Error::PreconditionFailed)
6727 } else {
6728 Err(Error::other(pool_metadata_error(
6729 crate::error::PoolMetadataFailure::TransactionUnknown,
6730 phase,
6731 Some(original),
6732 )))
6733 }
6734 }
6735 };
6736 #[cfg(feature = "e2e-test-hooks")]
6737 if let Some(mut observation) = observation {
6738 observation["ok"] = serde_json::json!(result.is_ok());
6739 observation["etag"] = serde_json::json!(result.as_ref().ok().and_then(|info| info.etag.as_deref()));
6740 observation["mod_time"] = serde_json::json!(
6741 result
6742 .as_ref()
6743 .ok()
6744 .and_then(|info| info.mod_time)
6745 .map(|time| time.unix_timestamp_nanos().to_string())
6746 );
6747 observation["error"] = serde_json::json!(result.as_ref().err().map(ToString::to_string));
6748 startup_cas_test_observe(observation);
6749 }
6750 PoolMetaCasWriteOutcome {
6751 result,
6752 source,
6753 may_have_mutated,
6754 }
6755}
6756
6757async fn save_pool_meta_object_cas<S>(
6758 pool: Arc<S>,
6759 object: &str,
6760 data: Vec<u8>,
6761 token: &PoolMetaCasToken,
6762 fence: &PoolMetaPersistenceFence<'_>,
6763 phase: &'static str,
6764 transaction_arm: &mut PoolMetaTransactionArm,
6765) -> Result<crate::object_api::ObjectInfo>
6766where
6767 S: EcstoreObjectIO,
6768{
6769 let opts = pool_meta_cas_options(object, token, fence)?;
6770 let previous_phase = transaction_arm.phase;
6773 transaction_arm.phase = Some(phase);
6774 let outcome = execute_pool_meta_object_cas(pool, object, data, opts, fence, phase).await;
6775 if !outcome.may_have_mutated {
6776 transaction_arm.phase = previous_phase;
6777 }
6778 if let Some(source) = outcome.source
6779 && (!matches!(source.as_ref(), Error::PreconditionFailed)
6780 || transaction_arm
6781 .source
6782 .as_ref()
6783 .is_none_or(|previous| matches!(previous.as_ref(), Error::PreconditionFailed)))
6784 {
6785 transaction_arm.source = Some(source);
6786 }
6787 outcome.result
6788}
6789
6790async fn save_pool_meta_phase<S>(
6791 pools: &[Arc<S>],
6792 data: &[u8],
6793 tokens: &[PoolMetaCasToken],
6794 fence: &PoolMetaPersistenceFence<'_>,
6795 phase: &'static str,
6796 transaction_arm: &mut PoolMetaTransactionArm,
6797 #[cfg(feature = "e2e-test-hooks")] barrier: Option<&PoolMetaPhaseBarrier>,
6798) -> Vec<Result<crate::object_api::ObjectInfo>>
6799where
6800 S: EcstoreObjectIO,
6801{
6802 if pools.len() != tokens.len() {
6803 return vec![Err(Error::other("pool metadata phase has inconsistent replica revisions"))];
6804 }
6805 let options = tokens
6806 .iter()
6807 .map(|token| pool_meta_cas_options(POOL_META_NAME, token, fence))
6808 .collect::<Vec<_>>();
6809 let previous_phase = transaction_arm.phase;
6810 if options.iter().any(Result::is_ok) {
6811 transaction_arm.phase = Some(phase);
6812 }
6813 let writes = futures::stream::iter(pools.iter().cloned().zip(options).enumerate().map(
6816 |(pool_index, (pool, opts))| async move {
6817 #[cfg(feature = "e2e-test-hooks")]
6818 if opts.is_ok()
6819 && let Some(barrier) = barrier
6820 && let Err(err) = barrier.before_write(phase, pool_index, data).await
6821 {
6822 return (pool_index, PoolMetaCasWriteOutcome::not_dispatched(err));
6823 }
6824 let outcome = match opts {
6825 Ok(opts) => execute_pool_meta_object_cas(pool, POOL_META_NAME, data.to_vec(), opts, fence, phase).await,
6826 Err(err) => PoolMetaCasWriteOutcome::not_dispatched(err),
6827 };
6828 #[cfg(feature = "e2e-test-hooks")]
6829 let outcome = {
6830 let mut outcome = outcome;
6831 if let Some(barrier) = barrier
6832 && let Err(err) = barrier.after_write(phase, pool_index, data, &outcome).await
6833 && outcome.result.is_ok()
6834 {
6835 outcome.result = Err(err);
6836 }
6837 outcome
6838 };
6839 (pool_index, outcome)
6840 },
6841 ))
6842 .buffer_unordered(4);
6843 futures::pin_mut!(writes);
6844 let mut results = Vec::with_capacity(pools.len());
6845 let mut may_have_mutated = false;
6846 let mut source_pool_index: Option<(bool, usize)> = None;
6847 while let Some((pool_index, outcome)) = writes.next().await {
6848 may_have_mutated |= outcome.may_have_mutated;
6849 if let Some(source) = outcome.source
6850 && (outcome.may_have_mutated
6851 || transaction_arm
6852 .source
6853 .as_ref()
6854 .is_none_or(|previous| matches!(previous.as_ref(), Error::PreconditionFailed)))
6855 && source_pool_index.is_none_or(|(previous_mutated, previous_index)| {
6856 (outcome.may_have_mutated && !previous_mutated)
6857 || (outcome.may_have_mutated == previous_mutated && pool_index < previous_index)
6858 })
6859 {
6860 transaction_arm.source = Some(source);
6862 source_pool_index = Some((outcome.may_have_mutated, pool_index));
6863 }
6864 results.push((pool_index, outcome.result));
6865 }
6866 if !may_have_mutated {
6867 transaction_arm.phase = previous_phase;
6868 }
6869 results.sort_unstable_by_key(|(pool_index, _)| *pool_index);
6870 results.into_iter().map(|(_, result)| result).collect()
6871}
6872
6873#[derive(Debug, Clone, Copy)]
6875enum PoolMetaIdentityWriteScope<'a> {
6876 All,
6879 Pools(&'a [usize]),
6886}
6887
6888fn identity_write_satisfied(
6889 selection: &PoolMetaIdentitySelection,
6890 identity: PersistedPoolMetaIdentity,
6891 scope: PoolMetaIdentityWriteScope<'_>,
6892 targets: &[usize],
6893) -> bool {
6894 if selection.identity != Some(identity) {
6895 return false;
6896 }
6897 match scope {
6898 PoolMetaIdentityWriteScope::All => !selection.needs_repair,
6899 PoolMetaIdentityWriteScope::Pools(_) => targets
6900 .iter()
6901 .all(|pool_idx| selection.valid_replicas.get(*pool_idx).copied().unwrap_or(false)),
6902 }
6903}
6904
6905async fn persist_pool_meta_identity<S>(
6906 pools: Vec<Arc<S>>,
6907 write_state: &mut PoolMetaWriteState,
6908 initialized: bool,
6909 scope: PoolMetaIdentityWriteScope<'_>,
6910 fence: &PoolMetaPersistenceFence<'_>,
6911 transaction_arm: &mut PoolMetaTransactionArm,
6912) -> Result<()>
6913where
6914 S: EcstoreObjectIO,
6915{
6916 let Some(cluster_id) = write_state.expected_cluster_id else {
6917 return Ok(());
6918 };
6919 let targets: Vec<usize> = match scope {
6920 PoolMetaIdentityWriteScope::All => (0..pools.len()).collect(),
6921 PoolMetaIdentityWriteScope::Pools(indices) => {
6922 if initialized {
6923 return Err(Error::other("pool metadata identity commit must address every pool"));
6924 }
6925 if indices
6926 .iter()
6927 .any(|pool_idx| *pool_idx >= pools.len() || !write_state.pool_bootstrap_authority_proven(*pool_idx))
6928 {
6929 return Err(Error::other(
6930 "pool metadata recovery required: a pending cluster identity can only be attested for pools this startup formatted or adopted first-hand",
6931 ));
6932 }
6933 indices.to_vec()
6934 }
6935 };
6936 for attempt in 0..POOL_META_CAS_MAX_ATTEMPTS {
6937 let selection = load_pool_meta_identity_selection_observing(pools.clone(), write_state, cluster_id).await?;
6938 if !selection.repair_write_safe {
6939 write_state.block_writes();
6940 return Err(Error::other(
6941 "pool metadata recovery required: cluster identity has an unreadable replica",
6942 ));
6943 }
6944 let identity = match (selection.identity, scope) {
6945 (Some(identity), PoolMetaIdentityWriteScope::Pools(_)) if identity.initialized => return Ok(()),
6949 (Some(identity), _) if identity.initialized || initialized => PersistedPoolMetaIdentity {
6950 initialized: true,
6951 fresh_bootstrap_nonce: None,
6952 ..identity
6953 },
6954 (Some(identity), _) => identity,
6955 (None, PoolMetaIdentityWriteScope::Pools(indices)) => {
6959 if !indices.contains(&0) {
6960 return Ok(());
6961 }
6962 PersistedPoolMetaIdentity {
6963 version: POOL_META_IDENTITY_VERSION,
6964 cluster_id,
6965 epoch: write_state.cluster_epoch.unwrap_or(POOL_META_INITIAL_EPOCH),
6966 initialized: false,
6967 fresh_bootstrap_nonce: Some(uuid::Uuid::new_v4()),
6968 }
6969 }
6970 (None, PoolMetaIdentityWriteScope::All) if !initialized && !write_state.bootstrap_identity_proven() => {
6971 write_state.block_writes();
6972 return Err(Error::other(
6973 "pool metadata recovery required: cannot create a pending cluster identity without verified fresh-bootstrap proof or legacy-adoption proof",
6974 ));
6975 }
6976 (None, PoolMetaIdentityWriteScope::All) => PersistedPoolMetaIdentity {
6977 version: POOL_META_IDENTITY_VERSION,
6978 cluster_id,
6979 epoch: write_state.cluster_epoch.unwrap_or(POOL_META_INITIAL_EPOCH),
6980 initialized,
6981 fresh_bootstrap_nonce: (!initialized).then(uuid::Uuid::new_v4),
6982 },
6983 };
6984 if identity_write_satisfied(&selection, identity, scope, &targets) {
6985 return Ok(());
6986 }
6987 let data = encode_pool_meta_identity(identity)?;
6988 let mut conflict = false;
6989 for (pool_idx, (pool, token)) in pools.iter().cloned().zip(&selection.cas_tokens).enumerate() {
6990 if !targets.contains(&pool_idx) {
6991 continue;
6992 }
6993 match save_pool_meta_object_cas(
6994 pool,
6995 POOL_META_IDENTITY_NAME,
6996 data.clone(),
6997 token,
6998 fence,
6999 "identity_cas",
7000 transaction_arm,
7001 )
7002 .await
7003 {
7004 Ok(_) => {}
7005 Err(Error::PreconditionFailed) => {
7006 conflict = true;
7007 break;
7008 }
7009 Err(err) => return Err(err),
7010 }
7011 }
7012 if conflict {
7013 if attempt + 1 < POOL_META_CAS_MAX_ATTEMPTS {
7014 continue;
7015 }
7016 return Err(Error::PreconditionFailed);
7017 }
7018 let confirmed = load_pool_meta_identity_selection_observing(pools.clone(), write_state, cluster_id).await?;
7019 if identity_write_satisfied(&confirmed, identity, scope, &targets) {
7020 return Ok(());
7021 }
7022 }
7023 Err(Error::PreconditionFailed)
7024}
7025
7026pub(crate) async fn load_pool_meta_identity_observing<S>(pools: Vec<Arc<S>>, write_state: &mut PoolMetaWriteState) -> Result<()>
7027where
7028 S: EcstoreObjectIO,
7029{
7030 let Some(cluster_id) = write_state.expected_cluster_id else {
7031 return Ok(());
7032 };
7033 load_pool_meta_identity_selection_observing(pools, write_state, cluster_id)
7034 .await
7035 .map(|_| ())
7036}
7037
7038pub(crate) async fn persist_pool_meta_identity_for_startup<S>(
7039 pools: Vec<Arc<S>>,
7040 write_state: &mut PoolMetaWriteState,
7041 initialized: bool,
7042) -> Result<()>
7043where
7044 S: EcstoreObjectIO,
7045{
7046 let mut transaction_arm = write_state.arm_transaction();
7047 persist_pool_meta_identity(
7048 pools,
7049 write_state,
7050 initialized,
7051 PoolMetaIdentityWriteScope::All,
7052 &PoolMetaPersistenceFence::Distributed(None),
7053 &mut transaction_arm,
7054 )
7055 .await?;
7056 transaction_arm.disarm();
7057 Ok(())
7058}
7059
7060pub(crate) async fn persist_pool_meta_identity_for_attested_pools<S>(
7066 pools: Vec<Arc<S>>,
7067 write_state: &mut PoolMetaWriteState,
7068 pool_indices: &[usize],
7069) -> Result<()>
7070where
7071 S: EcstoreObjectIO,
7072{
7073 let mut transaction_arm = write_state.arm_transaction();
7074 persist_pool_meta_identity(
7075 pools,
7076 write_state,
7077 false,
7078 PoolMetaIdentityWriteScope::Pools(pool_indices),
7079 &PoolMetaPersistenceFence::Distributed(None),
7080 &mut transaction_arm,
7081 )
7082 .await?;
7083 transaction_arm.disarm();
7084 Ok(())
7085}
7086
7087#[derive(Debug, Clone, Serialize, Deserialize)]
7088#[serde(deny_unknown_fields)]
7089struct PersistedPoolMeta {
7090 pub version: u16,
7091 pub pools: Vec<PersistedPoolStatus>,
7092}
7093
7094#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
7095#[serde(deny_unknown_fields)]
7096struct PersistedPoolStatus {
7097 #[serde(rename = "id")]
7098 pub id: usize,
7099 #[serde(rename = "cmdline")]
7100 pub cmd_line: String,
7101 #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
7102 pub last_update: OffsetDateTime,
7103 #[serde(rename = "decommissionInfo")]
7104 pub decommission: Option<PersistedPoolDecommissionInfo>,
7105}
7106
7107#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
7108#[serde(deny_unknown_fields)]
7109struct PersistedPoolDecommissionInfo {
7110 #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
7111 pub start_time: Option<OffsetDateTime>,
7112 #[serde(rename = "startSize")]
7113 pub start_size: usize,
7114 #[serde(rename = "totalSize")]
7115 pub total_size: usize,
7116 #[serde(rename = "currentSize")]
7117 pub current_size: usize,
7118 #[serde(rename = "complete")]
7119 pub complete: bool,
7120 #[serde(rename = "failed")]
7121 pub failed: bool,
7122 #[serde(rename = "canceled")]
7123 pub canceled: bool,
7124 #[serde(rename = "queued", default)]
7125 pub queued: bool,
7126 #[serde(rename = "queuedBuckets", default)]
7127 pub queued_buckets: Vec<String>,
7128 #[serde(rename = "decommissionedBuckets", default)]
7129 pub decommissioned_buckets: Vec<String>,
7130 #[serde(rename = "bucket", default)]
7131 pub bucket: String,
7132 #[serde(rename = "prefix", default)]
7133 pub prefix: String,
7134 #[serde(rename = "object", default)]
7135 pub object: String,
7136 #[serde(rename = "objectsDecommissioned")]
7137 pub items_decommissioned: usize,
7138 #[serde(rename = "objectsDecommissionedFailed")]
7139 pub items_decommission_failed: usize,
7140 #[serde(rename = "bytesDecommissioned")]
7141 pub bytes_done: usize,
7142 #[serde(rename = "bytesDecommissionedFailed")]
7143 pub bytes_failed: usize,
7144 #[serde(rename = "terminalReloadAttemptAt", with = "time::serde::rfc3339::option", default)]
7145 pub terminal_reload_attempt_at: Option<OffsetDateTime>,
7146 #[serde(rename = "terminalReloadFailures", default)]
7147 pub terminal_reload_failures: Vec<String>,
7148 #[serde(rename = "unresolvedEntries", default)]
7149 pub unresolved_entries: Vec<DecommissionUnresolvedEntry>,
7150 #[serde(rename = "capacityReservation", default)]
7151 pub capacity_reservation: Option<DecommissionCapacityReservation>,
7152 #[serde(rename = "capacityBlockedReason", default)]
7153 pub capacity_blocked_reason: Option<String>,
7154}
7155
7156#[derive(Debug, Clone, Serialize, Deserialize)]
7157#[serde(deny_unknown_fields)]
7158struct PersistedPoolMetaV1 {
7159 pub version: u16,
7160 pub pools: Vec<PersistedPoolStatusV1>,
7161}
7162
7163#[derive(Debug, Clone, Serialize, Deserialize)]
7164#[serde(deny_unknown_fields)]
7165struct PersistedPoolStatusV1 {
7166 #[serde(rename = "id")]
7167 pub id: usize,
7168 #[serde(rename = "cmdline")]
7169 pub cmd_line: String,
7170 #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
7171 pub last_update: OffsetDateTime,
7172 #[serde(rename = "decommissionInfo")]
7173 pub decommission: Option<PersistedPoolDecommissionInfoV1>,
7174}
7175
7176#[derive(Debug, Clone, Default, Serialize, Deserialize)]
7177#[serde(deny_unknown_fields)]
7178struct PersistedPoolDecommissionInfoV1 {
7179 #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
7180 pub start_time: Option<OffsetDateTime>,
7181 #[serde(rename = "startSize")]
7182 pub start_size: usize,
7183 #[serde(rename = "totalSize")]
7184 pub total_size: usize,
7185 #[serde(rename = "currentSize")]
7186 pub current_size: usize,
7187 #[serde(rename = "complete")]
7188 pub complete: bool,
7189 #[serde(rename = "failed")]
7190 pub failed: bool,
7191 #[serde(rename = "canceled")]
7192 pub canceled: bool,
7193 #[serde(rename = "queued", default)]
7194 pub queued: bool,
7195 #[serde(rename = "queuedBuckets", default)]
7196 pub queued_buckets: Vec<String>,
7197 #[serde(rename = "decommissionedBuckets", default)]
7198 pub decommissioned_buckets: Vec<String>,
7199 #[serde(rename = "bucket", default)]
7200 pub bucket: String,
7201 #[serde(rename = "prefix", default)]
7202 pub prefix: String,
7203 #[serde(rename = "object", default)]
7204 pub object: String,
7205 #[serde(rename = "objectsDecommissioned")]
7206 pub items_decommissioned: usize,
7207 #[serde(rename = "objectsDecommissionedFailed")]
7208 pub items_decommission_failed: usize,
7209 #[serde(rename = "bytesDecommissioned")]
7210 pub bytes_done: usize,
7211 #[serde(rename = "bytesDecommissionedFailed")]
7212 pub bytes_failed: usize,
7213 #[serde(rename = "terminalReloadAttemptAt", with = "time::serde::rfc3339::option", default)]
7214 pub terminal_reload_attempt_at: Option<OffsetDateTime>,
7215 #[serde(rename = "terminalReloadFailures", default)]
7216 pub terminal_reload_failures: Vec<String>,
7217}
7218
7219#[derive(Debug, Clone, Serialize, Deserialize)]
7220#[serde(deny_unknown_fields)]
7221struct LegacyPoolMeta {
7222 pub version: u16,
7223 pub pools: Vec<LegacyPoolStatus>,
7224 pub dont_save: bool,
7225}
7226
7227#[derive(Debug, Clone, Serialize, Deserialize)]
7228#[serde(deny_unknown_fields)]
7229struct LegacyPoolStatus {
7230 #[serde(rename = "id")]
7231 pub id: usize,
7232 #[serde(rename = "cmdline")]
7233 pub cmd_line: String,
7234 #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
7235 pub last_update: OffsetDateTime,
7236 #[serde(rename = "decommissionInfo")]
7237 pub decommission: Option<LegacyPoolDecommissionInfo>,
7238}
7239
7240#[derive(Debug, Clone, Default, Serialize, Deserialize)]
7241#[serde(deny_unknown_fields)]
7242struct LegacyPoolDecommissionInfo {
7243 #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
7244 pub start_time: Option<OffsetDateTime>,
7245 #[serde(rename = "startSize")]
7246 pub start_size: usize,
7247 #[serde(rename = "totalSize")]
7248 pub total_size: usize,
7249 #[serde(rename = "currentSize")]
7250 pub current_size: usize,
7251 #[serde(rename = "complete")]
7252 pub complete: bool,
7253 #[serde(rename = "failed")]
7254 pub failed: bool,
7255 #[serde(rename = "canceled")]
7256 pub canceled: bool,
7257 #[serde(rename = "objectsDecommissioned")]
7258 pub items_decommissioned: usize,
7259 #[serde(rename = "objectsDecommissionedFailed")]
7260 pub items_decommission_failed: usize,
7261 #[serde(rename = "bytesDecommissioned")]
7262 pub bytes_done: usize,
7263 #[serde(rename = "bytesDecommissionedFailed")]
7264 pub bytes_failed: usize,
7265}
7266
7267fn ensure_pool_meta_payload_version(actual: u16, expected: u16, kind: &str) -> Result<()> {
7268 if actual == expected {
7269 return Ok(());
7270 }
7271 Err(Error::other(format!(
7272 "pool metadata {kind} payload has version {actual}, expected {expected}"
7273 )))
7274}
7275
7276impl TryFrom<PersistedPoolMeta> for PoolMeta {
7277 type Error = Error;
7278
7279 fn try_from(value: PersistedPoolMeta) -> Result<Self> {
7280 ensure_pool_meta_payload_version(value.version, POOL_META_VERSION, "current")?;
7281 let meta = Self {
7282 version: POOL_META_VERSION,
7283 pools: value.pools.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?,
7284 dont_save: false,
7285 };
7286 validate_decommission_capacity_model_cohort(&meta)?;
7287 Ok(meta)
7288 }
7289}
7290
7291impl TryFrom<PersistedPoolMetaV1> for PoolMeta {
7292 type Error = Error;
7293
7294 fn try_from(value: PersistedPoolMetaV1) -> Result<Self> {
7295 ensure_pool_meta_payload_version(value.version, POOL_META_V1_VERSION, "v1")?;
7296 Ok(Self {
7297 version: POOL_META_V1_VERSION,
7298 pools: value.pools.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?,
7299 dont_save: false,
7300 })
7301 }
7302}
7303
7304impl TryFrom<LegacyPoolMeta> for PoolMeta {
7305 type Error = Error;
7306
7307 fn try_from(value: LegacyPoolMeta) -> Result<Self> {
7308 let LegacyPoolMeta {
7309 version,
7310 pools,
7311 dont_save: _,
7312 } = value;
7313 ensure_pool_meta_payload_version(version, POOL_META_V1_VERSION, "legacy v1")?;
7314 Ok(Self {
7315 version: POOL_META_V1_VERSION,
7316 pools: pools.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?,
7317 dont_save: false,
7318 })
7319 }
7320}
7321
7322impl TryFrom<PersistedPoolStatus> for PoolStatus {
7323 type Error = Error;
7324
7325 fn try_from(value: PersistedPoolStatus) -> Result<Self> {
7326 let decommission = value.decommission.map(TryInto::try_into).transpose()?;
7327 if decommission
7328 .as_ref()
7329 .and_then(|info: &PoolDecommissionInfo| info.capacity_reservation.as_ref())
7330 .is_some_and(|reservation| reservation.source_pool_index != value.id)
7331 {
7332 return Err(Error::other(
7333 "pool metadata load failed: decommission capacity reservation source pool does not match its owner",
7334 ));
7335 }
7336 Ok(Self {
7337 id: value.id,
7338 cmd_line: value.cmd_line,
7339 last_update: value.last_update,
7340 decommission,
7341 })
7342 }
7343}
7344
7345impl TryFrom<PersistedPoolStatusV1> for PoolStatus {
7346 type Error = Error;
7347
7348 fn try_from(value: PersistedPoolStatusV1) -> Result<Self> {
7349 Ok(Self {
7350 id: value.id,
7351 cmd_line: value.cmd_line,
7352 last_update: value.last_update,
7353 decommission: value.decommission.map(TryInto::try_into).transpose()?,
7354 })
7355 }
7356}
7357
7358impl TryFrom<LegacyPoolStatus> for PoolStatus {
7359 type Error = Error;
7360
7361 fn try_from(value: LegacyPoolStatus) -> Result<Self> {
7362 Ok(Self {
7363 id: value.id,
7364 cmd_line: value.cmd_line,
7365 last_update: value.last_update,
7366 decommission: value.decommission.map(TryInto::try_into).transpose()?,
7367 })
7368 }
7369}
7370
7371impl TryFrom<PersistedPoolDecommissionInfo> for PoolDecommissionInfo {
7372 type Error = Error;
7373
7374 fn try_from(value: PersistedPoolDecommissionInfo) -> Result<Self> {
7375 validate_decommission_terminal_state(value.complete, value.failed, value.canceled)?;
7376 validate_decommission_capacity_reservation(value.capacity_reservation.as_ref())?;
7377 Ok(Self {
7378 start_time: value.start_time,
7379 start_size: value.start_size,
7380 total_size: value.total_size,
7381 current_size: value.current_size,
7382 complete: value.complete,
7383 failed: value.failed,
7384 canceled: value.canceled,
7385 queued: value.queued,
7386 queued_buckets: value.queued_buckets,
7387 decommissioned_buckets: value.decommissioned_buckets,
7388 bucket: value.bucket,
7389 prefix: value.prefix,
7390 object: value.object,
7391 stage: String::new(),
7392 items_decommissioned: value.items_decommissioned,
7393 items_decommission_failed: value.items_decommission_failed,
7394 bytes_done: value.bytes_done,
7395 bytes_failed: value.bytes_failed,
7396 terminal_reload_attempt_at: value.terminal_reload_attempt_at,
7397 terminal_reload_failures: value.terminal_reload_failures,
7398 capacity_reservation: value.capacity_reservation,
7399 capacity_blocked_reason: value.capacity_blocked_reason,
7400 unresolved_entries: value.unresolved_entries,
7401 progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed),
7402 progress_save_retry_after: None,
7403 progress_save_last_at: None,
7404 })
7405 }
7406}
7407
7408impl TryFrom<PersistedPoolDecommissionInfoV1> for PoolDecommissionInfo {
7409 type Error = Error;
7410
7411 fn try_from(value: PersistedPoolDecommissionInfoV1) -> Result<Self> {
7412 validate_decommission_terminal_state(value.complete, value.failed, value.canceled)?;
7413 Ok(Self {
7414 start_time: value.start_time,
7415 start_size: value.start_size,
7416 total_size: value.total_size,
7417 current_size: value.current_size,
7418 complete: value.complete,
7419 failed: value.failed,
7420 canceled: value.canceled,
7421 queued: value.queued,
7422 queued_buckets: value.queued_buckets,
7423 decommissioned_buckets: value.decommissioned_buckets,
7424 bucket: value.bucket,
7425 prefix: value.prefix,
7426 object: value.object,
7427 stage: String::new(),
7428 items_decommissioned: value.items_decommissioned,
7429 items_decommission_failed: value.items_decommission_failed,
7430 bytes_done: value.bytes_done,
7431 bytes_failed: value.bytes_failed,
7432 terminal_reload_attempt_at: value.terminal_reload_attempt_at,
7433 terminal_reload_failures: value.terminal_reload_failures,
7434 capacity_reservation: None,
7435 capacity_blocked_reason: None,
7436 unresolved_entries: Vec::new(),
7437 progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed),
7438 progress_save_retry_after: None,
7439 progress_save_last_at: None,
7440 })
7441 }
7442}
7443
7444impl TryFrom<LegacyPoolDecommissionInfo> for PoolDecommissionInfo {
7445 type Error = Error;
7446
7447 fn try_from(value: LegacyPoolDecommissionInfo) -> Result<Self> {
7448 validate_decommission_terminal_state(value.complete, value.failed, value.canceled)?;
7449 Ok(Self {
7450 start_time: value.start_time,
7451 start_size: value.start_size,
7452 total_size: value.total_size,
7453 current_size: value.current_size,
7454 complete: value.complete,
7455 failed: value.failed,
7456 canceled: value.canceled,
7457 queued: false,
7458 queued_buckets: Vec::new(),
7459 decommissioned_buckets: Vec::new(),
7460 bucket: String::new(),
7461 prefix: String::new(),
7462 object: String::new(),
7463 stage: String::new(),
7464 items_decommissioned: value.items_decommissioned,
7465 items_decommission_failed: value.items_decommission_failed,
7466 bytes_done: value.bytes_done,
7467 bytes_failed: value.bytes_failed,
7468 terminal_reload_attempt_at: None,
7469 terminal_reload_failures: Vec::new(),
7470 capacity_reservation: None,
7471 capacity_blocked_reason: None,
7472 unresolved_entries: Vec::new(),
7473 progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed),
7474 progress_save_retry_after: None,
7475 progress_save_last_at: None,
7476 })
7477 }
7478}
7479
7480impl From<&PoolMeta> for PersistedPoolMeta {
7481 fn from(value: &PoolMeta) -> Self {
7482 Self {
7483 version: POOL_META_VERSION,
7484 pools: value.pools.iter().map(Into::into).collect(),
7485 }
7486 }
7487}
7488
7489impl From<&PoolMeta> for PersistedPoolMetaV1 {
7490 fn from(value: &PoolMeta) -> Self {
7491 Self {
7492 version: POOL_META_V1_VERSION,
7493 pools: value.pools.iter().map(Into::into).collect(),
7494 }
7495 }
7496}
7497
7498impl From<&PoolStatus> for PersistedPoolStatus {
7499 fn from(value: &PoolStatus) -> Self {
7500 Self {
7501 id: value.id,
7502 cmd_line: value.cmd_line.clone(),
7503 last_update: value.last_update,
7504 decommission: value.decommission.as_ref().map(Into::into),
7505 }
7506 }
7507}
7508
7509impl From<&PoolStatus> for PersistedPoolStatusV1 {
7510 fn from(value: &PoolStatus) -> Self {
7511 Self {
7512 id: value.id,
7513 cmd_line: value.cmd_line.clone(),
7514 last_update: value.last_update,
7515 decommission: value.decommission.as_ref().map(Into::into),
7516 }
7517 }
7518}
7519
7520impl From<&PoolDecommissionInfo> for PersistedPoolDecommissionInfo {
7521 fn from(value: &PoolDecommissionInfo) -> Self {
7522 Self {
7523 start_time: value.start_time,
7524 start_size: value.start_size,
7525 total_size: value.total_size,
7526 current_size: value.current_size,
7527 complete: value.complete,
7528 failed: value.failed,
7529 canceled: value.canceled,
7530 queued: value.queued,
7531 queued_buckets: value.queued_buckets.clone(),
7532 decommissioned_buckets: value.decommissioned_buckets.clone(),
7533 bucket: value.bucket.clone(),
7534 prefix: value.prefix.clone(),
7535 object: value.object.clone(),
7536 items_decommissioned: value.items_decommissioned,
7537 items_decommission_failed: value.items_decommission_failed,
7538 bytes_done: value.bytes_done,
7539 bytes_failed: value.bytes_failed,
7540 terminal_reload_attempt_at: value.terminal_reload_attempt_at,
7541 terminal_reload_failures: value.terminal_reload_failures.clone(),
7542 capacity_reservation: value.capacity_reservation.clone(),
7543 capacity_blocked_reason: value.capacity_blocked_reason.clone(),
7544 unresolved_entries: value.unresolved_entries.clone(),
7545 }
7546 }
7547}
7548
7549impl From<&PoolDecommissionInfo> for PersistedPoolDecommissionInfoV1 {
7550 fn from(value: &PoolDecommissionInfo) -> Self {
7551 Self {
7552 start_time: value.start_time,
7553 start_size: value.start_size,
7554 total_size: value.total_size,
7555 current_size: value.current_size,
7556 complete: value.complete,
7557 failed: value.failed,
7558 canceled: value.canceled,
7559 queued: value.queued,
7560 queued_buckets: value.queued_buckets.clone(),
7561 decommissioned_buckets: value.decommissioned_buckets.clone(),
7562 bucket: value.bucket.clone(),
7563 prefix: value.prefix.clone(),
7564 object: value.object.clone(),
7565 items_decommissioned: value.items_decommissioned,
7566 items_decommission_failed: value.items_decommission_failed,
7567 bytes_done: value.bytes_done,
7568 bytes_failed: value.bytes_failed,
7569 terminal_reload_attempt_at: value.terminal_reload_attempt_at,
7570 terminal_reload_failures: value.terminal_reload_failures.clone(),
7571 }
7572 }
7573}
7574
7575#[derive(Debug)]
7576struct PoolMetaSaveOutcome {
7577 transaction_arm: PoolMetaTransactionArm,
7578 committed: PoolMeta,
7579}
7580
7581async fn load_pool_meta_for_transaction_recovery<S>(
7585 pools: Vec<Arc<S>>,
7586 write_state: &mut PoolMetaWriteState,
7587) -> Result<PoolMetaSelection>
7588where
7589 S: EcstoreObjectIO,
7590{
7591 let result = async {
7592 if let Some(cluster_id) = write_state.expected_cluster_id {
7593 let reads = join_all(pools.iter().cloned().map(read_pool_meta_identity_replica)).await;
7594 for read in &reads {
7595 match &read.replica {
7596 PoolMetaIdentityReplica::Unreadable(source) => {
7597 return Err(Error::other(pool_metadata_error(
7598 crate::error::PoolMetadataFailure::ReadUnavailable,
7599 "recovery_identity_read",
7600 Some(Arc::clone(source)),
7601 )));
7602 }
7603 PoolMetaIdentityReplica::Corrupt(_) | PoolMetaIdentityReplica::Incompatible(_) => {
7604 return Err(Error::other("pool metadata recovery requires operator repair of cluster identity"));
7605 }
7606 _ => {}
7607 }
7608 }
7609 let identity = select_pool_meta_identity(reads, cluster_id)?;
7610 if !identity.identity.is_some_and(|identity| identity.initialized) {
7611 return Err(Error::other("pool metadata recovery requires an initialized durable cluster identity"));
7612 }
7613 write_state.observe_identity(&identity)?;
7614 }
7615 let reads = read_pool_meta_replicas(pools, true).await;
7616 if let Some(err) = pool_meta_replica_read_failure(&reads, "pool metadata transaction recovery") {
7617 return Err(err);
7618 }
7619 if reads
7620 .iter()
7621 .any(|read| matches!(&read.replica, PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)))
7622 {
7623 return Err(Error::other(
7624 "pool metadata recovery requires operator repair of corrupt or incompatible replicas",
7625 ));
7626 }
7627 let selection = select_pool_meta_replica_reads(reads)?;
7628 write_state.observe_selection(&selection)?;
7629 if selection.absent || selection.meta.pools.is_empty() {
7630 return Err(Error::other("pool metadata recovery cannot initialize missing metadata"));
7631 }
7632 selection
7633 .replica_state
7634 .ensure_write_safe("pool metadata transaction recovery")?;
7635 Ok(selection)
7636 }
7637 .await;
7638 result.map_err(|err: Error| {
7639 if err.pool_metadata_failure().is_some() {
7640 err
7641 } else {
7642 Error::other(pool_metadata_error(
7643 crate::error::PoolMetadataFailure::RecoveryRequired,
7644 "recovery_validation",
7645 Some(Arc::new(err)),
7646 ))
7647 }
7648 })
7649}
7650
7651async fn repair_pool_meta_transaction<S>(
7652 pools: Vec<Arc<S>>,
7653 write_state: &mut PoolMetaWriteState,
7654 selection: PoolMetaSelection,
7655 fence: &PoolMetaPersistenceFence<'_>,
7656) -> Result<PoolMetaSaveOutcome>
7657where
7658 S: EcstoreObjectIO,
7659{
7660 let mut transaction_arm = write_state.arm_transaction();
7661 let mut expected_meta = selection.meta.clone();
7662 let mut expected_revision = selection.revision;
7663 if selection.generation_protocol_observed && !expected_revision.is_generation_protocol() {
7666 let (cluster_id, epoch) = selection
7667 .generation_identity
7668 .ok_or_else(|| Error::other("pool metadata recovery has no V3 cluster identity"))?;
7669 expected_meta.version = POOL_META_GENERATION_VERSION;
7670 expected_revision = PoolMetaRevision {
7671 version: POOL_META_GENERATION_VERSION,
7672 cluster_id: Some(cluster_id),
7673 epoch,
7674 generation: 1,
7675 transaction_id: Some(uuid::Uuid::new_v4()),
7676 };
7677 }
7678 let expected_canonical = if expected_revision.is_generation_protocol() {
7679 encode_pool_meta_v3_envelope(&expected_meta, expected_revision, true, None)?
7680 } else {
7681 expected_meta.encode_config_data_for_v2_gate(true)?
7682 };
7683 if selection.replica_state.needs_repair {
7684 let data = if expected_revision.is_generation_protocol() {
7685 expected_canonical.clone()
7686 } else {
7687 expected_meta.encode_config_data_for_v2_gate(expected_revision.version == POOL_META_VERSION)?
7688 };
7689 for (pool, token) in pools.iter().cloned().zip(&selection.cas_tokens) {
7690 save_pool_meta_object_cas(pool, POOL_META_NAME, data.clone(), token, fence, "recovery_cas", &mut transaction_arm)
7691 .await?;
7692 }
7693 }
7694 persist_pool_meta_identity(
7695 pools.clone(),
7696 write_state,
7697 true,
7698 PoolMetaIdentityWriteScope::All,
7699 fence,
7700 &mut transaction_arm,
7701 )
7702 .await?;
7703 let confirmed = load_pool_meta_for_transaction_recovery(pools, write_state).await?;
7704 if confirmed.revision != expected_revision
7705 || confirmed.canonical.as_ref() != Some(&expected_canonical)
7706 || confirmed.replica_state.needs_repair
7707 || write_state.identity_requires_repair()
7708 {
7709 return Err(Error::PreconditionFailed);
7710 }
7711 fence.ensure_held()?;
7712 Ok(PoolMetaSaveOutcome {
7713 transaction_arm,
7714 committed: confirmed.meta,
7715 })
7716}
7717
7718impl PoolMetaSaveOutcome {
7719 fn disarm(mut self) {
7720 self.transaction_arm.disarm();
7721 }
7722
7723 fn into_committed(mut self) -> PoolMeta {
7724 self.transaction_arm.disarm();
7725 self.committed
7726 }
7727}
7728
7729impl PoolMeta {
7730 fn current_decommission_movement_update(&self) -> Option<OffsetDateTime> {
7731 self.pools
7732 .iter()
7733 .filter(|pool| pool.decommission.is_some())
7734 .map(|pool| pool.last_update)
7735 .max()
7736 }
7737
7738 fn current_rebalance_movement_update(rebalance_meta: Option<&RebalanceMeta>) -> Option<OffsetDateTime> {
7739 rebalance_meta
7740 .into_iter()
7741 .flat_map(|meta| {
7742 meta.stopped_at.into_iter().chain(
7743 meta.pool_stats
7744 .iter()
7745 .flat_map(|pool| [pool.info.start_time, pool.info.end_time])
7746 .flatten(),
7747 )
7748 })
7749 .max()
7750 }
7751
7752 pub(crate) fn next_scanner_data_movement_update(
7753 &self,
7754 now: OffsetDateTime,
7755 rebalance_meta: Option<&RebalanceMeta>,
7756 ) -> OffsetDateTime {
7757 let max_seen = self
7758 .current_decommission_movement_update()
7759 .max(Self::current_rebalance_movement_update(rebalance_meta));
7760 match max_seen {
7761 Some(max_seen) => max_seen
7762 .checked_add(Duration::nanoseconds(1))
7763 .map(|next| now.max(next))
7764 .unwrap_or(max_seen),
7765 None => now,
7766 }
7767 }
7768
7769 fn decode_pool_meta_payload(version: u16, payload: &[u8]) -> Result<Self> {
7770 match version {
7771 POOL_META_VERSION => rmp_serde::from_slice::<PersistedPoolMeta>(payload)
7772 .map_err(|err| Error::other(format!("PoolMeta v{POOL_META_VERSION} decode failed: {err}")))?
7773 .try_into(),
7774 POOL_META_V1_VERSION => match rmp_serde::from_slice::<PersistedPoolMetaV1>(payload) {
7775 Ok(meta) => meta.try_into(),
7776 Err(persisted_err) => {
7777 let legacy: LegacyPoolMeta = rmp_serde::from_slice(payload).map_err(|legacy_err| {
7778 Error::other(format!(
7779 "PoolMeta v1 decode failed for both persisted and legacy formats: persisted={persisted_err}; legacy={legacy_err}"
7780 ))
7781 })?;
7782 legacy.try_into()
7783 }
7784 },
7785 _ => Err(Error::other(format!("pool metadata load failed: unknown version {version}"))),
7786 }
7787 }
7788
7789 pub fn new(pools: &[Arc<Sets>], prev_meta: &PoolMeta) -> Self {
7790 let mut new_meta = Self {
7791 version: if prev_meta.version == POOL_META_GENERATION_VERSION || pool_meta_v3_writer_enabled() {
7792 POOL_META_GENERATION_VERSION
7793 } else if prev_meta.version == POOL_META_VERSION || pool_meta_v2_writer_enabled() {
7794 POOL_META_VERSION
7795 } else {
7796 POOL_META_V1_VERSION
7797 },
7798 pools: Vec::new(),
7799 ..Default::default()
7800 };
7801
7802 for (idx, pool) in pools.iter().enumerate() {
7803 let mut skip = false;
7804
7805 for current_pool in prev_meta.pools.iter() {
7806 if current_pool.cmd_line == pool.endpoints.cmd_line {
7807 new_meta.pools.push(current_pool.clone());
7808 skip = true;
7809 break;
7810 }
7811 }
7812
7813 if skip {
7814 continue;
7815 }
7816
7817 new_meta.pools.push(PoolStatus {
7818 cmd_line: pool.endpoints.cmd_line.clone(),
7819 id: idx,
7820 last_update: OffsetDateTime::now_utc(),
7821 decommission: None,
7822 });
7823 }
7824
7825 new_meta
7826 }
7827
7828 pub fn is_suspended(&self, idx: usize) -> bool {
7829 self.pools
7830 .get(idx)
7831 .and_then(|pool| pool.decommission.as_ref())
7832 .is_some_and(is_decommission_suspended)
7833 }
7834
7835 pub(crate) fn has_active_decommission_capacity_reservation(&self, idx: usize) -> bool {
7836 self.pools
7837 .get(idx)
7838 .and_then(|pool| pool.decommission.as_ref())
7839 .is_some_and(|info| {
7840 info.has_decommission_state()
7841 && is_decommission_active(info.complete, info.failed, info.canceled)
7842 && info
7843 .capacity_reservation
7844 .as_ref()
7845 .is_some_and(DecommissionCapacityReservation::active)
7846 })
7847 }
7848
7849 pub(crate) fn scanner_pause_backlog_pool_writable(&self, idx: usize) -> bool {
7850 self.pools.get(idx).is_some_and(|pool| {
7851 !pool
7852 .decommission
7853 .as_ref()
7854 .is_some_and(|info| info.has_decommission_state() && !info.failed && !info.canceled)
7855 })
7856 }
7857
7858 fn mark_decommission_progress_saved(&mut self) {
7859 for pool in &mut self.pools {
7860 if let Some(info) = pool.decommission.as_mut() {
7861 info.mark_progress_saved();
7862 info.progress_save_last_at = Some(pool.last_update);
7863 }
7864 }
7865 }
7866
7867 fn decommission_progress_checkpoint(
7868 &self,
7869 idx: usize,
7870 duration: Duration,
7871 now: OffsetDateTime,
7872 rebalance_meta: Option<&RebalanceMeta>,
7873 ) -> Result<Option<DecommissionProgressCheckpoint>> {
7874 let pool_count = self.pools.len();
7875 ensure_valid_decommission_pool_index(pool_count, idx)?;
7876
7877 let Some(pool) = self.pools.get(idx) else {
7878 return Err(invalid_decommission_pool_index_error(pool_count, idx));
7879 };
7880 let Some(info) = pool.decommission.as_ref() else {
7881 return Err(decommission_metadata_not_initialized_error("update decommission metadata timestamp"));
7882 };
7883
7884 if info.progress_save_retry_after.is_some_and(|retry_after| now < retry_after) {
7885 return Ok(None);
7886 }
7887
7888 let last_progress_save = info.progress_save_last_at.unwrap_or(pool.last_update);
7889 let time_threshold_reached = now.unix_timestamp() - last_progress_save.unix_timestamp() >= duration.whole_seconds();
7890 let item_threshold_reached = info.items_since_last_progress_save() >= DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD;
7891 if !time_threshold_reached && !item_threshold_reached {
7892 return Ok(None);
7893 }
7894
7895 Ok(Some(DecommissionProgressCheckpoint {
7896 start_time: info.start_time,
7897 queued: info.queued,
7898 counted_items: info.counted_items(),
7899 checkpoint_at: self.next_scanner_data_movement_update(now, rebalance_meta),
7900 capacity_operation_id: info
7901 .capacity_reservation
7902 .as_ref()
7903 .filter(|reservation| reservation.lease_active_at(now))
7904 .map(|reservation| reservation.operation_id),
7905 capacity_owner_nonce: info
7906 .capacity_reservation
7907 .as_ref()
7908 .filter(|reservation| reservation.lease_active_at(now))
7909 .map(|reservation| reservation.owner_nonce),
7910 capacity_lease_expires_at: info
7911 .capacity_reservation
7912 .as_ref()
7913 .filter(|reservation| reservation.lease_active_at(now))
7914 .map(|_| now + DECOMMISSION_CAPACITY_RESERVATION_TTL),
7915 }))
7916 }
7917
7918 fn commit_decommission_progress_checkpoint(&mut self, idx: usize, checkpoint: DecommissionProgressCheckpoint) -> bool {
7919 let Some(pool) = self.pools.get_mut(idx) else {
7920 return false;
7921 };
7922 let Some(info) = pool.decommission.as_mut() else {
7923 return false;
7924 };
7925
7926 if info.start_time != checkpoint.start_time
7927 || info.queued != checkpoint.queued
7928 || !is_decommission_active(info.complete, info.failed, info.canceled)
7929 {
7930 return false;
7931 }
7932
7933 info.progress_save_item_baseline = info.progress_save_item_baseline.max(checkpoint.counted_items);
7934 info.progress_save_retry_after = None;
7935 info.progress_save_last_at = Some(
7936 info.progress_save_last_at
7937 .map_or(checkpoint.checkpoint_at, |at| at.max(checkpoint.checkpoint_at)),
7938 );
7939 if let (Some(operation_id), Some(owner_nonce), Some(expires_at), Some(reservation)) = (
7940 checkpoint.capacity_operation_id,
7941 checkpoint.capacity_owner_nonce,
7942 checkpoint.capacity_lease_expires_at,
7943 info.capacity_reservation.as_mut(),
7944 ) && reservation.operation_id == operation_id
7945 && reservation.owner_nonce == owner_nonce
7946 && reservation.active()
7947 {
7948 reservation.renewed_at = checkpoint.checkpoint_at;
7949 reservation.expires_at = expires_at;
7950 }
7951 pool.last_update = pool.last_update.max(checkpoint.checkpoint_at);
7952 true
7953 }
7954
7955 fn defer_decommission_progress_checkpoint(
7956 &mut self,
7957 idx: usize,
7958 checkpoint: DecommissionProgressCheckpoint,
7959 retry_after: OffsetDateTime,
7960 ) {
7961 let Some(pool) = self.pools.get_mut(idx) else {
7962 return;
7963 };
7964 let Some(info) = pool.decommission.as_mut() else {
7965 return;
7966 };
7967
7968 if info.start_time == checkpoint.start_time
7969 && info.queued == checkpoint.queued
7970 && is_decommission_active(info.complete, info.failed, info.canceled)
7971 {
7972 info.progress_save_retry_after = Some(retry_after);
7973 }
7974 }
7975
7976 fn load_from_config_data(&mut self, data: Vec<u8>) -> Result<()> {
7977 if data.is_empty() {
7978 return Ok(());
7979 } else if data.len() <= 4 {
7980 return Err(Error::other("pool metadata load failed: metadata payload is too short"));
7981 }
7982
7983 let format = LittleEndian::read_u16(&data[0..2]);
7984 if format != POOL_META_FORMAT {
7985 return Err(Error::other(format!("pool metadata load failed: unknown format {format}")));
7986 }
7987 let version = LittleEndian::read_u16(&data[2..4]);
7988 if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) {
7989 return Err(Error::other(format!("pool metadata load failed: unknown version {version}")));
7990 }
7991
7992 if version == POOL_META_GENERATION_VERSION {
7993 let PoolMetaReplica::Valid {
7994 meta, committed: true, ..
7995 } = decode_pool_meta_replica(data)
7996 else {
7997 return Err(Error::other(
7998 "pool metadata load failed: V3 payload is corrupt, incompatible, or not committed",
7999 ));
8000 };
8001 *self = meta;
8002 return Ok(());
8003 }
8004
8005 *self = Self::decode_pool_meta_payload(version, &data[4..])?;
8006
8007 if !matches!(self.version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) {
8008 return Err(Error::other(format!(
8009 "pool metadata load failed: unexpected decoded version {}",
8010 self.version
8011 )));
8012 }
8013 Ok(())
8014 }
8015
8016 pub async fn load(&mut self, pool: Arc<Sets>, pools: Vec<Arc<Sets>>) -> Result<()> {
8017 let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
8018 let _pool_meta_guard = pool_meta_lock.get_read_lock(get_lock_acquire_timeout()).await?;
8019 let replica_state = self.load_no_lock_from_replicas(pools).await?;
8020 replica_state.ensure_write_safe("pool metadata load failed")?;
8021 Ok(())
8022 }
8023
8024 pub(crate) async fn load_no_lock_from_replicas<S>(&mut self, pools: Vec<Arc<S>>) -> Result<PoolMetaReplicaState>
8027 where
8028 S: EcstoreObjectIO,
8029 {
8030 let selection = load_pool_meta_replicas(pools, true).await?;
8031 *self = selection.meta;
8032 Ok(selection.replica_state)
8033 }
8034
8035 pub(crate) async fn load_no_lock_from_replicas_observing<S>(
8036 &mut self,
8037 pools: Vec<Arc<S>>,
8038 write_state: &mut PoolMetaWriteState,
8039 ) -> Result<PoolMetaReplicaState>
8040 where
8041 S: EcstoreObjectIO,
8042 {
8043 let selection = load_pool_meta_replicas_observing(pools, true, write_state).await?;
8044 *self = selection.meta;
8045 Ok(selection.replica_state)
8046 }
8047
8048 pub(crate) async fn load_for_startup_observing<S>(
8049 &mut self,
8050 pools: Vec<Arc<S>>,
8051 write_state: &mut PoolMetaWriteState,
8052 ) -> Result<PoolMetaReplicaState>
8053 where
8054 S: EcstoreObjectIO,
8055 {
8056 if let Some(cluster_id) = write_state.expected_cluster_id {
8059 let reads = join_all(pools.iter().cloned().map(read_pool_meta_identity_replica)).await;
8060 if let Some(source) = reads.iter().find_map(|read| match &read.replica {
8061 PoolMetaIdentityReplica::Unreadable(source) => Some(Arc::clone(source)),
8062 _ => None,
8063 }) {
8064 write_state.block_with_context(pool_metadata_error(
8065 crate::error::PoolMetadataFailure::RecoveryRequired,
8066 "startup_identity_read",
8067 Some(source),
8068 ));
8069 }
8070 let identity = select_pool_meta_identity(reads, cluster_id)
8071 .map_err(|err| block_pool_meta_validation(write_state, err, "startup_identity"))?;
8072 write_state.observe_identity(&identity)?;
8073 }
8074 let reads = read_pool_meta_replicas(pools, true).await;
8075 if let Some(err) = pool_meta_replica_read_failure(&reads, "pool metadata load") {
8076 write_state.block_with_context(pool_metadata_error(
8077 crate::error::PoolMetadataFailure::RecoveryRequired,
8078 "startup_read",
8079 Some(Arc::new(err)),
8080 ));
8081 }
8082 let selection = select_pool_meta_replica_reads(reads)
8083 .map_err(|err| block_pool_meta_validation(write_state, err, "startup_selection"))?;
8084 write_state
8085 .observe_selection(&selection)
8086 .map_err(|err| block_pool_meta_validation(write_state, err, "startup_selection"))?;
8087 *self = selection.meta;
8088 Ok(selection.replica_state)
8089 }
8090
8091 #[cfg(test)]
8092 fn encode_config_data(&self) -> Result<Vec<u8>> {
8093 self.encode_config_data_for_v2_gate(pool_meta_v2_writer_enabled())
8094 }
8095
8096 fn encode_config_data_for_v2_gate(&self, v2_enabled: bool) -> Result<Vec<u8>> {
8097 if self.dont_save {
8098 return Ok(Vec::new());
8099 }
8100 if self.version == POOL_META_GENERATION_VERSION {
8101 return Err(Error::other(
8102 "pool metadata V3 save requires cluster identity and generation transaction context",
8103 ));
8104 }
8105 if !matches!(self.version, 0 | POOL_META_V1_VERSION | POOL_META_VERSION) {
8106 return Err(Error::other(format!(
8107 "pool metadata save failed: unexpected runtime version {}",
8108 self.version
8109 )));
8110 }
8111 let version = if self.version == POOL_META_VERSION || v2_enabled {
8112 POOL_META_VERSION
8113 } else {
8114 POOL_META_V1_VERSION
8115 };
8116 if version == POOL_META_V1_VERSION {
8117 let mut decommission_infos = self.pools.iter().filter_map(|pool| pool.decommission.as_ref());
8118 if decommission_infos.clone().any(|info| !info.unresolved_entries.is_empty()) {
8119 return Err(Error::other(format!(
8120 "pool metadata V2 is required to persist unresolved decommission entries; enable both {} and {} only after every reader and writer supports V2",
8121 rustfs_config::ENV_POOL_META_V2_WRITE,
8122 rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
8123 )));
8124 }
8125 if decommission_infos.any(|info| info.capacity_reservation.is_some()) {
8126 return Err(Error::DecommissionCapacity(format!(
8127 "pool metadata V2 is required to persist decommission capacity reservations; enable both {} and {} only after every reader and writer supports V2",
8128 rustfs_config::ENV_POOL_META_V2_WRITE,
8129 rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
8130 )));
8131 }
8132 }
8133 let mut data = Vec::new();
8134 data.write_u16::<LittleEndian>(POOL_META_FORMAT)?;
8135 data.write_u16::<LittleEndian>(version)?;
8136 let mut buf = Vec::new();
8137 match version {
8138 POOL_META_V1_VERSION => PersistedPoolMetaV1::from(self).serialize(&mut Serializer::new(&mut buf))?,
8139 POOL_META_VERSION => PersistedPoolMeta::from(self).serialize(&mut Serializer::new(&mut buf))?,
8140 _ => unreachable!("pool metadata writer selected an unsupported version"),
8141 }
8142 data.write_all(&buf)?;
8143 Ok(data)
8144 }
8145
8146 #[cfg(test)]
8147 pub(crate) fn encode_config_data_for_test(&self) -> Result<Vec<u8>> {
8148 self.encode_config_data_for_v2_gate(true)
8149 }
8150
8151 pub async fn save(&self, pools: Vec<Arc<Sets>>) -> Result<()> {
8152 let pool = pools
8153 .first()
8154 .cloned()
8155 .ok_or_else(|| Error::other("pool metadata save failed: no storage pools available"))?;
8156 let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
8157 let pool_meta_guard = pool_meta_lock.get_write_lock(get_lock_acquire_timeout()).await?;
8158 let mut write_state = PoolMetaWriteState::default();
8159 let indices = (0..self.pools.len()).collect::<Vec<_>>();
8160 let outcome = self
8161 .save_no_lock_armed_scoped(pools, &mut write_state, pool_meta_guard.lock_lost_signal(), Some(&indices))
8162 .await?;
8163 outcome.disarm();
8164 Ok(())
8165 }
8166
8167 #[cfg(any(test, feature = "test-util"))]
8169 pub(crate) async fn save_for_startup<S>(&self, pools: Vec<Arc<S>>) -> Result<()>
8170 where
8171 S: EcstoreObjectIO,
8172 {
8173 let mut write_state = PoolMetaWriteState::for_test_bootstrap();
8174 self.save_for_startup_observing(pools, &mut write_state).await.map(|_| ())
8175 }
8176
8177 pub(crate) async fn save_for_startup_observing<S>(
8178 &self,
8179 pools: Vec<Arc<S>>,
8180 write_state: &mut PoolMetaWriteState,
8181 ) -> Result<PoolMeta>
8182 where
8183 S: EcstoreObjectIO,
8184 {
8185 let outcome = self.save_no_lock_armed_scoped(pools, write_state, None, None).await?;
8186 Ok(outcome.into_committed())
8187 }
8188
8189 async fn save_no_lock_with_fence<S>(
8190 &self,
8191 pools: Vec<Arc<S>>,
8192 lock_lost: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
8193 indices: &[usize],
8194 ) -> Result<()>
8195 where
8196 S: EcstoreObjectIO,
8197 {
8198 let mut write_state = PoolMetaWriteState::default();
8199 let outcome = self
8200 .save_no_lock_armed_scoped(pools, &mut write_state, lock_lost, Some(indices))
8201 .await?;
8202 outcome.disarm();
8203 Ok(())
8204 }
8205
8206 async fn save_no_lock_with_activation_fence<S>(
8207 &self,
8208 pools: Vec<Arc<S>>,
8209 write_state: &mut PoolMetaWriteState,
8210 activation_fence: &PoolRebalanceActivationFence,
8211 indices: &[usize],
8212 ) -> Result<PoolMetaSaveOutcome>
8213 where
8214 S: EcstoreObjectIO,
8215 {
8216 write_state.ensure_write_safe("pool metadata activation save failed")?;
8217 let mut transaction_arm = write_state.arm_transaction();
8218 let fence = PoolMetaPersistenceFence::Activation(activation_fence);
8219 let committed = self
8220 .save_no_lock_transaction(pools, write_state, &fence, Some(indices), &mut transaction_arm)
8221 .await
8222 .map_err(|err| transaction_arm.failure(err))?;
8223 if transaction_arm.phase.is_some() {
8224 transaction_arm.phase = Some("publication");
8225 }
8226 Ok(PoolMetaSaveOutcome {
8227 transaction_arm,
8228 committed,
8229 })
8230 }
8231
8232 async fn save_no_lock_armed<S>(
8233 &self,
8234 pools: Vec<Arc<S>>,
8235 write_state: &mut PoolMetaWriteState,
8236 lock_lost: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
8237 indices: &[usize],
8238 ) -> Result<PoolMetaSaveOutcome>
8239 where
8240 S: EcstoreObjectIO,
8241 {
8242 self.save_no_lock_armed_scoped(pools, write_state, lock_lost, Some(indices))
8243 .await
8244 }
8245
8246 async fn save_no_lock_armed_scoped<S>(
8247 &self,
8248 pools: Vec<Arc<S>>,
8249 write_state: &mut PoolMetaWriteState,
8250 lock_lost: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
8251 indices: Option<&[usize]>,
8252 ) -> Result<PoolMetaSaveOutcome>
8253 where
8254 S: EcstoreObjectIO,
8255 {
8256 write_state.ensure_write_safe("pool metadata save failed")?;
8257 let mut transaction_arm = write_state.arm_transaction();
8261 let fence = PoolMetaPersistenceFence::Distributed(lock_lost);
8262 let committed = self
8263 .save_no_lock_transaction(pools, write_state, &fence, indices, &mut transaction_arm)
8264 .await
8265 .map_err(|err| transaction_arm.failure(err))?;
8266 if transaction_arm.phase.is_some() {
8267 transaction_arm.phase = Some("publication");
8268 }
8269 Ok(PoolMetaSaveOutcome {
8270 transaction_arm,
8271 committed,
8272 })
8273 }
8274
8275 async fn save_no_lock_transaction<S>(
8276 &self,
8277 pools: Vec<Arc<S>>,
8278 write_state: &mut PoolMetaWriteState,
8279 fence: &PoolMetaPersistenceFence<'_>,
8280 indices: Option<&[usize]>,
8281 transaction_arm: &mut PoolMetaTransactionArm,
8282 ) -> Result<PoolMeta>
8283 where
8284 S: EcstoreObjectIO,
8285 {
8286 if pools.is_empty() {
8287 return Err(Error::other("pool metadata save failed: no storage pools available"));
8288 }
8289 if self.dont_save {
8290 return Ok(self.clone());
8291 }
8292 for attempt in 0..POOL_META_CAS_MAX_ATTEMPTS {
8293 match self
8294 .save_no_lock_transaction_once(pools.clone(), write_state, fence, indices, transaction_arm)
8295 .await
8296 {
8297 Ok(committed) => return Ok(committed),
8298 Err(Error::PreconditionFailed) if attempt + 1 < POOL_META_CAS_MAX_ATTEMPTS => continue,
8299 Err(err) => return Err(err),
8300 }
8301 }
8302 Err(Error::PreconditionFailed)
8303 }
8304
8305 async fn save_no_lock_transaction_once<S>(
8306 &self,
8307 pools: Vec<Arc<S>>,
8308 write_state: &mut PoolMetaWriteState,
8309 fence: &PoolMetaPersistenceFence<'_>,
8310 indices: Option<&[usize]>,
8311 transaction_arm: &mut PoolMetaTransactionArm,
8312 ) -> Result<PoolMeta>
8313 where
8314 S: EcstoreObjectIO,
8315 {
8316 fence.ensure_held()?;
8317 let selection = load_pool_meta_replicas_observing(pools.clone(), true, write_state).await?;
8318 write_state.observe_replicas(selection.replica_state);
8319 write_state.ensure_write_safe("pool metadata save failed")?;
8320 let mut bootstrap_generation_required = false;
8321 if let Some(cluster_id) = write_state.expected_cluster_id {
8322 let identity = load_pool_meta_identity_selection_observing(pools.clone(), write_state, cluster_id).await?;
8323 bootstrap_generation_required = selection.absent || write_state.identity_initialized == Some(false);
8324 if !identity.repair_write_safe {
8325 write_state.block_writes();
8326 return Err(Error::other(
8327 "pool metadata recovery required: cluster identity has an unreadable replica",
8328 ));
8329 }
8330 if let Some(identity) = identity.identity
8331 && selection.revision.is_generation_protocol()
8332 && identity.epoch != selection.revision.epoch
8333 {
8334 write_state.block_writes();
8335 return Err(Error::other(format!(
8336 "pool metadata recovery required: committed epoch {} does not match cluster identity epoch {}",
8337 selection.revision.epoch, identity.epoch
8338 )));
8339 }
8340 if !selection.absent && write_state.identity_requires_repair() {
8341 let initialized = write_state.identity_initialized != Some(false) || selection.revision.is_generation_protocol();
8342 persist_pool_meta_identity(
8343 pools.clone(),
8344 write_state,
8345 initialized,
8346 PoolMetaIdentityWriteScope::All,
8347 fence,
8348 transaction_arm,
8349 )
8350 .await?;
8351 }
8352 }
8353 write_state.ensure_missing_metadata_can_initialize()?;
8356
8357 let mut committed = if let Some(indices) = indices {
8358 if selection.meta.pools.is_empty() {
8359 self.clone()
8360 } else {
8361 let mut requested = self.clone();
8362 requested.version = selection.meta.version;
8363 let mut latest = selection.meta.clone();
8364 merge_pool_meta_updates_for_save(&mut latest, &requested, indices, "pool metadata save failed")?;
8365 latest
8366 }
8367 } else {
8368 self.clone()
8369 };
8370 let target_version = if selection.generation_protocol_observed
8371 || selection.revision.is_generation_protocol()
8372 || pool_meta_v3_writer_enabled()
8373 || bootstrap_generation_required
8374 {
8375 POOL_META_GENERATION_VERSION
8376 } else if selection.meta.version == POOL_META_VERSION
8377 || self.version == POOL_META_VERSION
8378 || pool_meta_v2_writer_enabled()
8379 {
8380 POOL_META_VERSION
8381 } else {
8382 POOL_META_V1_VERSION
8383 };
8384 committed.version = target_version;
8385
8386 if target_version != POOL_META_GENERATION_VERSION {
8387 let data = committed.encode_config_data_for_v2_gate(target_version == POOL_META_VERSION)?;
8388 let mut canonical_saved = false;
8389 for (pool_index, (pool, token)) in pools.iter().cloned().zip(&selection.cas_tokens).enumerate() {
8390 let result = save_pool_meta_object_cas(
8391 pool.clone(),
8392 POOL_META_NAME,
8393 data.clone(),
8394 token,
8395 fence,
8396 "legacy_cas",
8397 transaction_arm,
8398 )
8399 .await;
8400 if result.is_ok() && pool_index == 0 {
8401 canonical_saved = true;
8402 #[cfg(test)]
8403 if let PoolMetaPersistenceFence::Activation(activation_fence) = fence {
8404 pause_pool_activation_after_durable_save(&pool, activation_fence).await;
8405 }
8406 }
8407 if let Err(err) = result {
8408 if canonical_saved && fence.is_activation() {
8409 warn!(
8410 event = EVENT_DECOMMISSION_STATE,
8411 component = LOG_COMPONENT_ECSTORE,
8412 subsystem = LOG_SUBSYSTEM_POOLS,
8413 pool_index,
8414 state = "activation_replica_repair_pending",
8415 error = %err,
8416 "Decommission activation replica repair pending"
8417 );
8418 continue;
8419 }
8420 return Err(err);
8421 }
8422 }
8423 let confirmed = if fence.is_activation() {
8424 load_pool_meta_replicas(pools, true).await?
8425 } else {
8426 let confirmed = load_pool_meta_replicas_observing(pools, true, write_state).await?;
8427 write_state.observe_replicas(confirmed.replica_state);
8428 confirmed
8429 };
8430 let expected = committed.encode_config_data_for_v2_gate(true)?;
8431 if confirmed.canonical.as_ref() != Some(&expected) {
8432 record_pool_meta_stale_write_rejection("legacy_verify");
8433 return Err(Error::PreconditionFailed);
8434 }
8435 return Ok(confirmed.meta);
8436 }
8437
8438 let cluster_id = selection
8439 .revision
8440 .cluster_id
8441 .or(selection.generation_identity.map(|(cluster_id, _)| cluster_id))
8442 .or(write_state.expected_cluster_id)
8443 .ok_or_else(|| Error::other("pool metadata V3 save failed: cluster identity is not initialized"))?;
8444 let epoch = if selection.revision.is_generation_protocol() {
8445 selection.revision.epoch
8446 } else {
8447 selection
8448 .generation_identity
8449 .map(|(_, epoch)| epoch)
8450 .or(write_state.cluster_epoch)
8451 .unwrap_or(POOL_META_INITIAL_EPOCH)
8452 };
8453 let generation = if selection.revision.is_generation_protocol() {
8454 selection
8455 .revision
8456 .generation
8457 .checked_add(1)
8458 .ok_or_else(|| Error::other("pool metadata V3 generation exhausted"))?
8459 } else {
8460 1
8461 };
8462 let revision = PoolMetaRevision {
8463 version: POOL_META_GENERATION_VERSION,
8464 cluster_id: Some(cluster_id),
8465 epoch,
8466 generation,
8467 transaction_id: Some(uuid::Uuid::new_v4()),
8468 };
8469 let previous = if let Some(canonical) = selection.canonical.clone() {
8470 PoolMetaCommittedCandidate {
8471 canonical,
8472 meta: selection.meta.clone(),
8473 revision: selection.revision,
8474 }
8475 } else {
8476 let empty = PoolMeta {
8477 version: POOL_META_V1_VERSION,
8478 ..Default::default()
8479 };
8480 PoolMetaCommittedCandidate {
8481 canonical: empty.encode_config_data_for_v2_gate(true)?,
8482 meta: empty,
8483 revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
8484 }
8485 };
8486 let pending = encode_pool_meta_v3_envelope(&committed, revision, false, Some(&previous))?;
8487 let durable = encode_pool_meta_v3_envelope(&committed, revision, true, None)?;
8488 let concurrent_phases = pools.len() > 1
8489 && selection.revision.is_generation_protocol()
8490 && !selection.replica_state.needs_repair
8491 && !bootstrap_generation_required
8492 && matches!(fence, PoolMetaPersistenceFence::Distributed(Some(_)));
8493 #[cfg(feature = "e2e-test-hooks")]
8494 let phase_barrier = if concurrent_phases && pools.len() == 4 {
8495 PoolMetaPhaseBarrier::bind(&previous, &committed, revision, &durable).await?
8496 } else {
8497 None
8498 };
8499 let mut pending_tokens = Vec::with_capacity(pools.len());
8500 if concurrent_phases {
8501 for result in save_pool_meta_phase(
8503 pools.as_slice(),
8504 &pending,
8505 &selection.cas_tokens,
8506 fence,
8507 "prepare_cas",
8508 transaction_arm,
8509 #[cfg(feature = "e2e-test-hooks")]
8510 phase_barrier.as_ref(),
8511 )
8512 .await
8513 {
8514 let etag = result?
8515 .etag
8516 .filter(|etag| !etag.trim().is_empty())
8517 .ok_or_else(|| Error::other("pool metadata V3 prepare succeeded without a conditional-write revision"))?;
8518 pending_tokens.push(PoolMetaCasToken::Existing(etag));
8519 }
8520 } else {
8521 for (pool, token) in pools.iter().cloned().zip(&selection.cas_tokens) {
8522 let object_info = save_pool_meta_object_cas(
8523 pool,
8524 POOL_META_NAME,
8525 pending.clone(),
8526 token,
8527 fence,
8528 "prepare_cas",
8529 transaction_arm,
8530 )
8531 .await?;
8532 let etag = object_info
8533 .etag
8534 .filter(|etag| !etag.trim().is_empty())
8535 .ok_or_else(|| Error::other("pool metadata V3 prepare succeeded without a conditional-write revision"))?;
8536 pending_tokens.push(PoolMetaCasToken::Existing(etag));
8537 }
8538 }
8539
8540 #[cfg(feature = "e2e-test-hooks")]
8541 if let Some(barrier) = &phase_barrier
8542 && barrier.arm.case == PoolMetaPhaseBarrierCase::PreparedAll
8543 {
8544 barrier.ready().await?;
8545 }
8546 let mut commit_error = None;
8547 let mut commit_succeeded = false;
8548 if concurrent_phases {
8549 for result in save_pool_meta_phase(
8550 pools.as_slice(),
8551 &durable,
8552 &pending_tokens,
8553 fence,
8554 "commit_cas",
8555 transaction_arm,
8556 #[cfg(feature = "e2e-test-hooks")]
8557 phase_barrier.as_ref(),
8558 )
8559 .await
8560 {
8561 match result {
8562 Ok(_) => commit_succeeded = true,
8563 Err(err) => {
8564 commit_error.get_or_insert(err);
8565 }
8566 }
8567 }
8568 } else {
8569 #[cfg(test)]
8570 let mut first_pool = true;
8571 for (pool, token) in pools.iter().cloned().zip(&pending_tokens) {
8572 match save_pool_meta_object_cas(
8573 pool.clone(),
8574 POOL_META_NAME,
8575 durable.clone(),
8576 token,
8577 fence,
8578 "commit_cas",
8579 transaction_arm,
8580 )
8581 .await
8582 {
8583 Ok(_) => {
8584 commit_succeeded = true;
8585 #[cfg(test)]
8586 if first_pool && let PoolMetaPersistenceFence::Activation(activation_fence) = fence {
8587 pause_pool_activation_after_durable_save(&pool, activation_fence).await;
8588 }
8589 }
8590 Err(err) => {
8591 commit_error.get_or_insert(err);
8592 }
8593 }
8594 #[cfg(test)]
8595 {
8596 first_pool = false;
8597 }
8598 }
8599 }
8600 let confirmed = if fence.is_activation() {
8601 load_pool_meta_replicas(pools.clone(), true).await?
8602 } else {
8603 let confirmed = load_pool_meta_replicas_observing(pools.clone(), true, write_state).await?;
8604 write_state.observe_replicas(confirmed.replica_state);
8605 confirmed
8606 };
8607 if confirmed.revision == revision && confirmed.canonical.as_ref() == Some(&durable) {
8608 persist_pool_meta_identity(pools, write_state, true, PoolMetaIdentityWriteScope::All, fence, transaction_arm).await?;
8609 #[cfg(feature = "e2e-test-hooks")]
8610 startup_cas_test_observe(serde_json::json!({
8611 "kind": "confirmed", "object": POOL_META_NAME,
8612 "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&durable)),
8613 "generation": confirmed.revision.generation,
8614 "transaction_id": confirmed.revision.transaction_id,
8615 }));
8616 #[cfg(feature = "e2e-test-hooks")]
8617 if let Some(barrier) = &phase_barrier
8618 && barrier.arm.case == PoolMetaPhaseBarrierCase::BeforePublish
8619 {
8620 barrier.ready().await?;
8621 }
8622 return Ok(confirmed.meta);
8623 }
8624 if !commit_succeeded {
8625 return Err(commit_error.unwrap_or(Error::PreconditionFailed));
8626 }
8627 Err(commit_error.unwrap_or_else(|| {
8628 Error::other("pool metadata recovery required: committed V3 transaction was not selected after write")
8629 }))
8630 }
8631
8632 #[cfg(test)]
8633 async fn save_no_lock_observing<S>(&self, pools: Vec<Arc<S>>, write_state: &mut PoolMetaWriteState) -> Result<()>
8634 where
8635 S: EcstoreObjectIO,
8636 {
8637 let indices = (0..self.pools.len()).collect::<Vec<_>>();
8638 let outcome = self.save_no_lock_armed(pools, write_state, None, &indices).await?;
8639 outcome.disarm();
8640 Ok(())
8641 }
8642
8643 pub fn decommission_cancel(&mut self, idx: usize) -> bool {
8644 self.decommission_cancel_at(idx, OffsetDateTime::now_utc(), None)
8645 }
8646
8647 #[cfg(test)]
8648 pub(crate) fn decommission_cancel_at_for_test(
8649 &mut self,
8650 idx: usize,
8651 now: OffsetDateTime,
8652 rebalance_meta: Option<&RebalanceMeta>,
8653 ) -> bool {
8654 self.decommission_cancel_at(idx, now, rebalance_meta)
8655 }
8656
8657 fn decommission_cancel_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool {
8658 let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else {
8659 return false;
8660 };
8661 if d.canceled {
8662 return false;
8663 }
8664
8665 let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8666 let mut pd = d.clone();
8667 pd.canceled = true;
8668 pd.failed = false;
8669 pd.complete = false;
8670 pd.start_time = None;
8671 pd.terminal_reload_attempt_at = None;
8672 pd.terminal_reload_failures.clear();
8673 pd.capacity_blocked_reason = None;
8674 release_decommission_capacity_reservation(&mut pd, DECOMMISSION_CAPACITY_RELEASE_CANCELED, last_update);
8675
8676 let Some(stats) = self.pools.get_mut(idx) else {
8677 return false;
8678 };
8679 stats.last_update = last_update;
8680 stats.decommission = Some(pd);
8681 true
8682 }
8683 pub fn decommission_failed(&mut self, idx: usize) -> bool {
8684 self.decommission_failed_at(idx, OffsetDateTime::now_utc(), None)
8685 }
8686
8687 #[cfg(test)]
8688 pub(crate) fn decommission_failed_at_for_test(
8689 &mut self,
8690 idx: usize,
8691 now: OffsetDateTime,
8692 rebalance_meta: Option<&RebalanceMeta>,
8693 ) -> bool {
8694 self.decommission_failed_at(idx, now, rebalance_meta)
8695 }
8696
8697 fn decommission_failed_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool {
8698 let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else {
8699 return false;
8700 };
8701 if !is_decommission_active(d.complete, d.failed, d.canceled) {
8702 return false;
8703 }
8704
8705 let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8706 let mut pd = d.clone();
8707 pd.canceled = false;
8708 pd.failed = true;
8709 pd.complete = false;
8710 pd.start_time = None;
8711 pd.terminal_reload_attempt_at = None;
8712 pd.terminal_reload_failures.clear();
8713 pd.capacity_blocked_reason = None;
8714 release_decommission_capacity_reservation(&mut pd, DECOMMISSION_CAPACITY_RELEASE_FAILED, last_update);
8715
8716 let Some(stats) = self.pools.get_mut(idx) else {
8717 return false;
8718 };
8719 stats.last_update = last_update;
8720 stats.decommission = Some(pd);
8721 true
8722 }
8723
8724 pub fn clear_decommission(&mut self, idx: usize) -> Result<bool> {
8725 self.clear_decommission_at(idx, OffsetDateTime::now_utc(), None)
8726 }
8727
8728 #[cfg(test)]
8729 pub(crate) fn clear_decommission_at_for_test(
8730 &mut self,
8731 idx: usize,
8732 now: OffsetDateTime,
8733 rebalance_meta: Option<&RebalanceMeta>,
8734 ) -> Result<bool> {
8735 self.clear_decommission_at(idx, now, rebalance_meta)
8736 }
8737
8738 fn clear_decommission_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> Result<bool> {
8739 let pool_count = self.pools.len();
8740 ensure_valid_decommission_pool_index(pool_count, idx)?;
8741
8742 let Some(pool) = self.pools.get(idx) else {
8743 return Err(invalid_decommission_pool_index_error(pool_count, idx));
8744 };
8745
8746 let (decommission_present, complete, failed, canceled, unresolved_entries) = pool
8747 .decommission
8748 .as_ref()
8749 .map(|info| {
8750 (
8751 info.has_decommission_state(),
8752 info.complete,
8753 info.failed,
8754 info.canceled,
8755 info.unresolved_entries.len(),
8756 )
8757 })
8758 .unwrap_or((false, false, false, false, 0));
8759
8760 ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled, unresolved_entries)?;
8761
8762 let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8763 let Some(pool) = self.pools.get_mut(idx) else {
8764 return Err(invalid_decommission_pool_index_error(pool_count, idx));
8765 };
8766 pool.last_update = last_update;
8767 pool.decommission = Some(PoolDecommissionInfo::default());
8770 Ok(true)
8771 }
8772
8773 pub fn decommission_complete(&mut self, idx: usize) -> bool {
8774 self.decommission_complete_at(idx, OffsetDateTime::now_utc(), None)
8775 }
8776
8777 #[cfg(test)]
8778 pub(crate) fn decommission_complete_at_for_test(
8779 &mut self,
8780 idx: usize,
8781 now: OffsetDateTime,
8782 rebalance_meta: Option<&RebalanceMeta>,
8783 ) -> bool {
8784 self.decommission_complete_at(idx, now, rebalance_meta)
8785 }
8786
8787 fn decommission_complete_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool {
8788 let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else {
8789 return false;
8790 };
8791 if !is_decommission_active(d.complete, d.failed, d.canceled) {
8792 return false;
8793 }
8794
8795 let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8796 let mut pd = d.clone();
8797 pd.canceled = false;
8798 pd.failed = false;
8799 pd.complete = true;
8800 pd.terminal_reload_attempt_at = None;
8801 pd.terminal_reload_failures.clear();
8802 pd.capacity_blocked_reason = None;
8803 release_decommission_capacity_reservation(&mut pd, DECOMMISSION_CAPACITY_RELEASE_COMPLETED, last_update);
8804
8805 let Some(stats) = self.pools.get_mut(idx) else {
8806 return false;
8807 };
8808 stats.last_update = last_update;
8809 stats.decommission = Some(pd);
8810 true
8811 }
8812 fn set_decommission_state_at(
8813 &mut self,
8814 idx: usize,
8815 pi: PoolSpaceInfo,
8816 queued: bool,
8817 now: OffsetDateTime,
8818 rebalance_meta: Option<&RebalanceMeta>,
8819 ) -> Result<()> {
8820 let pool_count = self.pools.len();
8821 ensure_valid_decommission_pool_index(pool_count, idx)?;
8822
8823 let Some(pool) = self.pools.get(idx) else {
8824 return Err(invalid_decommission_pool_index_error(pool_count, idx));
8825 };
8826
8827 ensure_decommission_start_allowed(decommission_start_pool_state(Some(pool)))?;
8828
8829 let generation = self.next_scanner_data_movement_update(now, rebalance_meta);
8830 let Some(pool) = self.pools.get_mut(idx) else {
8831 return Err(invalid_decommission_pool_index_error(pool_count, idx));
8832 };
8833 let previous = pool.decommission.as_ref();
8834 pool.last_update = generation;
8835 pool.decommission = Some(build_decommission_start_state(pi, queued, generation, previous));
8836
8837 Ok(())
8838 }
8839
8840 fn set_decommission_state(&mut self, idx: usize, pi: PoolSpaceInfo, queued: bool) -> Result<()> {
8841 self.set_decommission_state_at(idx, pi, queued, OffsetDateTime::now_utc(), None)
8842 }
8843
8844 #[cfg(test)]
8845 pub(crate) fn decommission_at_for_test(
8846 &mut self,
8847 idx: usize,
8848 pi: PoolSpaceInfo,
8849 now: OffsetDateTime,
8850 rebalance_meta: Option<&RebalanceMeta>,
8851 ) -> Result<()> {
8852 self.set_decommission_state_at(idx, pi, false, now, rebalance_meta)
8853 }
8854
8855 pub fn decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> {
8856 self.set_decommission_state(idx, pi, false)
8857 }
8858
8859 pub fn queue_decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> {
8860 self.set_decommission_state(idx, pi, true)
8861 }
8862
8863 pub fn record_decommission_terminal_reload_failure(&mut self, idx: usize, stage: &str, message: String) -> Result<bool> {
8864 self.record_decommission_terminal_reload_failure_at(idx, stage, message, OffsetDateTime::now_utc(), None)
8865 }
8866
8867 #[cfg(test)]
8868 pub(crate) fn record_decommission_terminal_reload_failure_at_for_test(
8869 &mut self,
8870 idx: usize,
8871 stage: &str,
8872 message: String,
8873 now: OffsetDateTime,
8874 rebalance_meta: Option<&RebalanceMeta>,
8875 ) -> Result<bool> {
8876 self.record_decommission_terminal_reload_failure_at(idx, stage, message, now, rebalance_meta)
8877 }
8878
8879 fn record_decommission_terminal_reload_failure_at(
8880 &mut self,
8881 idx: usize,
8882 stage: &str,
8883 message: String,
8884 now: OffsetDateTime,
8885 rebalance_meta: Option<&RebalanceMeta>,
8886 ) -> Result<bool> {
8887 let pool_count = self.pools.len();
8888 ensure_valid_decommission_pool_index(pool_count, idx)?;
8889
8890 let Some(pool) = self.pools.get(idx) else {
8891 return Err(invalid_decommission_pool_index_error(pool_count, idx));
8892 };
8893 let Some(info) = pool.decommission.as_ref() else {
8894 return Err(decommission_metadata_not_initialized_error("record decommission terminal reload failure"));
8895 };
8896
8897 let failure = format!("{stage}: {message}");
8898 if info.terminal_reload_failures.last().is_some_and(|last| last == &failure) {
8899 return Ok(false);
8900 }
8901
8902 let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8903 let Some(pool) = self.pools.get_mut(idx) else {
8904 return Err(invalid_decommission_pool_index_error(pool_count, idx));
8905 };
8906 let Some(info) = pool.decommission.as_mut() else {
8907 return Err(decommission_metadata_not_initialized_error("record decommission terminal reload failure"));
8908 };
8909 pool.last_update = last_update;
8910 info.terminal_reload_attempt_at = Some(last_update);
8911 info.terminal_reload_failures.push(failure);
8912 Ok(true)
8913 }
8914
8915 fn mark_decommission_capacity_blocked(&mut self, idx: usize, reason: String, now: OffsetDateTime) -> Result<bool> {
8916 let pool_count = self.pools.len();
8917 let pool = self
8918 .pools
8919 .get_mut(idx)
8920 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, idx))?;
8921 let info = pool
8922 .decommission
8923 .as_mut()
8924 .ok_or_else(|| decommission_metadata_not_initialized_error("pause decommission for target capacity"))?;
8925 if !is_decommission_active(info.complete, info.failed, info.canceled) {
8926 return Ok(false);
8927 }
8928 if let Some(reservation) = info.capacity_reservation.as_mut().filter(|reservation| reservation.active()) {
8929 renew_decommission_capacity_reservation(reservation, now, true);
8930 }
8931 let changed = info.capacity_blocked_reason.as_deref() != Some(reason.as_str());
8932 info.capacity_blocked_reason = Some(reason);
8933 pool.last_update = now;
8934 Ok(changed)
8935 }
8936
8937 pub fn promote_queued_decommission(&mut self, idx: usize) -> bool {
8938 self.promote_queued_decommission_at(idx, OffsetDateTime::now_utc(), None)
8939 }
8940
8941 #[cfg(test)]
8942 pub(crate) fn promote_queued_decommission_at_for_test(
8943 &mut self,
8944 idx: usize,
8945 now: OffsetDateTime,
8946 rebalance_meta: Option<&RebalanceMeta>,
8947 ) -> bool {
8948 self.promote_queued_decommission_at(idx, now, rebalance_meta)
8949 }
8950
8951 fn promote_queued_decommission_at(
8952 &mut self,
8953 idx: usize,
8954 now: OffsetDateTime,
8955 rebalance_meta: Option<&RebalanceMeta>,
8956 ) -> bool {
8957 let Some(info) = self.pools.get(idx).and_then(|pool| pool.decommission.as_ref()) else {
8958 return false;
8959 };
8960 if !info.queued || !is_decommission_active(info.complete, info.failed, info.canceled) {
8961 return false;
8962 }
8963
8964 let generation = self.next_scanner_data_movement_update(now, rebalance_meta);
8965 let Some(pool) = self.pools.get_mut(idx) else {
8966 return false;
8967 };
8968 let Some(info) = pool.decommission.as_mut() else {
8969 return false;
8970 };
8971 pool.last_update = generation;
8972 info.queued = false;
8973 info.start_time = Some(generation);
8974 for entry in &mut info.unresolved_entries {
8975 entry.source_generation = generation;
8976 }
8977 true
8978 }
8979 pub fn queue_buckets(&mut self, idx: usize, bks: Vec<DecomBucketInfo>) {
8980 if let Some(pool) = self.pools.get_mut(idx)
8981 && let Some(dec) = pool.decommission.as_mut()
8982 {
8983 for bk in bks.iter() {
8984 dec.bucket_push(bk);
8985 }
8986 }
8987 }
8988 pub fn pending_buckets(&self, idx: usize) -> Vec<DecomBucketInfo> {
8989 let mut list = Vec::new();
8990
8991 if let Some(pool) = self.pools.get(idx)
8992 && let Some(ref info) = pool.decommission
8993 {
8994 for bk in info.queued_buckets.iter() {
8995 let (name, prefix) = path2_bucket_object(bk);
8996 list.push(DecomBucketInfo { name, prefix });
8997 }
8998 }
8999
9000 list
9001 }
9002
9003 pub fn is_bucket_decommissioned(&self, idx: usize, bucket: String) -> bool {
9004 self.pools
9005 .get(idx)
9006 .and_then(|pool| pool.decommission.as_ref())
9007 .is_some_and(|info| info.is_bucket_decommissioned(&bucket))
9008 }
9009
9010 pub fn bucket_done(&mut self, idx: usize, bucket: String) -> bool {
9011 if let Some(pool) = self.pools.get_mut(idx) {
9012 if let Some(info) = pool.decommission.as_mut() {
9013 info.bucket_pop(&bucket)
9014 } else {
9015 false
9016 }
9017 } else {
9018 false
9019 }
9020 }
9021
9022 pub fn count_item(&mut self, idx: usize, size: usize, failed: bool) {
9023 if let Some(pool) = self.pools.get_mut(idx)
9024 && let Some(info) = pool.decommission.as_mut()
9025 {
9026 if failed {
9027 info.items_decommission_failed += 1;
9028 info.bytes_failed += size;
9029 } else {
9030 info.items_decommissioned += 1;
9031 info.bytes_done += size;
9032 }
9033 }
9034 }
9035
9036 pub fn track_current_bucket_object(&mut self, idx: usize, bucket: String, object: String) {
9037 self.track_current_bucket_object_stage(idx, bucket, object, String::new());
9038 }
9039
9040 pub fn track_current_bucket_object_stage(&mut self, idx: usize, bucket: String, object: String, stage: String) {
9041 if self.pools.get(idx).is_none_or(|v| v.decommission.is_none()) {
9042 return;
9043 }
9044
9045 if let Some(pool) = self.pools.get_mut(idx)
9046 && let Some(info) = pool.decommission.as_mut()
9047 {
9048 info.object = object;
9049 info.bucket = bucket;
9050 info.stage = stage;
9051 }
9052 }
9053
9054 pub fn update_after(&mut self, idx: usize, duration: Duration) -> Result<bool> {
9055 Ok(self
9056 .decommission_progress_checkpoint(idx, duration, OffsetDateTime::now_utc(), None)?
9057 .is_some())
9058 }
9059
9060 pub fn validate(&self, pools: Vec<Arc<Sets>>) -> Result<bool> {
9061 struct PoolInfo {
9062 position: usize,
9063 completed: bool,
9064 #[allow(dead_code, reason = "written but never read back (backlog#1823)")]
9065 decom_started: bool,
9066 }
9067
9068 let mut remembered_pools = HashMap::new();
9069 for (idx, pool) in self.pools.iter().enumerate() {
9070 let mut complete = false;
9071 let mut decom_started = false;
9072 if let Some(decommission) = &pool.decommission {
9073 if decommission.complete {
9074 complete = true;
9075 }
9076 decom_started = true;
9077 }
9078 remembered_pools.insert(
9079 pool.cmd_line.clone(),
9080 PoolInfo {
9081 position: idx,
9082 completed: complete,
9083 decom_started,
9084 },
9085 );
9086 }
9087
9088 let mut specified_pools = HashMap::new();
9089 for (idx, pool) in pools.iter().enumerate() {
9090 specified_pools.insert(pool.endpoints.cmd_line.clone(), idx);
9091 }
9092
9093 let mut update = false;
9094
9095 for k in specified_pools.keys() {
9097 if let Some(pi) = remembered_pools.get(k) {
9098 ensure_pool_not_left_in_cmdline_after_decommission(pi.position, k, pi.completed)?;
9099 } else {
9100 update = true;
9102 }
9103 }
9104
9105 if specified_pools.len() == remembered_pools.len() {
9106 for (k, pi) in remembered_pools.iter() {
9107 if let Some(pos) = specified_pools.get(k)
9108 && *pos != pi.position
9109 {
9110 update = true; }
9112 }
9113 }
9114
9115 if !update {
9116 update = specified_pools.len() != remembered_pools.len();
9117 }
9118
9119 Ok(update)
9120 }
9121
9122 pub fn return_resumable_pools(&self) -> Vec<PoolStatus> {
9123 resumable_decommission_queue_indices(self)
9124 .into_iter()
9125 .map(|idx| self.pools[idx].clone())
9126 .collect()
9127 }
9128}
9129
9130pub fn path2_bucket_object(name: &str) -> (String, String) {
9131 path_to_bucket_object(name)
9132}
9133
9134pub fn path2_bucket_object_with_base_path(base_path: &str, path: &str) -> (String, String) {
9135 path_to_bucket_object_with_base_path(base_path, path)
9136}
9137
9138#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
9139#[serde(deny_unknown_fields)]
9140pub struct DecommissionUnresolvedEntry {
9141 pub bucket: String,
9142 pub object: String,
9143 #[serde(rename = "poolIndex")]
9144 pub pool_index: usize,
9145 #[serde(rename = "setIndex")]
9146 pub set_index: usize,
9147 #[serde(rename = "sourceGeneration", with = "time::serde::rfc3339")]
9148 pub source_generation: OffsetDateTime,
9149 #[serde(rename = "candidateCount")]
9150 pub candidate_count: usize,
9151 #[serde(rename = "diskErrorCount")]
9152 pub disk_error_count: usize,
9153 #[serde(rename = "observedAt", with = "time::serde::rfc3339")]
9154 pub observed_at: OffsetDateTime,
9155 pub reason: String,
9156}
9157
9158#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
9159#[serde(rename_all = "camelCase", deny_unknown_fields)]
9160pub struct DecommissionErasureLayout {
9161 pub data: usize,
9162 pub parity: usize,
9163}
9164
9165impl DecommissionErasureLayout {
9166 fn width(self) -> usize {
9167 self.data.saturating_add(self.parity)
9168 }
9169
9170 fn is_valid(self) -> bool {
9171 self.data > 0 && self.data.checked_add(self.parity).is_some()
9172 }
9173}
9174
9175#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
9176#[serde(rename_all = "camelCase", deny_unknown_fields)]
9177pub struct DecommissionCapacityTarget {
9178 pub pool_index: usize,
9179 pub layout: DecommissionErasureLayout,
9180 pub physical_total_at_reservation: usize,
9181 pub physical_free_at_reservation: usize,
9182 pub reserved_physical_bytes: usize,
9183 #[serde(default)]
9184 pub consumed_physical_bytes: usize,
9185 #[serde(default)]
9186 pub observed_physical_bytes: usize,
9187 #[serde(default)]
9188 pub inflight_physical_bytes: usize,
9189 #[serde(default)]
9190 pub pending_physical_bytes: usize,
9191 #[serde(default)]
9192 pub pending_mutation_id: Option<uuid::Uuid>,
9193 #[serde(default, skip_serializing_if = "Vec::is_empty")]
9194 pub temporary_mutations: Vec<DecommissionCapacityTemporaryMutation>,
9195}
9196
9197#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
9198#[serde(rename_all = "camelCase", deny_unknown_fields)]
9199pub struct DecommissionCapacityTemporaryMutation {
9200 pub mutation_id: uuid::Uuid,
9201 pub physical_bytes: usize,
9202}
9203
9204#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
9205#[serde(rename_all = "camelCase", deny_unknown_fields)]
9206pub struct DecommissionCapacityReservation {
9207 pub model_version: u16,
9208 pub operation_id: uuid::Uuid,
9209 pub generation: u64,
9210 pub owner_nonce: uuid::Uuid,
9211 pub source_pool_index: usize,
9212 pub source_layout: DecommissionErasureLayout,
9213 pub source_physical_total_bytes: usize,
9214 pub source_physical_bytes: usize,
9215 pub source_data_equivalent_bytes: usize,
9216 pub predicted_physical_bytes: usize,
9217 pub temporary_copies: usize,
9218 pub temporary_physical_bytes: usize,
9219 pub peak_physical_bytes: usize,
9220 pub committed_data_bytes: usize,
9221 #[serde(default)]
9222 pub consumed_target_physical_bytes: usize,
9223 #[serde(default)]
9224 pub observed_target_physical_bytes: usize,
9225 #[serde(default)]
9226 pub inflight_target_physical_bytes: usize,
9227 #[serde(default)]
9228 pub pending_target_physical_bytes: usize,
9229 pub prediction_error_bytes: i64,
9230 pub targets: Vec<DecommissionCapacityTarget>,
9231 #[serde(with = "time::serde::rfc3339")]
9232 pub created_at: OffsetDateTime,
9233 #[serde(with = "time::serde::rfc3339")]
9234 pub renewed_at: OffsetDateTime,
9235 #[serde(with = "time::serde::rfc3339")]
9236 pub expires_at: OffsetDateTime,
9237 #[serde(with = "time::serde::rfc3339::option", default)]
9238 pub recovered_at: Option<OffsetDateTime>,
9239 #[serde(with = "time::serde::rfc3339::option", default)]
9240 pub released_at: Option<OffsetDateTime>,
9241 #[serde(default, skip_serializing_if = "Option::is_none")]
9242 pub release_reason: Option<String>,
9243}
9244
9245#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9246enum DecommissionCapacityMutationMode {
9247 Durable,
9248 Temporary,
9249 NonGrowingReplacement,
9250 TemporaryRelease,
9251}
9252
9253#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9254struct DecommissionCapacityReleaseProof {
9255 confirmed_absent: bool,
9256 clear_pending: bool,
9257}
9258
9259impl DecommissionCapacityReleaseProof {
9260 const fn confirmed_absence(clear_pending: bool) -> Self {
9261 Self {
9262 confirmed_absent: true,
9263 clear_pending,
9264 }
9265 }
9266}
9267
9268#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9269pub(crate) struct DecommissionCapacityOwner {
9270 pub(crate) source_pool_index: usize,
9271 pub(crate) operation_id: uuid::Uuid,
9272 pub(crate) generation: u64,
9273 pub(crate) owner_nonce: uuid::Uuid,
9274 pub(crate) mutation_id: Option<uuid::Uuid>,
9275}
9276
9277#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
9278struct DecommissionCapacityTargetPermitKey {
9279 store_id: uuid::Uuid,
9280 target_pool_index: usize,
9281 source_pool_index: usize,
9282 operation_id: uuid::Uuid,
9283 generation: u64,
9284 owner_nonce: uuid::Uuid,
9285 mutation_id: uuid::Uuid,
9286}
9287
9288impl DecommissionCapacityTargetPermitKey {
9289 fn new(store_id: uuid::Uuid, target_pool_index: usize, owner: DecommissionCapacityOwner) -> Option<Self> {
9290 Some(Self {
9291 store_id,
9292 target_pool_index,
9293 source_pool_index: owner.source_pool_index,
9294 operation_id: owner.operation_id,
9295 generation: owner.generation,
9296 owner_nonce: owner.owner_nonce,
9297 mutation_id: owner.mutation_id?,
9298 })
9299 }
9300
9301 fn owns_same_mutation(&self, store_id: uuid::Uuid, owner: DecommissionCapacityOwner) -> bool {
9302 owner.mutation_id.is_some_and(|mutation_id| {
9303 self.store_id == store_id
9304 && self.source_pool_index == owner.source_pool_index
9305 && self.operation_id == owner.operation_id
9306 && self.generation == owner.generation
9307 && self.owner_nonce == owner.owner_nonce
9308 && self.mutation_id == mutation_id
9309 })
9310 }
9311}
9312
9313struct DecommissionCapacityTargetPermit {
9314 key: DecommissionCapacityTargetPermitKey,
9315}
9316
9317static DECOMMISSION_CAPACITY_TARGET_PERMITS: std::sync::OnceLock<
9321 std::sync::Mutex<HashMap<DecommissionCapacityTargetPermitKey, rustfs_lock::NamespaceLockGuard>>,
9322> = std::sync::OnceLock::new();
9323
9324fn decommission_capacity_target_permits()
9325-> &'static std::sync::Mutex<HashMap<DecommissionCapacityTargetPermitKey, rustfs_lock::NamespaceLockGuard>> {
9326 DECOMMISSION_CAPACITY_TARGET_PERMITS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
9327}
9328
9329fn install_decommission_capacity_target_permit(
9330 store_id: uuid::Uuid,
9331 target_pool_index: usize,
9332 owner: DecommissionCapacityOwner,
9333 guard: rustfs_lock::NamespaceLockGuard,
9334) -> Result<DecommissionCapacityTargetPermit> {
9335 let key = DecommissionCapacityTargetPermitKey::new(store_id, target_pool_index, owner)
9336 .ok_or_else(|| Error::other("decommission target permit is missing its mutation identity"))?;
9337 let mut permits = decommission_capacity_target_permits()
9338 .lock()
9339 .unwrap_or_else(std::sync::PoisonError::into_inner);
9340 if permits.keys().any(|candidate| candidate.owns_same_mutation(store_id, owner)) {
9341 return Err(Error::other("decommission target permit already exists for this mutation"));
9342 }
9343 permits.insert(key, guard);
9344 Ok(DecommissionCapacityTargetPermit { key })
9345}
9346
9347fn take_decommission_capacity_target_permit(
9348 store_id: uuid::Uuid,
9349 target_pool_index: usize,
9350 owner: DecommissionCapacityOwner,
9351) -> Option<rustfs_lock::NamespaceLockGuard> {
9352 let key = DecommissionCapacityTargetPermitKey::new(store_id, target_pool_index, owner)?;
9353 decommission_capacity_target_permits()
9354 .lock()
9355 .unwrap_or_else(std::sync::PoisonError::into_inner)
9356 .remove(&key)
9357}
9358
9359fn decommission_capacity_target_permit_index(store_id: uuid::Uuid, owner: DecommissionCapacityOwner) -> Option<usize> {
9360 decommission_capacity_target_permits()
9361 .lock()
9362 .unwrap_or_else(std::sync::PoisonError::into_inner)
9363 .keys()
9364 .find(|key| key.owns_same_mutation(store_id, owner))
9365 .map(|key| key.target_pool_index)
9366}
9367
9368fn discard_decommission_capacity_target_permit_except(
9369 store_id: uuid::Uuid,
9370 owner: DecommissionCapacityOwner,
9371 retained_target_pool_index: Option<usize>,
9372) {
9373 let guard = {
9374 let mut permits = decommission_capacity_target_permits()
9375 .lock()
9376 .unwrap_or_else(std::sync::PoisonError::into_inner);
9377 let key = permits
9378 .keys()
9379 .find(|key| key.owns_same_mutation(store_id, owner) && retained_target_pool_index != Some(key.target_pool_index))
9380 .copied();
9381 key.and_then(|key| permits.remove(&key))
9382 };
9383 drop(guard);
9384}
9385
9386impl Drop for DecommissionCapacityTargetPermit {
9387 fn drop(&mut self) {
9388 let guard = decommission_capacity_target_permits()
9389 .lock()
9390 .unwrap_or_else(std::sync::PoisonError::into_inner)
9391 .remove(&self.key);
9392 drop(guard);
9393 }
9394}
9395
9396#[derive(Debug, Clone, PartialEq, Eq)]
9397struct DecommissionCapacityTerminalFencePlan {
9398 model_version: u16,
9399 operation_id: uuid::Uuid,
9400 generation: u64,
9401 owner_nonce: uuid::Uuid,
9402 target_pool_indices: Vec<usize>,
9403}
9404
9405fn decommission_capacity_terminal_fence_plan(
9406 meta: &PoolMeta,
9407 source_pool_index: usize,
9408) -> Result<Option<DecommissionCapacityTerminalFencePlan>> {
9409 let Some(reservation) = meta
9410 .pools
9411 .get(source_pool_index)
9412 .and_then(|pool| pool.decommission.as_ref())
9413 .and_then(|info| info.capacity_reservation.as_ref())
9414 .filter(|reservation| reservation.active())
9415 else {
9416 return Ok(None);
9417 };
9418 if reservation.source_pool_index != source_pool_index {
9419 return Err(Error::DecommissionCapacity(
9420 "decommission terminal transition found a mismatched capacity source".to_string(),
9421 ));
9422 }
9423 if !matches!(
9424 reservation.model_version,
9425 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
9426 ) {
9427 return Err(Error::DecommissionCapacity(format!(
9428 "decommission terminal transition found unsupported capacity lock model {}",
9429 reservation.model_version
9430 )));
9431 }
9432 let mut target_pool_indices = reservation.targets.iter().map(|target| target.pool_index).collect::<Vec<_>>();
9433 target_pool_indices.sort_unstable();
9434 Ok(Some(DecommissionCapacityTerminalFencePlan {
9435 model_version: reservation.model_version,
9436 operation_id: reservation.operation_id,
9437 generation: reservation.generation,
9438 owner_nonce: reservation.owner_nonce,
9439 target_pool_indices,
9440 }))
9441}
9442
9443#[derive(Debug, Clone, PartialEq, Eq)]
9444pub(crate) struct DecommissionDurableIlmCheckpointTarget {
9445 pub(crate) source_pool_index: usize,
9446 pub(crate) target_pool_index: usize,
9447 pub(crate) capacity_owner: DecommissionCapacityOwner,
9448 pub(crate) already_committed: bool,
9449 pub(crate) target_etag: Option<String>,
9450}
9451
9452impl DecommissionCapacityOwner {
9453 pub(crate) fn apply_to(self, opts: &mut ObjectOptions) {
9454 opts.src_pool_idx = self.source_pool_index;
9455 let admission = opts
9456 .decommission_capacity
9457 .get_or_insert_with(|| Box::new(DecommissionCapacityOptions::default()));
9458 admission.operation_id = Some(self.operation_id);
9459 admission.generation = Some(self.generation);
9460 admission.owner_nonce = Some(self.owner_nonce);
9461 admission.mutation_id = self.mutation_id;
9462 }
9463
9464 pub(crate) fn from_options(opts: &ObjectOptions) -> Option<Self> {
9465 let capacity = opts.decommission_capacity.as_deref()?;
9466 Some(Self {
9467 source_pool_index: opts.src_pool_idx,
9468 operation_id: capacity.operation_id?,
9469 generation: capacity.generation?,
9470 owner_nonce: capacity.owner_nonce?,
9471 mutation_id: capacity.mutation_id,
9472 })
9473 }
9474
9475 pub(crate) fn with_mutation_id(self, mutation_id: uuid::Uuid) -> Self {
9476 Self {
9477 mutation_id: Some(mutation_id),
9478 ..self
9479 }
9480 }
9481}
9482
9483pub(crate) fn decommission_capacity_mutation_id(
9484 owner: DecommissionCapacityOwner,
9485 bucket: &str,
9486 object: &str,
9487 version_id: Option<&str>,
9488 delete_marker: bool,
9489 mod_time: Option<OffsetDateTime>,
9490) -> uuid::Uuid {
9491 let mut hasher = Sha256::new();
9492 hasher.update(owner.operation_id.as_bytes());
9493 hasher.update(owner.generation.to_le_bytes());
9494 hasher.update(owner.source_pool_index.to_le_bytes());
9495 for value in [bucket, object] {
9496 hasher.update((value.len() as u64).to_le_bytes());
9497 hasher.update(value.as_bytes());
9498 }
9499 hasher.update([u8::from(delete_marker)]);
9500 if let Some(version_id) = version_id {
9501 hasher.update([1]);
9502 hasher.update((version_id.len() as u64).to_le_bytes());
9503 hasher.update(version_id.as_bytes());
9504 } else {
9505 hasher.update([0]);
9506 }
9507 if let Some(mod_time) = mod_time {
9508 hasher.update([1]);
9509 hasher.update(mod_time.unix_timestamp_nanos().to_le_bytes());
9510 } else {
9511 hasher.update([0]);
9512 }
9513 let digest = hasher.finalize();
9514 let mut bytes = [0; 16];
9515 bytes.copy_from_slice(&digest[..16]);
9516 uuid::Uuid::from_bytes(bytes)
9517}
9518
9519#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9520struct ExactDeleteCapacityReconciliation {
9521 source_pool_index: usize,
9522 target_pool_index: usize,
9523 mutation_id: uuid::Uuid,
9524 expected_data_bytes: usize,
9525 expected_target_physical_bytes: usize,
9526}
9527
9528fn plan_exact_delete_capacity_reconciliations(
9529 meta: &PoolMeta,
9530 object: &str,
9531 exact: &ObjectInfo,
9532) -> Result<Vec<ExactDeleteCapacityReconciliation>> {
9533 let version_id = exact.version_id.map(|version_id| version_id.to_string());
9534 let mut matches = Vec::new();
9535
9536 for (source_pool_index, pool) in meta.pools.iter().enumerate() {
9537 let Some(reservation) = pool
9538 .decommission
9539 .as_ref()
9540 .and_then(|info| info.capacity_reservation.as_ref())
9541 .filter(|reservation| reservation.active())
9542 else {
9543 continue;
9544 };
9545 let owner = DecommissionCapacityOwner {
9546 source_pool_index,
9547 operation_id: reservation.operation_id,
9548 generation: reservation.generation,
9549 owner_nonce: reservation.owner_nonce,
9550 mutation_id: None,
9551 };
9552 let logical_mutation_id = decommission_capacity_mutation_id(
9553 owner,
9554 &exact.bucket,
9555 &exact.name,
9556 version_id.as_deref(),
9557 exact.delete_marker,
9558 exact.mod_time,
9559 );
9560 let internal_mutation_id = if object == exact.name {
9564 logical_mutation_id
9565 } else {
9566 decommission_capacity_mutation_id(
9567 owner,
9568 &exact.bucket,
9569 object,
9570 version_id.as_deref(),
9571 exact.delete_marker,
9572 exact.mod_time,
9573 )
9574 };
9575 let mut source_match = None;
9576
9577 for target in &reservation.targets {
9578 if target.pending_physical_bytes == 0 {
9579 continue;
9580 }
9581 let Some(pending_mutation_id) = target.pending_mutation_id else {
9582 return Err(decommission_capacity_blocked_error(format!(
9583 "source pool {source_pool_index} target pool {} has pending capacity without an object identity",
9584 target.pool_index
9585 )));
9586 };
9587 if pending_mutation_id != logical_mutation_id && pending_mutation_id != internal_mutation_id {
9588 continue;
9589 }
9590 if source_match.is_some() {
9591 return Err(decommission_capacity_blocked_error(format!(
9592 "source pool {source_pool_index} has the same exact-delete capacity intent on multiple targets"
9593 )));
9594 }
9595 source_match = Some((target.pool_index, target.layout, target.pending_physical_bytes, pending_mutation_id));
9596 }
9597
9598 let Some((target_pool_index, target_layout, pending_physical_bytes, mutation_id)) = source_match else {
9599 continue;
9600 };
9601 if exact.version_id.is_none() && exact.mod_time.is_none() {
9602 return Err(decommission_capacity_blocked_error(
9603 "unversioned exact delete cannot identify pending capacity without a modification time",
9604 ));
9605 }
9606 let expected_data_bytes = if exact.delete_marker {
9607 0
9608 } else {
9609 usize::try_from(exact.size).map_err(|_| {
9610 decommission_capacity_blocked_error("exact delete cannot reconcile a negative or overflowing object size")
9611 })?
9612 };
9613 let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
9614 if pending_physical_bytes != expected_target_physical_bytes {
9615 return Err(decommission_capacity_blocked_error(format!(
9616 "source pool {source_pool_index} target pool {target_pool_index} pending capacity does not match the exact object size"
9617 )));
9618 }
9619 let remaining_target_physical_bytes = reservation
9620 .targets
9621 .iter()
9622 .find(|target| target.pool_index == target_pool_index)
9623 .map(|target| {
9624 target.remaining_reserved_physical_bytes(reservation.temporary_copies)
9625 / 1usize.saturating_add(reservation.temporary_copies)
9626 })
9627 .unwrap_or_default();
9628 let remaining_total_physical_bytes = reservation
9629 .predicted_physical_bytes
9630 .saturating_sub(reservation.consumed_target_physical_bytes);
9631 let remaining_data_bytes = reservation
9632 .source_data_equivalent_bytes
9633 .saturating_sub(reservation.committed_data_bytes);
9634 if expected_target_physical_bytes > remaining_target_physical_bytes
9635 || expected_target_physical_bytes > remaining_total_physical_bytes
9636 || expected_data_bytes > remaining_data_bytes
9637 {
9638 return Err(decommission_capacity_blocked_error(format!(
9639 "source pool {source_pool_index} target pool {target_pool_index} lacks reservation capacity for the exact object"
9640 )));
9641 }
9642 matches.push(ExactDeleteCapacityReconciliation {
9643 source_pool_index,
9644 target_pool_index,
9645 mutation_id,
9646 expected_data_bytes,
9647 expected_target_physical_bytes,
9648 });
9649 }
9650
9651 Ok(matches)
9652}
9653
9654fn ensure_exact_delete_capacity_namespace_fences(opts: &ObjectOptions, bucket: &str, object: &str) -> Result<()> {
9655 let object_fence = opts.namespace_lock_fence.as_ref().ok_or_else(|| {
9656 decommission_capacity_blocked_error("exact delete capacity reconciliation requires an object namespace fence")
9657 })?;
9658 if object_fence.is_lock_lost() {
9659 return Err(StorageError::NamespaceLockQuorumUnavailable {
9660 mode: "exact_delete_capacity_reconciliation",
9661 bucket: bucket.to_string(),
9662 object: decode_dir_object(object),
9663 required: 1,
9664 achieved: 0,
9665 });
9666 }
9667 if opts
9668 .bucket_lifecycle_lock_fence
9669 .as_ref()
9670 .is_some_and(crate::object_api::NamespaceLockFence::is_lock_lost)
9671 {
9672 return Err(StorageError::NamespaceLockQuorumUnavailable {
9673 mode: "exact_delete_capacity_bucket_generation",
9674 bucket: bucket.to_string(),
9675 object: decode_dir_object(object),
9676 required: 1,
9677 achieved: 0,
9678 });
9679 }
9680 Ok(())
9681}
9682
9683pub(crate) fn ensure_decommission_capacity_mutation_id(bucket: &str, object: &str, opts: &mut ObjectOptions) {
9684 if opts
9685 .decommission_capacity
9686 .as_deref()
9687 .is_none_or(|capacity| capacity.mutation_id.is_some())
9688 {
9689 return;
9690 }
9691 let Some(owner) = DecommissionCapacityOwner::from_options(opts) else {
9692 return;
9693 };
9694 let mutation_id =
9695 decommission_capacity_mutation_id(owner, bucket, object, opts.version_id.as_deref(), opts.delete_marker, opts.mod_time);
9696 if let Some(capacity) = opts.decommission_capacity.as_mut() {
9697 capacity.mutation_id = Some(mutation_id);
9698 }
9699}
9700
9701impl DecommissionCapacityReservation {
9702 fn active(&self) -> bool {
9703 self.released_at.is_none()
9704 }
9705
9706 fn lease_active_at(&self, now: OffsetDateTime) -> bool {
9707 self.active() && self.expires_at > now
9708 }
9709
9710 fn admits_owner(&self, owner: DecommissionCapacityOwner, now: OffsetDateTime) -> bool {
9711 self.lease_active_at(now)
9712 && self.source_pool_index == owner.source_pool_index
9713 && self.operation_id == owner.operation_id
9714 && self.generation == owner.generation
9715 && self.owner_nonce == owner.owner_nonce
9716 }
9717
9718 fn admits_cleanup_owner(&self, owner: DecommissionCapacityOwner) -> bool {
9719 self.active()
9720 && self.source_pool_index == owner.source_pool_index
9721 && self.operation_id == owner.operation_id
9722 && self.generation == owner.generation
9723 }
9724
9725 fn remaining_peak_physical_bytes(&self) -> usize {
9726 self.predicted_physical_bytes
9727 .saturating_sub(self.consumed_target_physical_bytes)
9728 .saturating_mul(1usize.saturating_add(self.temporary_copies))
9729 }
9730}
9731
9732impl DecommissionCapacityTarget {
9733 fn remaining_reserved_physical_bytes(&self, temporary_copies: usize) -> usize {
9734 self.reserved_physical_bytes.saturating_sub(
9735 self.consumed_physical_bytes
9736 .saturating_mul(1usize.saturating_add(temporary_copies)),
9737 )
9738 }
9739}
9740
9741#[derive(Debug, Clone, Serialize, Deserialize, Default)]
9742pub struct PoolDecommissionInfo {
9743 #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
9744 pub start_time: Option<OffsetDateTime>,
9745 #[serde(rename = "startSize")]
9746 pub start_size: usize,
9747 #[serde(rename = "totalSize")]
9748 pub total_size: usize,
9749 #[serde(rename = "currentSize")]
9750 pub current_size: usize,
9751 #[serde(rename = "complete")]
9752 pub complete: bool,
9753 #[serde(rename = "failed")]
9754 pub failed: bool,
9755 #[serde(rename = "canceled")]
9756 pub canceled: bool,
9757 #[serde(skip)]
9758 pub queued: bool,
9759
9760 #[serde(skip)]
9761 pub queued_buckets: Vec<String>,
9762 #[serde(skip)]
9763 pub decommissioned_buckets: Vec<String>,
9764 #[serde(skip)]
9765 pub bucket: String,
9766 #[serde(skip)]
9767 pub prefix: String,
9768 #[serde(skip)]
9769 pub object: String,
9770 #[serde(skip)]
9771 pub stage: String,
9772
9773 #[serde(rename = "objectsDecommissioned")]
9774 pub items_decommissioned: usize,
9775 #[serde(rename = "objectsDecommissionedFailed")]
9776 pub items_decommission_failed: usize,
9777 #[serde(rename = "bytesDecommissioned")]
9778 pub bytes_done: usize,
9779 #[serde(rename = "bytesDecommissionedFailed")]
9780 pub bytes_failed: usize,
9781 #[serde(rename = "terminalReloadAttemptAt", with = "time::serde::rfc3339::option", default)]
9782 pub terminal_reload_attempt_at: Option<OffsetDateTime>,
9783 #[serde(rename = "terminalReloadFailures", default)]
9784 pub terminal_reload_failures: Vec<String>,
9785 #[serde(rename = "capacityReservation", default, skip_serializing_if = "Option::is_none")]
9786 pub capacity_reservation: Option<DecommissionCapacityReservation>,
9787 #[serde(rename = "capacityBlockedReason", default, skip_serializing_if = "Option::is_none")]
9788 pub capacity_blocked_reason: Option<String>,
9789 #[serde(skip)]
9790 pub unresolved_entries: Vec<DecommissionUnresolvedEntry>,
9791 #[serde(skip)]
9792 pub progress_save_item_baseline: usize,
9793 #[serde(skip)]
9794 pub progress_save_retry_after: Option<OffsetDateTime>,
9795 #[serde(skip)]
9796 pub progress_save_last_at: Option<OffsetDateTime>,
9797}
9798
9799#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9800struct DecommissionProgressCheckpoint {
9801 start_time: Option<OffsetDateTime>,
9802 queued: bool,
9803 counted_items: usize,
9804 checkpoint_at: OffsetDateTime,
9805 capacity_operation_id: Option<uuid::Uuid>,
9806 capacity_owner_nonce: Option<uuid::Uuid>,
9807 capacity_lease_expires_at: Option<OffsetDateTime>,
9808}
9809
9810impl PoolDecommissionInfo {
9811 pub fn has_decommission_state(&self) -> bool {
9812 self.complete
9813 || self.failed
9814 || self.canceled
9815 || self.queued
9816 || self.start_time.is_some()
9817 || self.start_size > 0
9818 || !self.queued_buckets.is_empty()
9819 || !self.decommissioned_buckets.is_empty()
9820 || !self.bucket.is_empty()
9821 || !self.prefix.is_empty()
9822 || !self.object.is_empty()
9823 || !self.stage.is_empty()
9824 || self.items_decommissioned > 0
9825 || self.items_decommission_failed > 0
9826 || self.bytes_done > 0
9827 || self.bytes_failed > 0
9828 || self.terminal_reload_attempt_at.is_some()
9829 || !self.terminal_reload_failures.is_empty()
9830 || self.capacity_reservation.is_some()
9831 || self.capacity_blocked_reason.is_some()
9832 || !self.unresolved_entries.is_empty()
9833 }
9834
9835 fn counted_items(&self) -> usize {
9836 self.items_decommissioned.saturating_add(self.items_decommission_failed)
9837 }
9838
9839 fn items_since_last_progress_save(&self) -> usize {
9840 self.counted_items().saturating_sub(self.progress_save_item_baseline)
9841 }
9842
9843 fn mark_progress_saved(&mut self) {
9844 self.progress_save_item_baseline = self.counted_items();
9845 self.progress_save_retry_after = None;
9846 }
9847
9848 pub fn bucket_push(&mut self, bucket: &DecomBucketInfo) {
9849 let bucket_key = bucket.to_string();
9850 if self.is_bucket_decommissioned(&bucket_key) {
9851 return;
9852 }
9853
9854 for b in self.queued_buckets.iter() {
9855 if b == &bucket_key {
9856 return;
9857 }
9858 }
9859
9860 self.queued_buckets.push(bucket_key);
9861
9862 self.bucket = bucket.name.clone();
9863 self.prefix = bucket.prefix.clone();
9864 }
9865 pub fn is_bucket_decommissioned(&self, bucket: &String) -> bool {
9866 for b in self.decommissioned_buckets.iter() {
9867 if b == bucket {
9868 return true;
9869 }
9870 }
9871 false
9872 }
9873 pub fn bucket_pop(&mut self, bucket: &String) -> bool {
9874 let mut found = None;
9875 for (i, b) in self.queued_buckets.iter().enumerate() {
9876 if b == bucket {
9877 found = Some(i);
9878 break;
9879 }
9880 }
9881
9882 if let Some(i) = found {
9883 if !self.is_bucket_decommissioned(bucket) {
9884 self.decommissioned_buckets.push(bucket.clone());
9885 }
9886 self.queued_buckets.remove(i);
9887 if &self.bucket == bucket {
9888 self.bucket = "".to_owned();
9889 self.prefix = "".to_owned();
9890 self.object = "".to_owned();
9891 }
9892
9893 return true;
9894 }
9895 false
9896 }
9897}
9898
9899#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9900pub struct PoolSpaceInfo {
9901 pub free: usize,
9902 pub total: usize,
9903 pub used: usize,
9904}
9905
9906#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9907pub(crate) struct DecommissionPoolCapacityInfo {
9908 pool_index: usize,
9909 space: PoolSpaceInfo,
9910 layout: DecommissionErasureLayout,
9911 physical_free: usize,
9912 physical_total: usize,
9913 physical_used: usize,
9914}
9915
9916impl DecommissionPoolCapacityInfo {
9917 #[cfg(any(test, feature = "test-util"))]
9918 pub(crate) fn for_test(
9919 pool_index: usize,
9920 layout: DecommissionErasureLayout,
9921 physical_free: usize,
9922 physical_total: usize,
9923 physical_used: usize,
9924 ) -> Self {
9925 Self {
9926 pool_index,
9927 space: PoolSpaceInfo {
9928 free: physical_free,
9929 total: physical_total,
9930 used: physical_used,
9931 },
9932 layout,
9933 physical_free,
9934 physical_total,
9935 physical_used,
9936 }
9937 }
9938}
9939
9940#[cfg(any(test, feature = "test-util"))]
9941type DecommissionCapacityInfoOverrides =
9942 std::sync::Mutex<HashMap<uuid::Uuid, std::collections::VecDeque<Vec<DecommissionPoolCapacityInfo>>>>;
9943
9944#[cfg(any(test, feature = "test-util"))]
9945static DECOMMISSION_CAPACITY_INFO_OVERRIDES: std::sync::OnceLock<DecommissionCapacityInfoOverrides> = std::sync::OnceLock::new();
9946
9947#[cfg(any(test, feature = "test-util"))]
9951pub(crate) fn set_decommission_capacity_info_overrides_for_test(
9952 store_id: uuid::Uuid,
9953 snapshots: Vec<Vec<DecommissionPoolCapacityInfo>>,
9954) {
9955 DECOMMISSION_CAPACITY_INFO_OVERRIDES
9956 .get_or_init(|| std::sync::Mutex::new(HashMap::new()))
9957 .lock()
9958 .expect("decommission capacity info override should not be poisoned")
9959 .insert(store_id, snapshots.into());
9960}
9961
9962#[cfg(any(test, feature = "test-util"))]
9963fn take_decommission_capacity_info_override_for_test(store_id: uuid::Uuid) -> Option<Vec<DecommissionPoolCapacityInfo>> {
9964 let mut overrides = DECOMMISSION_CAPACITY_INFO_OVERRIDES
9965 .get_or_init(|| std::sync::Mutex::new(HashMap::new()))
9966 .lock()
9967 .expect("decommission capacity info override should not be poisoned");
9968 let queue = overrides.get_mut(&store_id)?;
9969 if queue.len() > 1 {
9970 queue.pop_front()
9971 } else {
9972 queue.front().cloned()
9976 }
9977}
9978
9979#[cfg(test)]
9980struct DecommissionCapacityLockOrderBarrierState {
9981 owner_store_id: uuid::Uuid,
9982 external_store_id: uuid::Uuid,
9983 owner_arrived: tokio::sync::Notify,
9984 owner_release: tokio::sync::Notify,
9985 owner_pause_enabled: AtomicBool,
9986 external_capacity_released: tokio::sync::Notify,
9987 external_object_capacity_probe_acquired: tokio::sync::Notify,
9988 external_object_capacity_probe_release: tokio::sync::Notify,
9989 external_object_capacity_probe_paused: AtomicBool,
9990 external_object_commit_phase_started: tokio::sync::Notify,
9991 external_object_commit_phase_release: tokio::sync::Notify,
9992 external_object_commit_phase_paused: AtomicBool,
9993 external_heal_operation_started: tokio::sync::Notify,
9994 external_heal_target_lock_attempted: tokio::sync::Notify,
9995 target_gate_retry_entered: tokio::sync::Notify,
9996 target_gate_retry_entries: AtomicUsize,
9997 target_gate_inline_retry_entered: tokio::sync::Notify,
9998 target_gate_inline_retry_entries: AtomicUsize,
9999 target_gate_exact_reloads: AtomicUsize,
10000 target_gate_acquire_pause_target: AtomicUsize,
10001 target_gate_acquire_entered: tokio::sync::Notify,
10002 target_gate_acquire_release: tokio::sync::Notify,
10003 cancel_before_start_entered: tokio::sync::Notify,
10004 cancel_before_start_release: tokio::sync::Notify,
10005 cancel_before_start_paused: AtomicBool,
10006 cancel_target_timeout_entered: tokio::sync::Notify,
10007 cancel_target_timeout_release: tokio::sync::Notify,
10008 cancel_target_timeout_paused: AtomicBool,
10009 cancel_target_timeouts: AtomicUsize,
10010}
10011
10012#[cfg(test)]
10013pub(crate) struct DecommissionCapacityLockOrderBarrier {
10014 state: Arc<DecommissionCapacityLockOrderBarrierState>,
10015}
10016
10017#[cfg(test)]
10018static DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER: std::sync::OnceLock<
10019 std::sync::Mutex<Option<Arc<DecommissionCapacityLockOrderBarrierState>>>,
10020> = std::sync::OnceLock::new();
10021
10022#[cfg(test)]
10023impl DecommissionCapacityLockOrderBarrier {
10024 pub(crate) fn install(owner_store_id: uuid::Uuid, external_store_id: uuid::Uuid) -> Self {
10025 let state = Arc::new(DecommissionCapacityLockOrderBarrierState {
10026 owner_store_id,
10027 external_store_id,
10028 owner_arrived: tokio::sync::Notify::new(),
10029 owner_release: tokio::sync::Notify::new(),
10030 owner_pause_enabled: AtomicBool::new(true),
10031 external_capacity_released: tokio::sync::Notify::new(),
10032 external_object_capacity_probe_acquired: tokio::sync::Notify::new(),
10033 external_object_capacity_probe_release: tokio::sync::Notify::new(),
10034 external_object_capacity_probe_paused: AtomicBool::new(false),
10035 external_object_commit_phase_started: tokio::sync::Notify::new(),
10036 external_object_commit_phase_release: tokio::sync::Notify::new(),
10037 external_object_commit_phase_paused: AtomicBool::new(false),
10038 external_heal_operation_started: tokio::sync::Notify::new(),
10039 external_heal_target_lock_attempted: tokio::sync::Notify::new(),
10040 target_gate_retry_entered: tokio::sync::Notify::new(),
10041 target_gate_retry_entries: AtomicUsize::new(0),
10042 target_gate_inline_retry_entered: tokio::sync::Notify::new(),
10043 target_gate_inline_retry_entries: AtomicUsize::new(0),
10044 target_gate_exact_reloads: AtomicUsize::new(0),
10045 target_gate_acquire_pause_target: AtomicUsize::new(usize::MAX),
10046 target_gate_acquire_entered: tokio::sync::Notify::new(),
10047 target_gate_acquire_release: tokio::sync::Notify::new(),
10048 cancel_before_start_entered: tokio::sync::Notify::new(),
10049 cancel_before_start_release: tokio::sync::Notify::new(),
10050 cancel_before_start_paused: AtomicBool::new(false),
10051 cancel_target_timeout_entered: tokio::sync::Notify::new(),
10052 cancel_target_timeout_release: tokio::sync::Notify::new(),
10053 cancel_target_timeout_paused: AtomicBool::new(false),
10054 cancel_target_timeouts: AtomicUsize::new(0),
10055 });
10056 let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10057 .get_or_init(|| std::sync::Mutex::new(None))
10058 .lock()
10059 .expect("decommission capacity lock-order barrier should not be poisoned");
10060 assert!(slot.is_none(), "decommission capacity lock-order barrier must be unique");
10061 *slot = Some(Arc::clone(&state));
10062 Self { state }
10063 }
10064
10065 #[cfg(feature = "test-util")]
10066 pub(crate) async fn wait_until_owner_paused(&self) {
10067 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.owner_arrived.notified())
10068 .await
10069 .expect("owned capacity mutation should reach admission before its metadata write");
10070 }
10071
10072 pub(crate) async fn wait_until_external_capacity_released(&self) {
10073 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.external_capacity_released.notified())
10074 .await
10075 .expect("external mutation should release capacity before waiting for the object namespace");
10076 }
10077
10078 #[cfg(feature = "test-util")]
10079 pub(crate) async fn wait_until_external_object_capacity_probe_acquired(&self) {
10080 tokio::time::timeout(
10081 std::time::Duration::from_secs(30),
10082 self.state.external_object_capacity_probe_acquired.notified(),
10083 )
10084 .await
10085 .expect("external object mutation should acquire its no-active capacity probe");
10086 }
10087
10088 #[cfg(feature = "test-util")]
10089 pub(crate) async fn wait_until_external_object_commit_phase_started(&self) {
10090 tokio::time::timeout(
10091 std::time::Duration::from_secs(30),
10092 self.state.external_object_commit_phase_started.notified(),
10093 )
10094 .await
10095 .expect("external object mutation should reach its staged commit phase before waiting for the namespace");
10096 }
10097
10098 pub(crate) async fn wait_until_external_heal_operation_started(&self) {
10099 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.external_heal_operation_started.notified())
10100 .await
10101 .expect("external heal should reach the target operation after capacity admission");
10102 }
10103
10104 pub(crate) async fn wait_until_external_heal_target_lock_attempted(&self) {
10105 tokio::time::timeout(
10106 std::time::Duration::from_secs(30),
10107 self.state.external_heal_target_lock_attempted.notified(),
10108 )
10109 .await
10110 .expect("external heal should attempt the target namespace lock after capacity admission");
10111 }
10112
10113 #[cfg(feature = "test-util")]
10114 pub(crate) async fn wait_until_target_gate_retry(&self) {
10115 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.target_gate_retry_entered.notified())
10116 .await
10117 .expect("decommission entry should observe target gate contention");
10118 }
10119
10120 #[cfg(feature = "test-util")]
10121 pub(crate) async fn wait_until_target_gate_retries(&self, expected: usize) {
10122 tokio::time::timeout(std::time::Duration::from_secs(30), async {
10123 loop {
10124 let notified = self.state.target_gate_retry_entered.notified();
10125 if self.state.target_gate_retry_entries.load(Ordering::Acquire) >= expected {
10126 return;
10127 }
10128 notified.await;
10129 }
10130 })
10131 .await
10132 .expect("decommission entries should observe target gate contention");
10133 }
10134
10135 #[cfg(feature = "test-util")]
10136 pub(crate) fn target_gate_inline_retries(&self) -> usize {
10137 self.state.target_gate_inline_retry_entries.load(Ordering::Acquire)
10138 }
10139
10140 #[cfg(feature = "test-util")]
10141 pub(crate) fn target_gate_exact_reloads(&self) -> usize {
10142 self.state.target_gate_exact_reloads.load(Ordering::Acquire)
10143 }
10144
10145 #[cfg(feature = "test-util")]
10146 pub(crate) fn pause_target_gate_acquire(&self, target_pool_index: usize) {
10147 self.state
10148 .target_gate_acquire_pause_target
10149 .store(target_pool_index, Ordering::Release);
10150 }
10151
10152 #[cfg(feature = "test-util")]
10153 pub(crate) async fn wait_until_target_gate_acquire_paused(&self) {
10154 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.target_gate_acquire_entered.notified())
10155 .await
10156 .expect("decommission capacity mutation should pause before acquiring its target gate");
10157 }
10158
10159 #[cfg(feature = "test-util")]
10160 pub(crate) fn release_target_gate_acquire(&self) {
10161 self.state
10162 .target_gate_acquire_pause_target
10163 .store(usize::MAX, Ordering::Release);
10164 self.state.target_gate_acquire_release.notify_one();
10165 }
10166
10167 pub(crate) fn pause_cancel_before_start(&self) {
10168 self.state.cancel_before_start_paused.store(true, Ordering::Release);
10169 }
10170
10171 pub(crate) async fn wait_until_cancel_before_start(&self) {
10172 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.cancel_before_start_entered.notified())
10173 .await
10174 .expect("decommission cancel should pause before acquiring the start gate");
10175 }
10176
10177 pub(crate) fn release_cancel_before_start(&self) {
10178 self.state.cancel_before_start_release.notify_one();
10179 }
10180
10181 fn pause_cancel_target_timeout(&self) {
10182 self.state.cancel_target_timeout_paused.store(true, Ordering::Release);
10183 }
10184
10185 async fn wait_until_cancel_target_timeout(&self) {
10186 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.cancel_target_timeout_entered.notified())
10187 .await
10188 .expect("cancel should observe contention on its target capacity fence");
10189 }
10190
10191 fn release_cancel_target_timeout(&self) {
10192 self.state.cancel_target_timeout_paused.store(false, Ordering::Release);
10193 self.state.cancel_target_timeout_release.notify_one();
10194 }
10195
10196 #[cfg(feature = "test-util")]
10197 pub(crate) fn release_owner(&self) {
10198 self.state.owner_release.notify_one();
10199 self.state.target_gate_acquire_release.notify_one();
10200 }
10201
10202 #[cfg(feature = "test-util")]
10203 pub(crate) fn disable_owner_pause(&self) {
10204 self.state.owner_pause_enabled.store(false, Ordering::Release);
10205 self.state.owner_release.notify_waiters();
10206 }
10207
10208 #[cfg(feature = "test-util")]
10209 pub(crate) fn pause_external_object_commit_phase(&self) {
10210 self.state.external_object_commit_phase_paused.store(true, Ordering::Release);
10211 }
10212
10213 #[cfg(feature = "test-util")]
10214 pub(crate) fn release_external_object_commit_phase(&self) {
10215 self.state.external_object_commit_phase_release.notify_one();
10216 }
10217
10218 #[cfg(feature = "test-util")]
10219 pub(crate) fn pause_external_object_capacity_probe(&self) {
10220 self.state
10221 .external_object_capacity_probe_paused
10222 .store(true, Ordering::Release);
10223 }
10224
10225 #[cfg(feature = "test-util")]
10226 pub(crate) fn release_external_object_capacity_probe(&self) {
10227 self.state.external_object_capacity_probe_release.notify_one();
10228 }
10229}
10230
10231#[cfg(test)]
10232impl Drop for DecommissionCapacityLockOrderBarrier {
10233 fn drop(&mut self) {
10234 self.state.owner_release.notify_one();
10235 self.state.external_object_capacity_probe_release.notify_one();
10236 self.state.external_object_commit_phase_release.notify_one();
10237 self.state.cancel_before_start_release.notify_one();
10238 self.state.cancel_target_timeout_release.notify_one();
10239 let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10240 .get_or_init(|| std::sync::Mutex::new(None))
10241 .lock()
10242 .expect("decommission capacity lock-order barrier should not be poisoned");
10243 if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
10244 *slot = None;
10245 }
10246 }
10247}
10248
10249#[cfg(test)]
10250async fn pause_decommission_capacity_before_owner_write(store_id: uuid::Uuid) {
10251 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10252 .get_or_init(|| std::sync::Mutex::new(None))
10253 .lock()
10254 .expect("decommission capacity lock-order barrier should not be poisoned")
10255 .as_ref()
10256 .filter(|state| state.owner_store_id == store_id && state.owner_pause_enabled.load(Ordering::Acquire))
10257 .cloned();
10258 if let Some(barrier) = barrier {
10259 barrier.owner_arrived.notify_one();
10260 barrier.owner_release.notified().await;
10261 }
10262}
10263
10264#[cfg(test)]
10265async fn pause_decommission_capacity_before_target_gate_acquire(store_id: uuid::Uuid, target_pool_index: usize) {
10266 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10267 .get_or_init(|| std::sync::Mutex::new(None))
10268 .lock()
10269 .expect("decommission capacity lock-order barrier should not be poisoned")
10270 .as_ref()
10271 .filter(|state| {
10272 state.owner_store_id == store_id
10273 && state.target_gate_acquire_pause_target.load(Ordering::Acquire) == target_pool_index
10274 })
10275 .cloned();
10276 if let Some(barrier) = barrier {
10277 barrier.target_gate_acquire_entered.notify_one();
10278 barrier.target_gate_acquire_release.notified().await;
10279 }
10280}
10281
10282#[cfg(test)]
10283async fn pause_decommission_cancel_before_start_gate(store_id: uuid::Uuid) {
10284 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10285 .get_or_init(|| std::sync::Mutex::new(None))
10286 .lock()
10287 .expect("decommission capacity lock-order barrier should not be poisoned")
10288 .as_ref()
10289 .filter(|state| state.owner_store_id == store_id && state.cancel_before_start_paused.load(Ordering::Acquire))
10290 .cloned();
10291 if let Some(barrier) = barrier {
10292 barrier.cancel_before_start_entered.notify_one();
10293 barrier.cancel_before_start_release.notified().await;
10294 }
10295}
10296
10297#[cfg(test)]
10298async fn pause_decommission_cancel_target_timeout(store_id: uuid::Uuid) {
10299 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10300 .get_or_init(|| std::sync::Mutex::new(None))
10301 .lock()
10302 .expect("decommission capacity lock-order barrier should not be poisoned")
10303 .as_ref()
10304 .filter(|state| state.owner_store_id == store_id)
10305 .cloned();
10306 if let Some(barrier) = barrier {
10307 barrier.cancel_target_timeouts.fetch_add(1, Ordering::AcqRel);
10308 barrier.cancel_target_timeout_entered.notify_one();
10309 if barrier.cancel_target_timeout_paused.load(Ordering::Acquire) {
10310 barrier.cancel_target_timeout_release.notified().await;
10311 }
10312 }
10313}
10314
10315#[cfg(test)]
10316fn notify_decommission_target_gate_retry(store_id: uuid::Uuid) {
10317 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10318 .get_or_init(|| std::sync::Mutex::new(None))
10319 .lock()
10320 .expect("decommission capacity lock-order barrier should not be poisoned")
10321 .as_ref()
10322 .filter(|state| state.owner_store_id == store_id)
10323 .cloned();
10324 if let Some(barrier) = barrier {
10325 barrier.target_gate_retry_entries.fetch_add(1, Ordering::AcqRel);
10326 barrier.target_gate_retry_entered.notify_one();
10327 }
10328}
10329
10330#[cfg(test)]
10331fn notify_decommission_target_gate_inline_retry(store_id: uuid::Uuid) {
10332 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10333 .get_or_init(|| std::sync::Mutex::new(None))
10334 .lock()
10335 .expect("decommission capacity lock-order barrier should not be poisoned")
10336 .as_ref()
10337 .filter(|state| state.owner_store_id == store_id)
10338 .cloned();
10339 if let Some(barrier) = barrier {
10340 barrier.target_gate_inline_retry_entries.fetch_add(1, Ordering::AcqRel);
10341 barrier.target_gate_inline_retry_entered.notify_one();
10342 }
10343}
10344
10345#[cfg(test)]
10346fn notify_decommission_target_gate_exact_reload(store_id: uuid::Uuid) {
10347 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10348 .get_or_init(|| std::sync::Mutex::new(None))
10349 .lock()
10350 .expect("decommission capacity lock-order barrier should not be poisoned")
10351 .as_ref()
10352 .filter(|state| state.owner_store_id == store_id)
10353 .cloned();
10354 if let Some(barrier) = barrier {
10355 barrier.target_gate_exact_reloads.fetch_add(1, Ordering::AcqRel);
10356 }
10357}
10358
10359#[cfg(test)]
10360pub(crate) fn notify_decommission_external_object_capacity_released(store_id: uuid::Uuid) {
10361 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10362 .get_or_init(|| std::sync::Mutex::new(None))
10363 .lock()
10364 .expect("decommission capacity lock-order barrier should not be poisoned")
10365 .as_ref()
10366 .filter(|state| state.external_store_id == store_id)
10367 .cloned();
10368 if let Some(barrier) = barrier {
10369 barrier.external_capacity_released.notify_one();
10370 }
10371}
10372
10373#[cfg(test)]
10374pub(crate) fn notify_decommission_external_object_capacity_probe_acquired(store_id: uuid::Uuid) {
10375 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10376 .get_or_init(|| std::sync::Mutex::new(None))
10377 .lock()
10378 .expect("decommission capacity lock-order barrier should not be poisoned")
10379 .as_ref()
10380 .filter(|state| state.external_store_id == store_id)
10381 .cloned();
10382 if let Some(barrier) = barrier {
10383 barrier.external_object_capacity_probe_acquired.notify_one();
10384 }
10385}
10386
10387#[cfg(test)]
10388pub(crate) async fn wait_for_decommission_external_object_capacity_probe_release(store_id: uuid::Uuid) {
10389 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10390 .get_or_init(|| std::sync::Mutex::new(None))
10391 .lock()
10392 .expect("decommission capacity lock-order barrier should not be poisoned")
10393 .as_ref()
10394 .filter(|state| state.external_store_id == store_id)
10395 .cloned();
10396 if let Some(barrier) = barrier
10397 && barrier.external_object_capacity_probe_paused.load(Ordering::Acquire)
10398 {
10399 barrier.external_object_capacity_probe_release.notified().await;
10400 }
10401}
10402
10403#[cfg(test)]
10404pub(crate) fn notify_decommission_external_object_commit_phase_started(store_id: uuid::Uuid) {
10405 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10406 .get_or_init(|| std::sync::Mutex::new(None))
10407 .lock()
10408 .expect("decommission capacity lock-order barrier should not be poisoned")
10409 .as_ref()
10410 .filter(|state| state.external_store_id == store_id)
10411 .cloned();
10412 if let Some(barrier) = barrier {
10413 barrier.external_object_commit_phase_started.notify_one();
10414 }
10415}
10416
10417#[cfg(test)]
10418pub(crate) async fn wait_for_decommission_external_object_commit_phase_release(store_id: uuid::Uuid) {
10419 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10420 .get_or_init(|| std::sync::Mutex::new(None))
10421 .lock()
10422 .expect("decommission capacity lock-order barrier should not be poisoned")
10423 .as_ref()
10424 .filter(|state| state.external_store_id == store_id)
10425 .cloned();
10426 if let Some(barrier) = barrier
10427 && barrier.external_object_commit_phase_paused.load(Ordering::Acquire)
10428 {
10429 barrier.external_object_commit_phase_release.notified().await;
10430 }
10431}
10432
10433#[cfg(test)]
10434pub(crate) fn notify_decommission_external_heal_operation_started(store_id: uuid::Uuid) {
10435 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10436 .get_or_init(|| std::sync::Mutex::new(None))
10437 .lock()
10438 .expect("decommission capacity lock-order barrier should not be poisoned")
10439 .as_ref()
10440 .filter(|state| state.external_store_id == store_id)
10441 .cloned();
10442 if let Some(barrier) = barrier {
10443 barrier.external_heal_operation_started.notify_one();
10444 }
10445}
10446
10447#[cfg(test)]
10448pub(crate) fn notify_decommission_external_heal_target_lock_attempted() {
10449 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10450 .get_or_init(|| std::sync::Mutex::new(None))
10451 .lock()
10452 .expect("decommission capacity lock-order barrier should not be poisoned")
10453 .as_ref()
10454 .cloned();
10455 if let Some(barrier) = barrier {
10456 barrier.external_heal_target_lock_attempted.notify_one();
10457 }
10458}
10459
10460#[derive(Debug, Default, Clone)]
10461pub struct DecomBucketInfo {
10462 pub name: String,
10463 pub prefix: String,
10464}
10465
10466impl Display for DecomBucketInfo {
10467 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10468 write!(
10469 f,
10470 "{}",
10471 path_join(&[PathBuf::from(self.name.clone()), PathBuf::from(self.prefix.clone())]).to_string_lossy()
10472 )
10473 }
10474}
10475
10476#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10477enum DecommissionFinalState {
10478 Complete,
10479 Failed,
10480}
10481
10482fn determine_decommission_final_state(items_failed: usize, was_cancelled: bool) -> DecommissionFinalState {
10483 if items_failed > 0 || was_cancelled {
10484 DecommissionFinalState::Failed
10485 } else {
10486 DecommissionFinalState::Complete
10487 }
10488}
10489
10490fn decommission_remaining_version_count(versions: &[rustfs_filemeta::FileInfo], expired: usize) -> usize {
10491 versions
10492 .iter()
10493 .filter(|version| !version.tier_free_version())
10494 .count()
10495 .saturating_sub(expired)
10496}
10497
10498fn should_skip_decommission_delete_marker(
10499 version: &rustfs_filemeta::FileInfo,
10500 remaining_versions: usize,
10501 replication_configured: bool,
10502) -> bool {
10503 version.deleted && remaining_versions == 1 && !replication_configured
10506}
10507
10508fn decommission_delete_marker_opts(
10509 version: &rustfs_filemeta::FileInfo,
10510 version_id: Option<String>,
10511 src_pool_idx: usize,
10512 expected_bucket_incarnation_id: Option<uuid::Uuid>,
10513) -> ObjectOptions {
10514 let version_suspended = version.version_id.is_none() && version_id.is_none();
10515 ObjectOptions {
10516 versioned: !version_suspended,
10517 version_suspended,
10518 version_id: version_id.or_else(|| version_suspended.then(|| uuid::Uuid::nil().to_string())),
10519 mod_time: version.mod_time,
10520 src_pool_idx,
10521 data_movement: true,
10522 delete_marker: true,
10523 skip_decommissioned: true,
10524 expected_bucket_incarnation_id,
10525 delete_replication: version
10526 .replication_state_internal
10527 .as_ref()
10528 .map(replication_state_from_filemeta),
10529 ..Default::default()
10530 }
10531}
10532
10533fn decommission_object_migration_read_opts(version_id: Option<String>) -> ObjectOptions {
10534 ObjectOptions {
10535 version_id,
10536 no_lock: true,
10537 data_movement: true,
10538 raw_data_movement_read: true,
10539 skip_decommissioned: true,
10540 skip_rebalancing: true,
10541 ..Default::default()
10542 }
10543}
10544
10545fn decommission_remote_tiered_opts(
10546 version: &rustfs_filemeta::FileInfo,
10547 version_id: Option<String>,
10548 src_pool_idx: usize,
10549 expected_bucket_incarnation_id: Option<uuid::Uuid>,
10550) -> ObjectOptions {
10551 ObjectOptions {
10552 versioned: version_id.is_some(),
10553 version_id,
10554 mod_time: version.mod_time,
10555 user_defined: version.metadata.clone(),
10556 src_pool_idx,
10557 data_movement: true,
10558 incl_free_versions: version.tier_free_version(),
10559 include_part_checksums: true,
10560 http_preconditions: Some(data_movement::data_movement_target_precondition()),
10561 expected_bucket_incarnation_id,
10562 ..Default::default()
10563 }
10564}
10565
10566fn decommission_capacity_version_mutation_id(
10567 owner: DecommissionCapacityOwner,
10568 bucket: &str,
10569 version: &rustfs_filemeta::FileInfo,
10570) -> uuid::Uuid {
10571 let version_id = if version.deleted && version.version_id.is_none() {
10572 Some(uuid::Uuid::nil().to_string())
10573 } else {
10574 version.version_id.map(|version_id| version_id.to_string())
10575 };
10576 decommission_capacity_mutation_id(owner, bucket, &version.name, version_id.as_deref(), version.deleted, version.mod_time)
10577}
10578
10579fn decommission_capacity_owned_opts(mut opts: ObjectOptions, capacity_owner: Option<DecommissionCapacityOwner>) -> ObjectOptions {
10580 if let Some(capacity_owner) = capacity_owner {
10581 capacity_owner.apply_to(&mut opts);
10582 }
10583 opts
10584}
10585
10586fn lifecycle_action_removes_data_movement_version(action: IlmAction) -> bool {
10587 matches!(
10588 action,
10589 IlmAction::DeleteVersionAction | IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction
10590 )
10591}
10592
10593fn lifecycle_action_skips_heal_version(action: IlmAction) -> bool {
10594 action.delete()
10595}
10596
10597#[cfg(test)]
10598struct LifecycleDataMovementMutationBarrierState {
10599 bucket: String,
10600 object: String,
10601 arrived: tokio::sync::Notify,
10602 release: tokio::sync::Notify,
10603}
10604
10605#[cfg(test)]
10606pub(crate) struct LifecycleDataMovementMutationBarrier {
10607 state: Arc<LifecycleDataMovementMutationBarrierState>,
10608}
10609
10610#[cfg(test)]
10611static LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER: std::sync::OnceLock<
10612 std::sync::Mutex<Option<Arc<LifecycleDataMovementMutationBarrierState>>>,
10613> = std::sync::OnceLock::new();
10614
10615#[cfg(test)]
10616impl LifecycleDataMovementMutationBarrier {
10617 pub(crate) fn install(bucket: &str, object: &str) -> Self {
10618 let state = Arc::new(LifecycleDataMovementMutationBarrierState {
10619 bucket: bucket.to_string(),
10620 object: object.to_string(),
10621 arrived: tokio::sync::Notify::new(),
10622 release: tokio::sync::Notify::new(),
10623 });
10624 let mut slot = LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER
10625 .get_or_init(|| std::sync::Mutex::new(None))
10626 .lock()
10627 .expect("lifecycle data movement mutation barrier should not be poisoned");
10628 assert!(slot.is_none(), "lifecycle data movement mutation barrier must be unique");
10629 *slot = Some(Arc::clone(&state));
10630 Self { state }
10631 }
10632
10633 pub(crate) async fn wait_until_paused(&self) {
10634 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.arrived.notified())
10635 .await
10636 .expect("lifecycle data movement should reach its mutation boundary");
10637 }
10638
10639 pub(crate) fn release(&self) {
10640 self.state.release.notify_one();
10641 }
10642}
10643
10644#[cfg(test)]
10645impl Drop for LifecycleDataMovementMutationBarrier {
10646 fn drop(&mut self) {
10647 self.state.release.notify_one();
10648 let mut slot = LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER
10649 .get_or_init(|| std::sync::Mutex::new(None))
10650 .lock()
10651 .expect("lifecycle data movement mutation barrier should not be poisoned");
10652 if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
10653 *slot = None;
10654 }
10655 }
10656}
10657
10658#[cfg(test)]
10659async fn pause_lifecycle_data_movement_mutation(bucket: &str, object: &str, has_run_fence_signal: bool) {
10660 if !has_run_fence_signal {
10661 return;
10662 }
10663 let barrier = LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER
10664 .get_or_init(|| std::sync::Mutex::new(None))
10665 .lock()
10666 .expect("lifecycle data movement mutation barrier should not be poisoned")
10667 .as_ref()
10668 .filter(|barrier| barrier.bucket == bucket && barrier.object == object)
10669 .cloned();
10670 if let Some(barrier) = barrier {
10671 barrier.arrived.notify_one();
10672 barrier.release.notified().await;
10673 }
10674}
10675
10676fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_actions: bool, applied: bool) -> Result<bool> {
10677 if !apply_actions || applied {
10678 return Ok(true);
10679 }
10680
10681 Err(Error::other(format!(
10682 "failed to apply lifecycle expiry action {action:?} during data movement"
10683 )))
10684}
10685
10686#[allow(clippy::too_many_arguments)]
10687pub(crate) async fn should_skip_lifecycle_for_data_movement(
10688 store: Arc<ECStore>,
10689 bucket: &str,
10690 version: &rustfs_filemeta::FileInfo,
10691 lifecycle_config: Option<&BucketLifecycleConfiguration>,
10692 object_lock_config: Option<&ObjectLockConfiguration>,
10693 apply_actions: bool,
10694 event_source: &LcEventSrc,
10695 lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
10696) -> Result<bool> {
10697 let Some(lifecycle_config) = lifecycle_config else {
10698 return Ok(false);
10699 };
10700
10701 let versioned = match BucketVersioningSys::get_in(&store.ctx, bucket).await {
10702 Ok(config) => config.prefix_enabled(&version.name),
10703 Err(err) => {
10704 warn!(
10705 event = EVENT_DECOMMISSION_ENTRY,
10706 component = LOG_COMPONENT_ECSTORE,
10707 subsystem = LOG_SUBSYSTEM_POOLS,
10708 state = "versioning_config_unavailable",
10709 bucket = %bucket,
10710 error = %err,
10711 "Decommission lifecycle versioning config unavailable; treating object as unversioned"
10712 );
10713 false
10714 }
10715 };
10716 let object_info = crate::object_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned);
10717 let event = eval_action_from_lifecycle(lifecycle_config, object_lock_config, &object_info).await;
10718
10719 match event.action {
10720 IlmAction::DeleteRestoredAction | IlmAction::DeleteRestoredVersionAction => {
10721 if apply_actions && object_info.is_remote() {
10722 let Ok(bucket_incarnation_id) = store.bucket_incarnation_id_from_disk(bucket).await else {
10723 return Ok(false);
10724 };
10725 let _ = match lock_lost_signal {
10726 Some(signal) => {
10727 apply_expiry_rule_for_data_movement(store, &event, event_source, &object_info, Some(signal)).await
10728 }
10729 None => {
10730 apply_expiry_on_transitioned_object(store, &object_info, &event, event_source, bucket_incarnation_id)
10731 .await
10732 }
10733 };
10734 }
10735 Ok(false)
10736 }
10737 action if lifecycle_action_removes_data_movement_version(action) => {
10738 #[cfg(test)]
10739 pause_lifecycle_data_movement_mutation(bucket, &version.name, lock_lost_signal.is_some()).await;
10740 if lifecycle_delete_all_versions_blocked_by_replication(store.clone(), bucket, &object_info.name, action).await? {
10741 return Ok(false);
10742 }
10743 let applied = !apply_actions
10744 || match lock_lost_signal {
10745 Some(signal) => {
10746 apply_expiry_rule_for_data_movement(store, &event, event_source, &object_info, Some(signal)).await
10747 }
10748 None => apply_expiry_rule_in(store, &event, event_source, &object_info).await,
10749 };
10750 resolve_data_movement_lifecycle_expiry_result(action, apply_actions, applied)
10751 }
10752 _ => Ok(false),
10753 }
10754}
10755
10756pub struct HealLifecycleExpiryContext {
10757 configs: LifecycleExpiryConfigs,
10758}
10759
10760impl ECStore {
10761 pub async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
10762 if bucket == RUSTFS_META_BUCKET {
10763 return Ok(None);
10764 }
10765
10766 let configs = get_expiry_configs(self, bucket).await?;
10767 if configs.lifecycle.is_none() {
10768 return Ok(None);
10769 }
10770
10771 Ok(Some(HealLifecycleExpiryContext { configs }))
10772 }
10773
10774 pub async fn enqueue_heal_lifecycle_expiry(
10775 self: &Arc<Self>,
10776 context: &HealLifecycleExpiryContext,
10777 bucket: &str,
10778 object: &str,
10779 version_id: Option<&str>,
10780 object_info: Option<&crate::object_api::ObjectInfo>,
10781 ) -> Result<bool> {
10782 let Some(lifecycle_config) = context.configs.lifecycle.as_ref() else {
10783 return Ok(false);
10784 };
10785
10786 let object_info = if let Some(object_info) = object_info {
10787 if object_info.bucket != bucket || object_info.name != object {
10788 return Ok(false);
10789 }
10790 let snapshot_version_id = object_info
10791 .version_id
10792 .filter(|version_id| !version_id.is_nil())
10793 .map(|version_id| version_id.to_string());
10794 if snapshot_version_id.as_deref() != version_id {
10795 return Ok(false);
10796 }
10797 object_info.clone()
10798 } else {
10799 match self
10800 .get_object_info(
10801 bucket,
10802 object,
10803 &ObjectOptions {
10804 version_id: version_id.map(str::to_string),
10805 versioned: version_id.is_some(),
10806 expected_bucket_incarnation_id: Some(context.configs.bucket_incarnation_id),
10807 ..Default::default()
10808 },
10809 )
10810 .await
10811 {
10812 Ok(object_info) => object_info,
10813 Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => return Ok(false),
10814 Err(err) => return Err(err),
10815 }
10816 };
10817
10818 let event = eval_action_from_lifecycle(lifecycle_config, context.configs.object_lock.as_deref(), &object_info).await;
10819 if !lifecycle_action_skips_heal_version(event.action) {
10820 return Ok(false);
10821 }
10822
10823 if lifecycle_delete_all_versions_blocked_by_replication(self.clone(), bucket, &object_info.name, event.action).await? {
10824 return Ok(false);
10825 }
10826
10827 Ok(apply_expiry_rule_in(self.clone(), &event, &LcEventSrc::Scanner, &object_info).await)
10828 }
10829
10830 async fn acquire_pool_meta_write_guard(
10831 &self,
10832 write_state: &mut PoolMetaWriteState,
10833 operation: &str,
10834 ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> {
10835 self.acquire_pool_meta_write_guard_with_lock_error(write_state, operation, activation_pool_meta_lock_error)
10836 .await
10837 }
10838
10839 async fn acquire_pool_meta_write_guard_with_lock_error<F>(
10840 &self,
10841 write_state: &mut PoolMetaWriteState,
10842 operation: &str,
10843 map_lock_error: F,
10844 ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)>
10845 where
10846 F: FnOnce(rustfs_lock::LockError) -> Error,
10847 {
10848 write_state.ensure_write_safe(operation)?;
10849 load_pool_meta_identity_observing(self.pools.clone(), write_state).await?;
10850 let pool = self.pools.first().cloned().ok_or_else(|| {
10851 Error::InvalidArgument(
10852 operation.to_string(),
10853 "storage-pools".to_string(),
10854 "no storage pools available".to_string(),
10855 )
10856 })?;
10857 let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
10858 let pool_meta_guard = pool_meta_lock
10859 .get_write_lock(get_lock_acquire_timeout())
10860 .await
10861 .map_err(map_lock_error)?;
10862 let selection = load_pool_meta_replicas_observing(self.pools.clone(), true, write_state).await?;
10863 write_state.observe_replicas(selection.replica_state);
10864 write_state.ensure_write_safe(operation)?;
10865 Ok((pool_meta_guard, selection.meta))
10866 }
10867
10868 async fn acquire_pool_meta_read_guard(
10869 &self,
10870 write_state: &PoolMetaWriteState,
10871 operation: &str,
10872 ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> {
10873 write_state.ensure_write_safe(operation)?;
10874 let pool = self.pools.first().cloned().ok_or_else(|| {
10875 Error::InvalidArgument(
10876 operation.to_string(),
10877 "storage-pools".to_string(),
10878 "no storage pools available".to_string(),
10879 )
10880 })?;
10881 let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
10882 let pool_meta_guard = pool_meta_lock.get_read_lock(get_lock_acquire_timeout()).await?;
10883 let selection = load_pool_meta_replicas_for_read_probe(self.pools.clone(), true, write_state, operation).await?;
10884 Ok((pool_meta_guard, selection.meta))
10885 }
10886
10887 async fn acquire_decommission_capacity_target_guard(
10888 &self,
10889 target_pool_index: usize,
10890 ) -> Result<rustfs_lock::NamespaceLockGuard> {
10891 let pool = self.pools.first().cloned().ok_or_else(|| {
10895 Error::InvalidArgument(
10896 "decommission-capacity".to_string(),
10897 "storage-pools".to_string(),
10898 "no storage pools available".to_string(),
10899 )
10900 })?;
10901 let object = format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}");
10902 let target_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
10903 #[cfg(test)]
10904 pause_decommission_capacity_before_target_gate_acquire(self.id, target_pool_index).await;
10905 match target_lock
10906 .get_write_lock_quiet(DECOMMISSION_CAPACITY_TARGET_LOCK_TIMEOUT)
10907 .await
10908 {
10909 Ok(guard) => Ok(guard),
10910 Err(rustfs_lock::LockError::Timeout { .. } | rustfs_lock::LockError::AlreadyLocked { .. }) => {
10911 Err(decommission_capacity_blocked_error(format!(
10912 "{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}{target_pool_index}{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}"
10913 )))
10914 }
10915 Err(rustfs_lock::LockError::QuorumNotReached { required, achieved }) => Err(Error::NamespaceLockQuorumUnavailable {
10916 mode: "write",
10917 bucket: RUSTFS_META_BUCKET.to_string(),
10918 object,
10919 required,
10920 achieved,
10921 }),
10922 Err(err) => Err(Error::Lock(err)),
10923 }
10924 }
10925
10926 async fn acquire_decommission_capacity_target_guard_retried(
10934 &self,
10935 target_pool_index: usize,
10936 ) -> Result<rustfs_lock::NamespaceLockGuard> {
10937 let mut gate_attempt = 0;
10938 loop {
10939 match self.acquire_decommission_capacity_target_guard(target_pool_index).await {
10940 Ok(guard) => return Ok(guard),
10941 Err(err) if is_decommission_capacity_target_gate_busy(&err) => {
10942 gate_attempt += 1;
10943 if gate_attempt >= DECOMMISSION_MUTATION_GATE_MAX_INLINE_ATTEMPTS {
10944 return Err(err);
10945 }
10946 #[cfg(test)]
10947 notify_decommission_target_gate_inline_retry(self.id);
10948 record_decommission_capacity_gate_retry(target_pool_index);
10949 let retry_delay = decommission_capacity_gate_retry_delay(gate_attempt);
10950 warn!(
10951 event = EVENT_DECOMMISSION_STATE,
10952 component = LOG_COMPONENT_ECSTORE,
10953 subsystem = LOG_SUBSYSTEM_POOLS,
10954 store_id = %self.id,
10955 target_pool_index,
10956 attempt = gate_attempt,
10957 max_attempts = DECOMMISSION_MUTATION_GATE_MAX_INLINE_ATTEMPTS,
10958 retry_delay_ms = retry_delay.as_millis(),
10959 state = "capacity_gate_inline_retry",
10960 error = %err,
10961 "Decommission target capacity gate busy; retrying mutation inline"
10962 );
10963 tokio::time::sleep(retry_delay).await;
10964 }
10965 Err(err) => return Err(err),
10966 }
10967 }
10968 }
10969
10970 async fn acquire_decommission_capacity_owner_target_guard(
10971 &self,
10972 owner: DecommissionCapacityOwner,
10973 target_pool_index: usize,
10974 ) -> Result<Option<rustfs_lock::NamespaceLockGuard>> {
10975 let model_version = self
10976 .pool_meta
10977 .read()
10978 .await
10979 .pools
10980 .get(owner.source_pool_index)
10981 .and_then(|pool| pool.decommission.as_ref())
10982 .and_then(|info| info.capacity_reservation.as_ref())
10983 .filter(|reservation| {
10984 reservation.active()
10985 && reservation.source_pool_index == owner.source_pool_index
10986 && reservation.operation_id == owner.operation_id
10987 && reservation.generation == owner.generation
10988 })
10989 .map(|reservation| reservation.model_version)
10990 .ok_or_else(|| decommission_capacity_blocked_error("decommission capacity owner is stale"))?;
10991 match model_version {
10992 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION => Ok(None),
10993 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION => {
10994 discard_decommission_capacity_target_permit_except(self.id, owner, Some(target_pool_index));
10995 match take_decommission_capacity_target_permit(self.id, target_pool_index, owner) {
10996 Some(guard) => Ok(Some(guard)),
10997 None => self
10998 .acquire_decommission_capacity_target_guard(target_pool_index)
10999 .await
11000 .map(Some),
11001 }
11002 }
11003 version => Err(Error::DecommissionCapacity(format!(
11004 "decommission capacity owner uses unsupported lock model {version}"
11005 ))),
11006 }
11007 }
11008
11009 async fn acquire_decommission_capacity_terminal_guards(
11010 &self,
11011 source_pool_index: usize,
11012 plan: Option<&DecommissionCapacityTerminalFencePlan>,
11013 ) -> Result<Vec<rustfs_lock::NamespaceLockGuard>> {
11014 let Some(plan) = plan else {
11015 return Ok(Vec::new());
11016 };
11017 match plan.model_version {
11018 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION => return Ok(Vec::new()),
11019 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION => {}
11020 version => {
11021 return Err(Error::DecommissionCapacity(format!(
11022 "decommission terminal transition found unsupported capacity lock model {version}"
11023 )));
11024 }
11025 }
11026 let pool = self.pools.first().cloned().ok_or_else(|| {
11029 Error::InvalidArgument(
11030 "decommission-capacity".to_string(),
11031 "storage-pools".to_string(),
11032 "no storage pools available".to_string(),
11033 )
11034 })?;
11035 let mut attempt = 1;
11038 'acquire_targets: loop {
11039 let mut guards = Vec::with_capacity(plan.target_pool_indices.len());
11040 for &target_pool_index in &plan.target_pool_indices {
11041 let object = format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}");
11042 let target_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
11043 let started = std::time::Instant::now();
11044 match target_lock.get_write_lock(get_lock_acquire_timeout()).await {
11045 Ok(guard) => guards.push(guard),
11046 Err(err @ rustfs_lock::LockError::Timeout { .. }) => {
11047 drop(guards);
11050 #[cfg(test)]
11051 pause_decommission_cancel_target_timeout(self.id).await;
11052 if attempt >= DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS {
11053 return Err(Error::Lock(err));
11054 }
11055 warn!(
11056 event = EVENT_DECOMMISSION_STATE,
11057 component = LOG_COMPONENT_ECSTORE,
11058 subsystem = LOG_SUBSYSTEM_POOLS,
11059 state = "cancel_target_fence_retry",
11060 pool_index = source_pool_index,
11061 target_pool_index,
11062 operation_id = %plan.operation_id,
11063 generation = plan.generation,
11064 owner_nonce = %plan.owner_nonce,
11065 attempt,
11066 max_attempts = DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS,
11067 wait_ms = %started.elapsed().as_millis(),
11068 error = %err,
11069 "Decommission cancel will retry target capacity fencing"
11070 );
11071 tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await;
11072 let save_guard = self.pool_meta_save_gate.lock().await;
11073 let (_read_guard, snapshot) = self
11074 .acquire_pool_meta_read_guard(&save_guard, "decommission cancel fence retry failed")
11075 .await?;
11076 if decommission_capacity_terminal_fence_plan(&snapshot, source_pool_index)?.as_ref() != Some(plan) {
11077 return Err(decommission_capacity_blocked_error(
11078 "decommission capacity owner or target cohort changed while retrying terminal fences",
11079 ));
11080 }
11081 attempt += 1;
11082 continue 'acquire_targets;
11083 }
11084 Err(rustfs_lock::LockError::QuorumNotReached { required, achieved }) => {
11085 return Err(Error::NamespaceLockQuorumUnavailable {
11086 mode: "write",
11087 bucket: RUSTFS_META_BUCKET.to_string(),
11088 object,
11089 required,
11090 achieved,
11091 });
11092 }
11093 Err(err) => return Err(Error::Lock(err)),
11094 }
11095 }
11096 return Ok(guards);
11097 }
11098 }
11099
11100 pub(crate) async fn acquire_external_decommission_capacity_fence(
11101 &self,
11102 target_pool_indices: &[usize],
11103 admission: DecommissionCapacityAdmission,
11104 ) -> Result<rustfs_lock::NamespaceLockGuard> {
11105 Ok(self
11106 .acquire_external_decommission_capacity_fence_with_active_source(target_pool_indices, admission)
11107 .await?
11108 .0)
11109 }
11110
11111 pub(crate) async fn acquire_external_decommission_capacity_fence_with_active_source(
11112 &self,
11113 target_pool_indices: &[usize],
11114 admission: DecommissionCapacityAdmission,
11115 ) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
11116 let save_guard = self.pool_meta_save_gate.lock().await;
11117 let (pool_meta_guard, snapshot) = self
11118 .acquire_pool_meta_read_guard(&save_guard, "target capacity admission failed")
11119 .await?;
11120 for target_pool_index in target_pool_indices.iter().copied() {
11121 ensure_external_decommission_target_admission(&snapshot, target_pool_index, admission)?;
11122 }
11123 let has_active_source = pool_meta_has_active_decommission(&snapshot);
11124 drop(save_guard);
11125 Ok((pool_meta_guard, has_active_source))
11126 }
11127
11128 pub(crate) async fn acquire_pool_meta_object_heal_fence(
11133 &self,
11134 target_pool_indices: &[usize],
11135 ) -> Result<(rustfs_lock::NamespaceLockGuard, Vec<Result<()>>)> {
11136 let mut save_guard = self.pool_meta_save_gate.lock().await;
11137 let (pool_meta_guard, snapshot) = self
11138 .acquire_pool_meta_write_guard_with_lock_error(&mut save_guard, "pool metadata heal admission failed", Error::from)
11139 .await?;
11140 let admissions = target_pool_indices
11141 .iter()
11142 .copied()
11143 .map(|target_pool_index| {
11144 ensure_external_decommission_target_admission(&snapshot, target_pool_index, DecommissionCapacityAdmission::Heal)
11145 })
11146 .collect();
11147 drop(save_guard);
11148 Ok((pool_meta_guard, admissions))
11149 }
11150
11151 pub(crate) async fn acquire_decommission_capacity_release_fence_with_active_source(
11152 &self,
11153 ) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
11154 let save_guard = self.pool_meta_save_gate.lock().await;
11155 let (pool_meta_guard, snapshot) = self
11156 .acquire_pool_meta_read_guard(&save_guard, "capacity release fence failed")
11157 .await?;
11158 let has_active_source = pool_meta_has_active_decommission(&snapshot);
11159 drop(save_guard);
11160 Ok((pool_meta_guard, has_active_source))
11161 }
11162
11163 pub(crate) async fn run_decommission_capacity_admitted_mutation<T, F, Fut>(
11164 &self,
11165 target_pool_index: usize,
11166 capacity_owner: Option<DecommissionCapacityOwner>,
11167 expected_data_bytes: Option<usize>,
11168 operation: F,
11169 ) -> Result<T>
11170 where
11171 F: FnOnce() -> Fut,
11172 Fut: std::future::Future<Output = Result<T>>,
11173 {
11174 self.run_decommission_capacity_mutation(
11175 target_pool_index,
11176 capacity_owner,
11177 expected_data_bytes,
11178 DecommissionCapacityMutationMode::Durable,
11179 |_| false,
11180 |_| operation(),
11181 )
11182 .await
11183 }
11184
11185 pub(crate) async fn run_decommission_capacity_admitted_mutation_with_capacity_lease<T, F, Fut>(
11186 &self,
11187 target_pool_index: usize,
11188 capacity_owner: Option<DecommissionCapacityOwner>,
11189 expected_data_bytes: Option<usize>,
11190 operation: F,
11191 ) -> Result<T>
11192 where
11193 F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11194 Fut: std::future::Future<Output = Result<T>>,
11195 {
11196 self.run_decommission_capacity_mutation(
11197 target_pool_index,
11198 capacity_owner,
11199 expected_data_bytes,
11200 DecommissionCapacityMutationMode::Durable,
11201 |_| false,
11202 operation,
11203 )
11204 .await
11205 }
11206
11207 pub(crate) async fn reconcile_decommission_capacity_before_exact_delete(
11208 &self,
11209 bucket: &str,
11210 object: &str,
11211 opts: &ObjectOptions,
11212 exact: &ObjectInfo,
11213 ) -> Result<()> {
11214 if exact.bucket != bucket || exact.name != decode_dir_object(object) {
11215 return Err(decommission_capacity_blocked_error(
11216 "exact delete object identity changed before capacity reconciliation",
11217 ));
11218 }
11219
11220 let (reconciliations, model_version) = {
11221 let save_guard = self.pool_meta_save_gate.lock().await;
11222 let (_read_guard, snapshot) = self
11223 .acquire_pool_meta_read_guard(&save_guard, "exact delete capacity reconciliation failed")
11224 .await?;
11225 let reconciliations = plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)?;
11226 let model_version = active_decommission_capacity_model(&snapshot)?;
11227 (reconciliations, model_version)
11228 };
11229 if reconciliations.is_empty() {
11230 return Ok(());
11231 }
11232 ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
11233
11234 let mut target_guards = Vec::new();
11235 let mut legacy_write_fence = None;
11236 match model_version {
11237 Some(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION) => {
11238 let mut save_guard = self.pool_meta_save_gate.lock().await;
11239 let (write_guard, snapshot) = self
11240 .acquire_pool_meta_write_guard(&mut save_guard, "exact delete capacity reconciliation failed")
11241 .await?;
11242 if active_decommission_capacity_model(&snapshot)? != model_version
11243 || plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)? != reconciliations
11244 {
11245 return Err(decommission_capacity_blocked_error(
11246 "pending capacity changed before exact target evidence could be verified",
11247 ));
11248 }
11249 legacy_write_fence = Some((save_guard, write_guard, snapshot));
11250 }
11251 Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) => {
11252 let mut target_pool_indices = reconciliations
11253 .iter()
11254 .map(|reconciliation| reconciliation.target_pool_index)
11255 .collect::<Vec<_>>();
11256 target_pool_indices.sort_unstable();
11257 target_pool_indices.dedup();
11258 target_guards.reserve(target_pool_indices.len());
11259 for target_pool_index in target_pool_indices {
11260 let guard = self.acquire_decommission_capacity_target_guard(target_pool_index).await?;
11261 ensure_decommission_capacity_target_fence(&guard, target_pool_index, "exact delete evidence")?;
11262 target_guards.push((target_pool_index, guard));
11263 }
11264 }
11265 Some(version) => {
11266 return Err(Error::DecommissionCapacity(format!(
11267 "exact delete capacity reconciliation found unsupported lock model {version}"
11268 )));
11269 }
11270 None => {
11271 return Err(decommission_capacity_blocked_error(
11272 "exact delete capacity reconciliation has no active reservation lock model",
11273 ));
11274 }
11275 }
11276
11277 let target_lookup_options = ObjectOptions {
11278 versioned: opts.versioned,
11279 version_suspended: opts.version_suspended,
11280 version_id: opts.version_id.clone(),
11281 metadata_chg: opts.version_id.is_some(),
11282 no_lock: true,
11283 ..Default::default()
11284 };
11285 for reconciliation in &reconciliations {
11286 let target_pool = self.pools.get(reconciliation.target_pool_index).ok_or_else(|| {
11287 decommission_capacity_blocked_error(format!(
11288 "source pool {} exact-delete capacity target pool {} is out of range",
11289 reconciliation.source_pool_index, reconciliation.target_pool_index
11290 ))
11291 })?;
11292 let target = target_pool
11293 .get_object_info(bucket, object, &target_lookup_options)
11294 .await
11295 .map_err(|err| {
11296 decommission_capacity_blocked_error(format!(
11297 "source pool {} target pool {} exact object evidence could not be read: {err}",
11298 reconciliation.source_pool_index, reconciliation.target_pool_index
11299 ))
11300 })?;
11301 if !Self::is_equivalent_decommission_capacity_target(exact, &target) {
11302 return Err(decommission_capacity_blocked_error(format!(
11303 "source pool {} target pool {} does not contain an equivalent exact object for its pending capacity intent",
11304 reconciliation.source_pool_index, reconciliation.target_pool_index
11305 )));
11306 }
11307 }
11308 ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
11309
11310 let (mut save_guard, write_guard, mut snapshot) = if let Some(fence) = legacy_write_fence {
11311 fence
11312 } else {
11313 let mut save_guard = self.pool_meta_save_gate.lock().await;
11314 let (write_guard, snapshot) = self
11315 .acquire_pool_meta_write_guard(&mut save_guard, "exact delete capacity reconciliation failed")
11316 .await?;
11317 (save_guard, write_guard, snapshot)
11318 };
11319 if active_decommission_capacity_model(&snapshot)? != model_version {
11320 return Err(decommission_capacity_blocked_error(
11321 "capacity lock model changed while exact target evidence was being verified",
11322 ));
11323 }
11324 let current_reconciliations = plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)?;
11325 if current_reconciliations != reconciliations {
11326 return Err(decommission_capacity_blocked_error(
11327 "pending capacity changed while exact target evidence was being verified",
11328 ));
11329 }
11330 ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
11331 for (target_pool_index, target_guard) in &target_guards {
11332 ensure_decommission_capacity_target_fence(target_guard, *target_pool_index, "exact delete capacity finalize")?;
11333 }
11334
11335 let now = OffsetDateTime::now_utc();
11336 let mut source_pool_indices = Vec::with_capacity(current_reconciliations.len());
11337 for reconciliation in current_reconciliations {
11338 resolve_decommission_target_pending(
11339 &mut snapshot,
11340 reconciliation.source_pool_index,
11341 reconciliation.target_pool_index,
11342 reconciliation.expected_target_physical_bytes,
11343 reconciliation.mutation_id,
11344 )?;
11345 record_decommission_target_consumption(
11346 &mut snapshot,
11347 reconciliation.source_pool_index,
11348 reconciliation.target_pool_index,
11349 DecommissionTargetConsumption {
11350 committed_data_bytes: reconciliation.expected_data_bytes,
11351 target_physical_bytes: reconciliation.expected_target_physical_bytes,
11352 observed_physical_bytes: 0,
11353 },
11354 reconciliation.mutation_id,
11355 now,
11356 )?;
11357 source_pool_indices.push(reconciliation.source_pool_index);
11358 }
11359 source_pool_indices.sort_unstable();
11360 source_pool_indices.dedup();
11361 ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
11362
11363 let outcome = snapshot
11364 .save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &source_pool_indices)
11365 .await?;
11366 ensure_pool_meta_write_fence(&write_guard, "exact delete capacity reconciliation save failed")?;
11367 for (target_pool_index, target_guard) in &target_guards {
11368 ensure_decommission_capacity_target_fence(target_guard, *target_pool_index, "exact delete capacity save")?;
11369 }
11370 {
11371 let mut pool_meta = self.pool_meta.write().await;
11372 for source_pool_index in source_pool_indices {
11373 publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, source_pool_index)?;
11374 }
11375 }
11376 ensure_pool_meta_write_fence(&write_guard, "exact delete capacity reconciliation save failed")?;
11377 for (target_pool_index, target_guard) in &target_guards {
11378 ensure_decommission_capacity_target_fence(target_guard, *target_pool_index, "exact delete capacity publication")?;
11379 }
11380 outcome.disarm();
11381 Ok(())
11382 }
11383
11384 pub(crate) async fn reconcile_decommission_capacity_after_equivalent_target(
11385 &self,
11386 owner: DecommissionCapacityOwner,
11387 target_pool_index: usize,
11388 expected_data_bytes: usize,
11389 ) -> Result<()> {
11390 let target_guard = self
11391 .acquire_decommission_capacity_owner_target_guard(owner, target_pool_index)
11392 .await?;
11393 if let Some(target_guard) = target_guard.as_ref() {
11394 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target reconciliation")?;
11395 }
11396 let mut save_guard = self.pool_meta_save_gate.lock().await;
11397 let (pool_meta_guard, mut snapshot) = self
11398 .acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent target reconciliation failed")
11399 .await?;
11400 let source_pool_index = owner.source_pool_index;
11401 let (target_layout, target_pending_physical_bytes, target_consumed_physical_bytes) = {
11402 let reservation = snapshot
11403 .pools
11404 .get(source_pool_index)
11405 .and_then(|pool| pool.decommission.as_ref())
11406 .and_then(|info| info.capacity_reservation.as_ref())
11407 .filter(|reservation| {
11408 reservation.active()
11409 && reservation.source_pool_index == owner.source_pool_index
11410 && reservation.operation_id == owner.operation_id
11411 && reservation.generation == owner.generation
11412 && reservation.owner_nonce == owner.owner_nonce
11413 })
11414 .ok_or_else(|| decommission_capacity_blocked_error("equivalent target reconciliation owner is stale"))?;
11415 let target = reservation
11416 .targets
11417 .iter()
11418 .find(|target| target.pool_index == target_pool_index)
11419 .ok_or_else(|| decommission_capacity_blocked_error("equivalent target reconciliation allocation is missing"))?;
11420 (target.layout, target.pending_physical_bytes, target.consumed_physical_bytes)
11421 };
11422 let mutation_id = owner.mutation_id.ok_or_else(|| {
11423 decommission_capacity_blocked_error("equivalent target reconciliation mutation identity is missing")
11424 })?;
11425 let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
11426 if target_pending_physical_bytes == 0 {
11427 if target_consumed_physical_bytes >= expected_target_physical_bytes {
11428 let mut pool_meta = self.pool_meta.write().await;
11429 ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent target idempotent publication")?;
11430 publish_decommission_capacity_update(&mut pool_meta, &snapshot, source_pool_index)?;
11431 if let Some(target_guard) = target_guard.as_ref() {
11432 ensure_decommission_capacity_target_fence(
11433 target_guard,
11434 target_pool_index,
11435 "equivalent target idempotent publication",
11436 )?;
11437 }
11438 return Ok(());
11439 }
11440 return Err(decommission_capacity_blocked_error(
11441 "equivalent target has no pending capacity intent to reconcile",
11442 ));
11443 }
11444 if target_pending_physical_bytes < expected_target_physical_bytes {
11445 return Err(decommission_capacity_blocked_error(
11446 "equivalent target pending capacity is smaller than the committed object",
11447 ));
11448 }
11449
11450 resolve_decommission_target_pending(
11451 &mut snapshot,
11452 source_pool_index,
11453 target_pool_index,
11454 expected_target_physical_bytes,
11455 mutation_id,
11456 )?;
11457 record_decommission_target_consumption(
11458 &mut snapshot,
11459 source_pool_index,
11460 target_pool_index,
11461 DecommissionTargetConsumption {
11462 committed_data_bytes: expected_data_bytes,
11463 target_physical_bytes: expected_target_physical_bytes,
11464 observed_physical_bytes: 0,
11465 },
11466 mutation_id,
11467 OffsetDateTime::now_utc(),
11468 )?;
11469 let outcome = snapshot
11470 .save_no_lock_armed(
11471 self.pools.clone(),
11472 &mut save_guard,
11473 pool_meta_guard.lock_lost_signal(),
11474 &[source_pool_index],
11475 )
11476 .await?;
11477 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission equivalent target reconciliation save failed")?;
11478 if let Some(target_guard) = target_guard.as_ref() {
11479 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target capacity save")?;
11480 }
11481 {
11482 let mut pool_meta = self.pool_meta.write().await;
11483 publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, source_pool_index)?;
11484 }
11485 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission equivalent target reconciliation save failed")?;
11486 if let Some(target_guard) = target_guard.as_ref() {
11487 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target capacity publication")?;
11488 }
11489 outcome.disarm();
11490 Ok(())
11491 }
11492
11493 pub(crate) async fn run_decommission_capacity_temporary_mutation<T, F, Fut>(
11494 &self,
11495 target_pool_index: usize,
11496 capacity_owner: Option<DecommissionCapacityOwner>,
11497 expected_data_bytes: Option<usize>,
11498 operation: F,
11499 ) -> Result<T>
11500 where
11501 F: FnOnce() -> Fut,
11502 Fut: std::future::Future<Output = Result<T>>,
11503 {
11504 self.run_decommission_capacity_mutation(
11505 target_pool_index,
11506 capacity_owner,
11507 expected_data_bytes,
11508 DecommissionCapacityMutationMode::Temporary,
11509 |_| false,
11510 |_| operation(),
11511 )
11512 .await
11513 }
11514
11515 pub(crate) async fn run_decommission_capacity_temporary_mutation_with_capacity_lease<T, F, Fut>(
11516 &self,
11517 target_pool_index: usize,
11518 capacity_owner: Option<DecommissionCapacityOwner>,
11519 expected_data_bytes: Option<usize>,
11520 operation: F,
11521 ) -> Result<T>
11522 where
11523 F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11524 Fut: std::future::Future<Output = Result<T>>,
11525 {
11526 self.run_decommission_capacity_mutation(
11527 target_pool_index,
11528 capacity_owner,
11529 expected_data_bytes,
11530 DecommissionCapacityMutationMode::Temporary,
11531 |_| false,
11532 operation,
11533 )
11534 .await
11535 }
11536
11537 pub(crate) async fn run_decommission_capacity_non_growing_replacement_with_capacity_lease<T, F, Fut>(
11542 &self,
11543 target_pool_index: usize,
11544 capacity_owner: Option<DecommissionCapacityOwner>,
11545 expected_data_bytes: Option<usize>,
11546 operation: F,
11547 ) -> Result<T>
11548 where
11549 F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11550 Fut: std::future::Future<Output = Result<T>>,
11551 {
11552 self.run_decommission_capacity_mutation(
11553 target_pool_index,
11554 capacity_owner,
11555 expected_data_bytes,
11556 DecommissionCapacityMutationMode::NonGrowingReplacement,
11557 |_| false,
11558 operation,
11559 )
11560 .await
11561 }
11562
11563 pub(crate) async fn reconcile_decommission_capacity_after_equivalent_temporary_target(
11569 &self,
11570 owner: DecommissionCapacityOwner,
11571 target_pool_index: usize,
11572 expected_data_bytes: usize,
11573 ) -> Result<()> {
11574 let target_guard = self
11575 .acquire_decommission_capacity_owner_target_guard(owner, target_pool_index)
11576 .await?;
11577 if let Some(target_guard) = target_guard.as_ref() {
11578 ensure_decommission_capacity_target_fence(
11579 target_guard,
11580 target_pool_index,
11581 "equivalent temporary target reconciliation",
11582 )?;
11583 }
11584 let mut save_guard = self.pool_meta_save_gate.lock().await;
11585 let (pool_meta_guard, mut snapshot) = self
11586 .acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent temporary target reconciliation failed")
11587 .await?;
11588 let source_pool_index = owner.source_pool_index;
11589 let mutation_id = owner
11590 .mutation_id
11591 .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target mutation identity is missing"))?;
11592 let (target_layout, pending_physical_bytes, pending_mutation_id, has_temporary_mutation) = {
11593 let reservation = snapshot
11594 .pools
11595 .get(source_pool_index)
11596 .and_then(|pool| pool.decommission.as_ref())
11597 .and_then(|info| info.capacity_reservation.as_ref())
11598 .filter(|reservation| reservation.admits_cleanup_owner(owner))
11599 .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target owner is stale"))?;
11600 let target = reservation
11601 .targets
11602 .iter()
11603 .find(|target| target.pool_index == target_pool_index)
11604 .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target allocation is missing"))?;
11605 (
11606 target.layout,
11607 target.pending_physical_bytes,
11608 target.pending_mutation_id,
11609 target
11610 .temporary_mutations
11611 .iter()
11612 .any(|mutation| mutation.mutation_id == mutation_id),
11613 )
11614 };
11615 if pending_physical_bytes == 0 && !has_temporary_mutation {
11616 ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation fence failed")?;
11618 if let Some(target_guard) = target_guard.as_ref() {
11619 ensure_decommission_capacity_target_fence(
11620 target_guard,
11621 target_pool_index,
11622 "equivalent temporary target idempotent reconciliation",
11623 )?;
11624 }
11625 return Ok(());
11626 }
11627 if pending_physical_bytes > 0 && pending_mutation_id != Some(mutation_id) {
11628 return Err(decommission_capacity_blocked_error(
11629 "equivalent temporary target pending intent belongs to another mutation",
11630 ));
11631 }
11632 let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
11633 if pending_physical_bytes > 0 && pending_physical_bytes != expected_target_physical_bytes {
11634 return Err(decommission_capacity_blocked_error(
11635 "equivalent temporary target pending capacity does not match the committed checkpoint",
11636 ));
11637 }
11638 if pending_physical_bytes > 0 {
11639 resolve_decommission_target_pending(
11640 &mut snapshot,
11641 source_pool_index,
11642 target_pool_index,
11643 expected_target_physical_bytes,
11644 mutation_id,
11645 )?;
11646 }
11647 settle_decommission_target_non_growing_replacement(
11652 &mut snapshot,
11653 source_pool_index,
11654 target_pool_index,
11655 mutation_id,
11656 OffsetDateTime::now_utc(),
11657 )?;
11658 let outcome = snapshot
11659 .save_no_lock_armed(
11660 self.pools.clone(),
11661 &mut save_guard,
11662 pool_meta_guard.lock_lost_signal(),
11663 &[source_pool_index],
11664 )
11665 .await?;
11666 ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?;
11667 if let Some(target_guard) = target_guard.as_ref() {
11668 ensure_decommission_capacity_target_fence(
11669 target_guard,
11670 target_pool_index,
11671 "equivalent temporary target capacity save",
11672 )?;
11673 }
11674 {
11675 let mut pool_meta = self.pool_meta.write().await;
11676 publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, source_pool_index)?;
11677 }
11678 ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?;
11679 if let Some(target_guard) = target_guard.as_ref() {
11680 ensure_decommission_capacity_target_fence(
11681 target_guard,
11682 target_pool_index,
11683 "equivalent temporary target capacity publication",
11684 )?;
11685 }
11686 outcome.disarm();
11687 Ok(())
11688 }
11689
11690 pub(crate) async fn has_decommission_capacity_temporary_mutation_state(
11691 &self,
11692 target_pool_index: usize,
11693 owner: DecommissionCapacityOwner,
11694 ) -> bool {
11695 let Some(mutation_id) = owner.mutation_id else {
11696 return false;
11697 };
11698 self.pool_meta
11699 .read()
11700 .await
11701 .pools
11702 .get(owner.source_pool_index)
11703 .and_then(|pool| pool.decommission.as_ref())
11704 .and_then(|info| info.capacity_reservation.as_ref())
11705 .filter(|reservation| reservation.admits_cleanup_owner(owner))
11706 .and_then(|reservation| {
11707 reservation
11708 .targets
11709 .iter()
11710 .find(|target| target.pool_index == target_pool_index)
11711 .map(|target| (reservation.model_version, target))
11712 })
11713 .is_some_and(|(model_version, target)| {
11714 if model_version == DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION {
11715 true
11718 } else {
11719 (target.pending_physical_bytes > 0 && target.pending_mutation_id == Some(mutation_id))
11720 || target
11721 .temporary_mutations
11722 .iter()
11723 .any(|mutation| mutation.mutation_id == mutation_id)
11724 }
11725 })
11726 }
11727
11728 pub(crate) async fn decommission_capacity_cleanup_target_indices(
11729 &self,
11730 owner: DecommissionCapacityOwner,
11731 ) -> Result<Vec<usize>> {
11732 self.pool_meta
11733 .read()
11734 .await
11735 .pools
11736 .get(owner.source_pool_index)
11737 .and_then(|pool| pool.decommission.as_ref())
11738 .and_then(|info| info.capacity_reservation.as_ref())
11739 .filter(|reservation| reservation.admits_cleanup_owner(owner))
11740 .map(|reservation| reservation.targets.iter().map(|target| target.pool_index).collect())
11741 .ok_or_else(|| decommission_capacity_blocked_error("decommission multipart cleanup reservation is stale"))
11742 }
11743
11744 pub(crate) async fn run_decommission_capacity_temporary_release_with_capacity_lease<T, F, Fut>(
11745 &self,
11746 target_pool_index: usize,
11747 capacity_owner: Option<DecommissionCapacityOwner>,
11748 operation: F,
11749 ) -> Result<T>
11750 where
11751 F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11752 Fut: std::future::Future<Output = Result<(T, bool)>>,
11753 {
11754 self.run_decommission_capacity_mutation(
11755 target_pool_index,
11756 capacity_owner,
11757 None,
11758 DecommissionCapacityMutationMode::TemporaryRelease,
11759 |result: &(T, bool)| result.1,
11760 operation,
11761 )
11762 .await
11763 .map(|(result, _)| result)
11764 }
11765
11766 async fn run_decommission_capacity_mutation<T, F, Fut, P>(
11767 &self,
11768 target_pool_index: usize,
11769 capacity_owner: Option<DecommissionCapacityOwner>,
11770 expected_data_bytes: Option<usize>,
11771 mode: DecommissionCapacityMutationMode,
11772 clear_pending_on_temporary_release: P,
11773 operation: F,
11774 ) -> Result<T>
11775 where
11776 F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11777 Fut: std::future::Future<Output = Result<T>>,
11778 P: Fn(&T) -> bool,
11779 {
11780 let temporary = matches!(
11781 mode,
11782 DecommissionCapacityMutationMode::Temporary | DecommissionCapacityMutationMode::NonGrowingReplacement
11783 );
11784 let non_growing_replacement = matches!(mode, DecommissionCapacityMutationMode::NonGrowingReplacement);
11785 let temporary_release = matches!(mode, DecommissionCapacityMutationMode::TemporaryRelease);
11786 let mut operation = Some(operation);
11787 let save_guard = self.pool_meta_save_gate.lock().await;
11788 let (read_guard, snapshot) = self
11789 .acquire_pool_meta_read_guard(&save_guard, "target capacity admission failed")
11790 .await?;
11791 let admission_now = OffsetDateTime::now_utc();
11792 let admitted_owner = capacity_owner.and_then(|owner| {
11793 snapshot
11794 .pools
11795 .get(owner.source_pool_index)
11796 .and_then(|pool| pool.decommission.as_ref())
11797 .and_then(|info| info.capacity_reservation.as_ref())
11798 .filter(|reservation| {
11799 if temporary_release {
11800 reservation.admits_cleanup_owner(owner)
11801 } else {
11802 reservation.admits_owner(owner, admission_now)
11803 }
11804 })
11805 .filter(|reservation| {
11806 !temporary_release
11807 || reservation
11808 .targets
11809 .iter()
11810 .any(|target| target.pool_index == target_pool_index)
11811 })
11812 .map(|reservation| (owner, reservation.model_version))
11813 });
11814 if capacity_owner.is_some() && admitted_owner.is_none() {
11815 return Err(decommission_capacity_blocked_error(
11816 "decommission target mutation reservation identity is stale",
11817 ));
11818 }
11819 let Some((owner, model_version)) = admitted_owner else {
11820 ensure_external_decommission_target_admission(&snapshot, target_pool_index, DecommissionCapacityAdmission::Mutation)?;
11821 drop(save_guard);
11822 let capacity_lease = read_guard.lock_lost_signal();
11823 return operation.take().expect("capacity-admitted operation should run once")(capacity_lease).await;
11824 };
11825
11826 drop(read_guard);
11829 drop(save_guard);
11830 if model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION {
11831 discard_decommission_capacity_target_permit_except(self.id, owner, Some(target_pool_index));
11832 }
11833 let target_guard = match model_version {
11834 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION => None,
11835 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION => {
11836 Some(match take_decommission_capacity_target_permit(self.id, target_pool_index, owner) {
11837 Some(guard) => guard,
11838 None => {
11839 self.acquire_decommission_capacity_target_guard_retried(target_pool_index)
11840 .await?
11841 }
11842 })
11843 }
11844 version => {
11845 return Err(Error::DecommissionCapacity(format!(
11846 "decommission target mutation found unsupported capacity lock model {version}"
11847 )));
11848 }
11849 };
11850
11851 #[cfg(test)]
11852 pause_decommission_capacity_before_owner_write(self.id).await;
11853 let mut save_guard = self.pool_meta_save_gate.lock().await;
11854 let (write_guard, mut snapshot) = self
11855 .acquire_pool_meta_write_guard(&mut save_guard, "decommission target capacity admission failed")
11856 .await?;
11857 let mutation_id = owner
11858 .mutation_id
11859 .ok_or_else(|| decommission_capacity_blocked_error("decommission mutation identity is missing"))?;
11860 let source_pool_index = owner.source_pool_index;
11861 let owner_current = snapshot
11862 .pools
11863 .get(source_pool_index)
11864 .and_then(|pool| pool.decommission.as_ref())
11865 .and_then(|info| info.capacity_reservation.as_ref())
11866 .filter(|reservation| {
11867 (if temporary_release {
11868 reservation.admits_cleanup_owner(owner)
11869 } else {
11870 reservation.admits_owner(owner, OffsetDateTime::now_utc())
11871 }) && reservation.model_version == model_version
11872 && (!temporary_release
11873 || reservation
11874 .targets
11875 .iter()
11876 .any(|target| target.pool_index == target_pool_index))
11877 })
11878 .is_some();
11879 if !owner_current {
11880 return Err(decommission_capacity_blocked_error(
11881 "decommission target mutation reservation identity changed before commit",
11882 ));
11883 }
11884 let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
11885 let reservation_grew = if temporary_release {
11886 false
11887 } else {
11888 let data_bytes = expected_data_bytes.unwrap_or(1).max(1);
11889 let grew = grow_decommission_target_reservation(
11890 &mut snapshot,
11891 &capacity_infos,
11892 source_pool_index,
11893 target_pool_index,
11894 data_bytes,
11895 mutation_id,
11896 OffsetDateTime::now_utc(),
11897 )?;
11898 ensure_decommission_capacity_reservations_available(&snapshot, &capacity_infos, "mutation")?;
11899 grew
11900 };
11901 let target_layout = snapshot
11902 .pools
11903 .get(source_pool_index)
11904 .and_then(|pool| pool.decommission.as_ref())
11905 .and_then(|info| info.capacity_reservation.as_ref())
11906 .and_then(|reservation| {
11907 reservation
11908 .targets
11909 .iter()
11910 .find(|target| target.pool_index == target_pool_index)
11911 })
11912 .map(|target| target.layout)
11913 .ok_or_else(|| Error::SlowDown)?;
11914 let expected_target_physical_bytes = if temporary_release {
11915 0
11916 } else {
11917 capacity_target_physical_bytes(expected_data_bytes.unwrap_or(1).max(1), target_layout)?
11918 };
11919 if temporary {
11920 let (remaining, inflight) = snapshot
11921 .pools
11922 .get(source_pool_index)
11923 .and_then(|pool| pool.decommission.as_ref())
11924 .and_then(|info| info.capacity_reservation.as_ref())
11925 .and_then(|reservation| {
11926 reservation
11927 .targets
11928 .iter()
11929 .find(|target| target.pool_index == target_pool_index)
11930 .map(|target| {
11931 (
11932 target.remaining_reserved_physical_bytes(reservation.temporary_copies)
11933 / 1usize.saturating_add(reservation.temporary_copies),
11934 target.inflight_physical_bytes,
11935 )
11936 })
11937 })
11938 .unwrap_or_default();
11939 let available = remaining.saturating_sub(inflight);
11940 if expected_target_physical_bytes > available {
11941 return Err(decommission_capacity_blocked_error(format!(
11942 "source pool {source_pool_index} target pool {target_pool_index} temporary operation requires {expected_target_physical_bytes} physical bytes, but only {available} temporary-copy bytes remain"
11943 )));
11944 }
11945 } else {
11946 ensure_decommission_target_owner_admission(
11947 &snapshot,
11948 owner,
11949 target_pool_index,
11950 expected_target_physical_bytes,
11951 OffsetDateTime::now_utc(),
11952 )?;
11953 }
11954 let pending_added = if temporary_release {
11955 0
11956 } else {
11957 reserve_decommission_target_pending(
11958 &mut snapshot,
11959 source_pool_index,
11960 target_pool_index,
11961 expected_target_physical_bytes,
11962 mutation_id,
11963 OffsetDateTime::now_utc(),
11964 )?
11965 };
11966 if reservation_grew || pending_added > 0 {
11967 let outcome = snapshot
11968 .save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &[source_pool_index])
11969 .await?;
11970 ensure_pool_meta_write_fence(&write_guard, "decommission target capacity intent save failed")?;
11971 snapshot = outcome.committed.clone();
11972 {
11973 let mut pool_meta = self.pool_meta.write().await;
11974 publish_decommission_capacity_update(&mut pool_meta, &snapshot, source_pool_index)?;
11975 }
11976 ensure_pool_meta_write_fence(&write_guard, "decommission target capacity intent save failed")?;
11977 outcome.disarm();
11978 }
11979 let mut write_guard = Some(write_guard);
11980 let mut save_guard = Some(save_guard);
11981 let capacity_lease = if let Some(target_guard) = target_guard.as_ref() {
11982 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity intent prepare")?;
11983 let capacity_lease = target_guard.lock_lost_signal();
11984 drop(write_guard.take());
11985 drop(save_guard.take());
11986 capacity_lease
11987 } else {
11988 write_guard
11989 .as_ref()
11990 .ok_or_else(|| Error::other("legacy decommission capacity fence disappeared before target mutation"))?
11991 .lock_lost_signal()
11992 };
11993
11994 let capacity_infos = if pending_added > 0 {
11995 self.get_decommission_all_pool_capacity_infos().await?
11996 } else {
11997 capacity_infos
11998 };
11999 let before_free = capacity_infos
12000 .iter()
12001 .find(|capacity| capacity.pool_index == target_pool_index)
12002 .map(|capacity| capacity.physical_free)
12003 .ok_or_else(|| decommission_capacity_blocked_error("target capacity snapshot is missing before mutation"))?;
12004
12005 let result = operation.take().expect("capacity-admitted operation should run once")(capacity_lease).await;
12006 let clear_pending_on_temporary_release = result.as_ref().ok().is_some_and(&clear_pending_on_temporary_release);
12007 if let Some(target_guard) = target_guard.as_ref() {
12008 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "target mutation")?;
12009 let mut finalize_save_guard = self.pool_meta_save_gate.lock().await;
12010 let (finalize_write_guard, finalize_snapshot) = self
12011 .acquire_pool_meta_write_guard(&mut finalize_save_guard, "decommission target capacity finalize failed")
12012 .await?;
12013 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity finalize")?;
12014 save_guard = Some(finalize_save_guard);
12015 write_guard = Some(finalize_write_guard);
12016 snapshot = finalize_snapshot;
12017 }
12018 let write_guard = write_guard
12019 .as_ref()
12020 .ok_or_else(|| Error::other("decommission capacity write fence disappeared before finalize"))?;
12021 let save_guard = save_guard
12022 .as_mut()
12023 .ok_or_else(|| Error::other("decommission capacity save fence disappeared before finalize"))?;
12024 ensure_pool_meta_write_fence(write_guard, "decommission target capacity finalize failed")?;
12025 ensure_decommission_capacity_mutation_intent_current(
12026 &snapshot,
12027 owner,
12028 target_pool_index,
12029 expected_target_physical_bytes,
12030 mutation_id,
12031 temporary_release,
12032 model_version,
12033 )?;
12034 let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
12035 let after_free = capacity_infos
12036 .iter()
12037 .find(|capacity| capacity.pool_index == target_pool_index)
12038 .map(|capacity| capacity.physical_free)
12039 .ok_or_else(|| decommission_capacity_blocked_error("target capacity snapshot is missing after mutation"))?;
12040 let observed_physical_bytes = before_free.saturating_sub(after_free);
12041 let released_physical_bytes = after_free.saturating_sub(before_free);
12042 let now = OffsetDateTime::now_utc();
12043 let progress_changed = if temporary_release {
12044 if result.is_ok() {
12045 release_decommission_target_inflight(
12046 &mut snapshot,
12047 source_pool_index,
12048 target_pool_index,
12049 released_physical_bytes,
12050 mutation_id,
12051 DecommissionCapacityReleaseProof::confirmed_absence(clear_pending_on_temporary_release),
12052 now,
12053 )?
12054 } else {
12055 false
12056 }
12057 } else if result.is_ok() {
12058 resolve_decommission_target_pending(
12059 &mut snapshot,
12060 source_pool_index,
12061 target_pool_index,
12062 expected_target_physical_bytes,
12063 mutation_id,
12064 )?;
12065 if temporary {
12066 if non_growing_replacement {
12067 settle_decommission_target_non_growing_replacement(
12068 &mut snapshot,
12069 source_pool_index,
12070 target_pool_index,
12071 mutation_id,
12072 now,
12073 )?;
12074 } else {
12075 record_decommission_target_inflight(
12076 &mut snapshot,
12077 source_pool_index,
12078 target_pool_index,
12079 observed_physical_bytes,
12080 mutation_id,
12081 now,
12082 )?;
12083 }
12084 } else {
12085 let consumed_physical_bytes = expected_data_bytes
12086 .map(|_| expected_target_physical_bytes)
12087 .unwrap_or(observed_physical_bytes.max(expected_target_physical_bytes));
12088 let committed_data_bytes =
12089 expected_data_bytes.unwrap_or(capacity_source_data_equivalent(observed_physical_bytes, target_layout)?);
12090 record_decommission_target_consumption(
12091 &mut snapshot,
12092 source_pool_index,
12093 target_pool_index,
12094 DecommissionTargetConsumption {
12095 committed_data_bytes,
12096 target_physical_bytes: consumed_physical_bytes,
12097 observed_physical_bytes,
12098 },
12099 mutation_id,
12100 now,
12101 )?;
12102 }
12103 true
12104 } else if temporary {
12105 record_decommission_target_inflight(
12106 &mut snapshot,
12107 source_pool_index,
12108 target_pool_index,
12109 observed_physical_bytes,
12110 mutation_id,
12111 now,
12112 )?
12113 } else {
12114 record_decommission_target_observation(
12115 &mut snapshot,
12116 source_pool_index,
12117 target_pool_index,
12118 observed_physical_bytes,
12119 now,
12120 )?;
12121 observed_physical_bytes > 0
12122 };
12123 let capacity_result = if temporary_release {
12124 Ok(())
12125 } else {
12126 ensure_decommission_capacity_reservations_available(&snapshot, &capacity_infos, "mutation")
12127 };
12128 if let Err(err) = &capacity_result {
12129 snapshot.mark_decommission_capacity_blocked(source_pool_index, err.to_string(), now)?;
12130 }
12131 if temporary_release && !progress_changed {
12132 ensure_pool_meta_write_fence(write_guard, "decommission target capacity cleanup fence failed")?;
12133 if let Some(target_guard) = target_guard.as_ref() {
12134 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity cleanup")?;
12135 }
12136 capacity_result?;
12137 return result;
12138 }
12139 let outcome = snapshot
12140 .save_no_lock_armed(self.pools.clone(), save_guard, write_guard.lock_lost_signal(), &[source_pool_index])
12141 .await?;
12142 ensure_pool_meta_write_fence(write_guard, "decommission target capacity progress save failed")?;
12143 if let Some(target_guard) = target_guard.as_ref() {
12144 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity progress save")?;
12145 }
12146 {
12147 let mut pool_meta = self.pool_meta.write().await;
12148 publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, source_pool_index)?;
12149 }
12150 ensure_pool_meta_write_fence(write_guard, "decommission target capacity progress save failed")?;
12151 if let Some(target_guard) = target_guard.as_ref() {
12152 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity progress publication")?;
12153 }
12154 outcome.disarm();
12155 capacity_result?;
12156 result
12157 }
12158
12159 pub(crate) async fn ensure_pool_meta_side_effects_safe(&self, operation: &str) -> Result<()> {
12160 self.pool_meta_save_gate.lock().await.ensure_write_safe(operation)
12161 }
12162
12163 pub async fn pool_meta_writes_ready(&self) -> bool {
12167 self.pool_meta_write_status().await.is_ok()
12168 }
12169
12170 pub async fn pool_meta_write_gate_status(&self) -> PoolMetaWriteGateStatus {
12172 let Ok(write_state) = tokio::time::timeout(std::time::Duration::from_millis(100), self.pool_meta_save_gate.lock()).await
12173 else {
12174 return PoolMetaWriteGateStatus {
12175 writes_ready: false,
12176 check_timed_out: true,
12177 ..PoolMetaWriteGateStatus::default()
12178 };
12179 };
12180 let transaction_aborted = write_state.aborted_transaction.load(Ordering::SeqCst);
12181 let writes_ready = !write_state.write_blocked && !transaction_aborted;
12182 let failure = if writes_ready {
12183 None
12184 } else {
12185 write_state.ensure_write_safe("pool metadata snapshot").err()
12186 };
12187 let context = failure.as_ref().and_then(Error::pool_metadata_failure);
12188 PoolMetaWriteGateStatus {
12189 writes_ready,
12190 check_timed_out: false,
12191 write_blocked: write_state.write_blocked,
12192 transaction_aborted,
12193 pool_meta_absent: write_state.pool_meta_absent,
12194 identity_initialized: write_state.identity_initialized,
12195 identity_needs_repair: write_state.identity_needs_repair,
12196 cluster_epoch: write_state.cluster_epoch,
12197 reason: context.map(|context| context.kind.as_str()),
12198 phase: context.map(|context| context.phase),
12199 since_unix_secs: context.map(|context| context.since.unix_timestamp()),
12200 }
12201 }
12202
12203 pub async fn pool_meta_write_status(&self) -> Result<()> {
12205 let write_state = tokio::time::timeout(std::time::Duration::from_millis(100), self.pool_meta_save_gate.lock())
12206 .await
12207 .map_err(|_| Error::Timeout)?;
12208 write_state.ensure_write_safe("pool metadata readiness")
12209 }
12210
12211 pub(crate) async fn recover_pool_meta_transaction(&self) -> Result<bool> {
12212 {
12213 let Ok(state) = self.pool_meta_save_gate.try_lock() else {
12214 return Ok(false);
12215 };
12216 if state.ensure_write_safe("pool metadata recovery").is_ok() {
12217 return Ok(false);
12218 }
12219 if state.write_blocked
12220 || state.active_transactions.load(Ordering::SeqCst) != 0
12221 || state
12222 .transaction_failure
12223 .lock()
12224 .unwrap_or_else(std::sync::PoisonError::into_inner)
12225 .is_none()
12226 {
12227 return Ok(false);
12228 }
12229 }
12230 let _start_guard = self.start_gate.lock().await;
12231 {
12235 let cancelers = self.decommission_cancelers.read().await;
12236 let mut draining = false;
12237 for canceler in cancelers.iter().flatten().filter(|canceler| canceler.is_active()) {
12238 canceler.cancel();
12239 draining = true;
12240 }
12241 if draining {
12242 return Err(Error::OperationCanceled);
12243 }
12244 }
12245 let movement_gate = self.ctx.data_movement_operation_gate();
12246 let _movement_guard = movement_gate.write().await;
12247 let mut state = self.pool_meta_save_gate.lock().await;
12248 if state.write_blocked || state.active_transactions.load(Ordering::SeqCst) != 0 {
12251 return Ok(false);
12252 }
12253 let Some(blocked) = state
12254 .transaction_failure
12255 .lock()
12256 .unwrap_or_else(std::sync::PoisonError::into_inner)
12257 .clone()
12258 else {
12259 return Ok(false);
12260 };
12261 if self
12262 .rebalance_meta
12263 .read()
12264 .await
12265 .as_ref()
12266 .is_some_and(|meta| meta.cancel.is_some())
12267 {
12268 return Err(Error::other("pool metadata recovery requires rebalance workers to be quiescent"));
12269 }
12270 let pool = self
12271 .pools
12272 .first()
12273 .ok_or_else(|| Error::other("pool metadata recovery has no storage pools"))?;
12274 let lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
12275 let guard = lock.get_write_lock(get_lock_acquire_timeout()).await?;
12276 let fence = PoolMetaPersistenceFence::Distributed(guard.lock_lost_signal());
12277 let mut candidate = PoolMetaWriteState {
12278 aborted_transaction: Arc::new(AtomicBool::new(false)),
12279 transaction_failure: Arc::default(),
12280 active_transactions: Arc::default(),
12281 block_context: None,
12282 recovery_failure: None,
12283 write_blocked: false,
12284 ..state.clone()
12285 };
12286 let selection = load_pool_meta_for_transaction_recovery(self.pools.clone(), &mut candidate).await?;
12287 if selection.meta.pools.len() != self.pools.len()
12288 || selection
12289 .meta
12290 .pools
12291 .iter()
12292 .zip(&self.pools)
12293 .enumerate()
12294 .any(|(index, (meta, pool))| meta.id != index || meta.cmd_line != pool.endpoints.cmd_line)
12295 {
12296 return Err(Error::other(
12297 "pool metadata recovery requires operator reconciliation of the pool topology",
12298 ));
12299 }
12300 let outcome = repair_pool_meta_transaction(self.pools.clone(), &mut candidate, selection, &fence).await?;
12301 let mut pool_meta = self.pool_meta.write().await;
12302 fence.ensure_held()?;
12303 self.ctx.advance_data_movement_operation_epoch();
12304 *pool_meta = outcome.committed.clone();
12305 fence.ensure_held()?;
12306 outcome.disarm();
12307 candidate.aborted_transaction = Arc::clone(&state.aborted_transaction);
12309 candidate.transaction_failure = Arc::clone(&state.transaction_failure);
12310 candidate.active_transactions = Arc::clone(&state.active_transactions);
12311 candidate.block_started_at = Arc::clone(&state.block_started_at);
12312 *state
12313 .block_started_at
12314 .lock()
12315 .unwrap_or_else(std::sync::PoisonError::into_inner) = None;
12316 *state
12317 .transaction_failure
12318 .lock()
12319 .unwrap_or_else(std::sync::PoisonError::into_inner) = None;
12320 state.aborted_transaction.store(false, Ordering::SeqCst);
12321 *state = candidate;
12322 info!(
12323 event = EVENT_DECOMMISSION_STATE,
12324 component = LOG_COMPONENT_ECSTORE,
12325 subsystem = LOG_SUBSYSTEM_POOLS,
12326 state = "pool_metadata_recovered",
12327 phase = blocked.phase,
12328 blocked_since = %blocked.since,
12329 "Pool metadata writes recovered"
12330 );
12331 metrics::counter!("rustfs_pool_metadata_recoveries_total").increment(1);
12332 Ok(true)
12333 }
12334
12335 pub(crate) fn record_pool_meta_recovery_failure(&self, error: Error) {
12336 let Ok(mut state) = self.pool_meta_save_gate.try_lock() else {
12337 return;
12338 };
12339 if state.ensure_write_safe("pool metadata recovery").is_ok() {
12340 return;
12341 }
12342 let classify = |error: &Error| match error.pool_metadata_failure() {
12343 Some(context) => (context.kind.as_str(), context.phase),
12344 None => match error {
12345 Error::OperationCanceled => ("workers_draining", "recovery_quiesce"),
12346 Error::Timeout => ("timeout", "recovery"),
12347 Error::Lock(_) => ("fence_unavailable", "recovery"),
12348 _ => ("unavailable", "recovery"),
12349 },
12350 };
12351 let (reason, phase) = classify(&error);
12352 if state
12353 .recovery_failure
12354 .as_ref()
12355 .is_none_or(|previous| classify(previous) != (reason, phase))
12356 {
12357 warn!(
12358 event = EVENT_DECOMMISSION_STATE,
12359 component = LOG_COMPONENT_ECSTORE,
12360 subsystem = LOG_SUBSYSTEM_POOLS,
12361 state = "pool_metadata_recovery_pending",
12362 reason,
12363 phase,
12364 "Pool metadata recovery remains blocked"
12365 );
12366 }
12367 state.recovery_failure = Some(Arc::new(error));
12368 }
12369
12370 async fn load_runtime_pool_meta_observing(&self, write_state: &mut PoolMetaWriteState, operation: &str) -> Result<PoolMeta> {
12371 write_state.ensure_write_safe(operation)?;
12372 load_pool_meta_identity_observing(self.pools.clone(), write_state).await?;
12373 let mut pool_meta = PoolMeta::default();
12374 let replica_state = pool_meta
12375 .load_no_lock_from_replicas_observing(self.pools.clone(), write_state)
12376 .await?;
12377 write_state.observe_replicas(replica_state);
12378 write_state.ensure_missing_metadata_can_initialize()?;
12379 write_state.ensure_write_safe(operation)?;
12380 Ok(pool_meta)
12381 }
12382
12383 pub(crate) async fn load_runtime_pool_meta_under_activation_fence(
12384 &self,
12385 write_state: &mut PoolMetaWriteState,
12386 activation_fence: &PoolRebalanceActivationFence,
12387 operation: &str,
12388 ) -> Result<PoolMeta> {
12389 activation_fence.ensure_held()?;
12390 let pool_meta = self.load_runtime_pool_meta_observing(write_state, operation).await?;
12391 activation_fence.ensure_held()?;
12392 Ok(pool_meta)
12393 }
12394
12395 pub(crate) async fn load_runtime_pool_meta(&self, operation: &str) -> Result<PoolMeta> {
12396 let mut write_state = self.pool_meta_save_gate.lock().await;
12397 write_state.ensure_write_safe(operation)?;
12398 let pool = self
12399 .pools
12400 .first()
12401 .cloned()
12402 .ok_or_else(|| Error::other(format!("{operation}: no storage pools available")))?;
12403 let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
12404 let _pool_meta_guard = pool_meta_lock.get_read_lock(get_lock_acquire_timeout()).await?;
12405 self.load_runtime_pool_meta_observing(&mut write_state, operation).await
12406 }
12407
12408 async fn run_guarded_decommission_side_effect<T, E, F, Fut>(
12409 &self,
12410 rx: &CancellationToken,
12411 operation_gate: &Arc<tokio::sync::RwLock<()>>,
12412 operation: F,
12413 ) -> std::result::Result<T, E>
12414 where
12415 F: FnOnce() -> Fut,
12416 Fut: std::future::Future<Output = std::result::Result<T, E>>,
12417 E: From<Error>,
12418 {
12419 let _operation_guard = tokio::select! {
12420 biased;
12421 _ = rx.cancelled() => return Err(Error::OperationCanceled.into()),
12422 guard = operation_gate.read() => guard,
12423 };
12424
12425 if rx.is_cancelled() {
12426 return Err(Error::OperationCanceled.into());
12427 }
12428 self.ensure_pool_meta_side_effects_safe("decommission side effect blocked because pool metadata requires recovery")
12429 .await
12430 .map_err(E::from)?;
12431 if rx.is_cancelled() {
12432 return Err(Error::OperationCanceled.into());
12433 }
12434
12435 let result = operation().await;
12436 if rx.is_cancelled() {
12437 return Err(Error::OperationCanceled.into());
12438 }
12439 result
12440 }
12441
12442 async fn save_current_pool_meta(&self, indices: &[usize]) -> Result<()> {
12443 let mut save_guard = self.pool_meta_save_gate.lock().await;
12444 let (pool_meta_guard, mut snapshot) = self
12445 .acquire_pool_meta_write_guard(&mut save_guard, "pool metadata save failed")
12446 .await?;
12447 {
12448 let pool_meta = self.pool_meta.read().await;
12449 merge_pool_meta_updates_for_save(&mut snapshot, &pool_meta, indices, "pool metadata save failed")?;
12450 }
12451 let outcome = snapshot
12452 .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), indices)
12453 .await?;
12454 let mut pool_meta = self.pool_meta.write().await;
12455 ensure_pool_meta_write_fence(&pool_meta_guard, "pool metadata save failed")?;
12456 publish_pool_meta_updates(&mut pool_meta, &outcome.committed, indices);
12457 ensure_pool_meta_write_fence(&pool_meta_guard, "pool metadata save failed")?;
12458 drop(pool_meta);
12459 outcome.disarm();
12460 Ok(())
12461 }
12462
12463 #[cfg(test)]
12464 pub(crate) async fn save_current_pool_meta_for_test(&self, indices: &[usize]) -> Result<()> {
12465 self.save_current_pool_meta(indices).await
12466 }
12467
12468 async fn persist_decommission_unresolved_entry(
12469 &self,
12470 idx: usize,
12471 generation: OffsetDateTime,
12472 entry: DecommissionUnresolvedEntry,
12473 ) -> Result<()> {
12474 {
12475 let rebalance_meta = self.rebalance_meta.read().await.clone();
12476 let mut pool_meta = self.pool_meta.write().await;
12477 ensure_decommission_ledger_persistence_supported(&pool_meta)?;
12478 record_decommission_unresolved_entry(
12479 &mut pool_meta,
12480 idx,
12481 generation,
12482 entry,
12483 OffsetDateTime::now_utc(),
12484 rebalance_meta.as_ref(),
12485 )?;
12486 }
12487 self.save_current_pool_meta(&[idx])
12488 .await
12489 .map_err(|err| Error::other(format!("decommission unresolved entry ledger save failed: {err}")))
12490 }
12491
12492 async fn save_decommission_progress_checkpoint(&self, idx: usize, generation: OffsetDateTime) -> Result<bool> {
12493 self.save_decommission_progress_checkpoint_at(idx, generation, OffsetDateTime::now_utc())
12494 .await
12495 }
12496
12497 async fn save_decommission_progress_checkpoint_at(
12498 &self,
12499 idx: usize,
12500 generation: OffsetDateTime,
12501 now: OffsetDateTime,
12502 ) -> Result<bool> {
12503 let mut save_guard = self.pool_meta_save_gate.lock().await;
12507 let (pool_meta_guard, mut snapshot) = self
12508 .acquire_pool_meta_write_guard(&mut save_guard, "decommission progress save failed")
12509 .await?;
12510 let (snapshot, checkpoint) = {
12511 let rebalance_meta = self.rebalance_meta.read().await.clone();
12512 let pool_meta = self.pool_meta.read().await;
12513 ensure_decommission_generation(&pool_meta, idx, generation)?;
12514 let Some(checkpoint) = pool_meta.decommission_progress_checkpoint(
12515 idx,
12516 DECOMMISSION_PROGRESS_SAVE_INTERVAL,
12517 now,
12518 rebalance_meta.as_ref(),
12519 )?
12520 else {
12521 return Ok(false);
12522 };
12523
12524 let mut current = pool_meta.clone();
12525 let current_count = current.pools.len();
12526 let Some(pool) = current.pools.get_mut(idx) else {
12527 return Err(invalid_decommission_pool_index_error(current_count, idx));
12528 };
12529 pool.last_update = checkpoint.checkpoint_at;
12530 if let (Some(operation_id), Some(owner_nonce), Some(expires_at), Some(reservation)) = (
12531 checkpoint.capacity_operation_id,
12532 checkpoint.capacity_owner_nonce,
12533 checkpoint.capacity_lease_expires_at,
12534 pool.decommission.as_mut().and_then(|info| info.capacity_reservation.as_mut()),
12535 ) && reservation.operation_id == operation_id
12536 && reservation.owner_nonce == owner_nonce
12537 && reservation.active()
12538 {
12539 reservation.renewed_at = checkpoint.checkpoint_at;
12540 reservation.expires_at = expires_at;
12541 }
12542 merge_pool_meta_updates_for_save(&mut snapshot, ¤t, &[idx], "decommission progress save failed")?;
12543 (snapshot, checkpoint)
12544 };
12545
12546 let outcome = match snapshot
12547 .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx])
12548 .await
12549 {
12550 Ok(outcome) => outcome,
12551 Err(err) => {
12552 let retry_after = OffsetDateTime::now_utc() + DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
12553 let mut pool_meta = self.pool_meta.write().await;
12554 pool_meta.defer_decommission_progress_checkpoint(idx, checkpoint, retry_after);
12555 return Err(err);
12556 }
12557 };
12558
12559 let mut pool_meta = self.pool_meta.write().await;
12560 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission progress save failed")?;
12561 pool_meta.version = pool_meta.version.max(outcome.committed.version);
12562 let committed = pool_meta.commit_decommission_progress_checkpoint(idx, checkpoint);
12563 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission progress save failed")?;
12564 drop(pool_meta);
12565 outcome.disarm();
12566 Ok(committed)
12567 }
12568
12569 async fn mark_decommission_bucket_done_and_save(
12570 &self,
12571 idx: usize,
12572 generation: OffsetDateTime,
12573 bucket: &DecomBucketInfo,
12574 ) -> Result<bool> {
12575 let mut save_guard = self.pool_meta_save_gate.lock().await;
12578 let (pool_meta_guard, mut snapshot) = self
12579 .acquire_pool_meta_write_guard(&mut save_guard, "decommission bucket completion save failed")
12580 .await?;
12581 ensure_decommission_generation(&snapshot, idx, generation)?;
12582 let rebalance_meta = self.rebalance_meta.read().await.clone();
12583 let checkpoint = {
12584 let pool_meta = self.pool_meta.read().await;
12585 ensure_decommission_generation(&pool_meta, idx, generation)?;
12586 let mut current = pool_meta.clone();
12587 publish_decommission_capacity_update(&mut current, &snapshot, idx)?;
12588 if !mark_decommission_bucket_done(&mut current, idx, bucket)? {
12589 return Ok(false);
12590 }
12591 let checkpoint_at = current.next_scanner_data_movement_update(OffsetDateTime::now_utc(), rebalance_meta.as_ref());
12592 current.pools[idx].last_update = checkpoint_at;
12593 let info = current.pools[idx]
12594 .decommission
12595 .as_ref()
12596 .ok_or_else(|| decommission_metadata_not_initialized_error("prepare decommission bucket completion"))?;
12597 let checkpoint = DecommissionProgressCheckpoint {
12598 start_time: Some(generation),
12599 queued: false,
12600 counted_items: info.counted_items(),
12601 checkpoint_at,
12602 capacity_operation_id: None,
12603 capacity_owner_nonce: None,
12604 capacity_lease_expires_at: None,
12605 };
12606 merge_pool_meta_updates_for_save(&mut snapshot, ¤t, &[idx], "decommission bucket completion save failed")?;
12607 checkpoint
12608 };
12609
12610 #[cfg(test)]
12611 decommission_test_wrap_result("bucket_completion_before_save", &bucket.name, &bucket.prefix, 0, Ok(()))?;
12612 let outcome = snapshot
12613 .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx])
12614 .await?;
12615 #[cfg(test)]
12616 decommission_test_wrap_result("bucket_completion_after_save", &bucket.name, &bucket.prefix, 0, Ok(()))?;
12617 let mut pool_meta = self.pool_meta.write().await;
12618 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission bucket completion save failed")?;
12619 ensure_decommission_generation(&pool_meta, idx, generation)?;
12620 publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, idx)?;
12621 mark_decommission_bucket_done(&mut pool_meta, idx, bucket)?;
12622 pool_meta.commit_decommission_progress_checkpoint(idx, checkpoint);
12623 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission bucket completion save failed")?;
12624 drop(pool_meta);
12625 outcome.disarm();
12626 Ok(true)
12627 }
12628
12629 pub(crate) async fn save_current_pool_meta_for_decommission_start(
12630 &self,
12631 indices: &[usize],
12632 decom_buckets: Vec<DecomBucketInfo>,
12633 ) -> Result<PoolMeta> {
12634 let mut save_guard = self.pool_meta_save_gate.lock().await;
12635 save_guard.ensure_write_safe("decommission start failed")?;
12636 let rebalance_pool = self
12637 .pools
12638 .first()
12639 .cloned()
12640 .ok_or_else(|| Error::other("decommission start rebalance metadata load failed: no storage pools available"))?;
12641 #[cfg(test)]
12642 observe_pool_activation_start_attempt(PoolActivationStartKind::Decommission);
12643 let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
12644 let mut activation_fence = acquire_pool_rebalance_activation_locks(rebalance_pool.clone(), fleet_proof).await?;
12645
12646 let mut rebalance_meta = RebalanceMeta::new();
12647 match rebalance_meta
12648 .load_with_opts(
12649 rebalance_pool.clone(),
12650 ObjectOptions {
12651 no_lock: true,
12652 ..Default::default()
12653 },
12654 )
12655 .await
12656 {
12657 Ok(()) => ensure_decommission_start_rebalance_meta_allowed(Some(&rebalance_meta))?,
12658 Err(Error::ConfigNotFound) => {}
12659 Err(err) => {
12660 return Err(Error::other(format!(
12661 "rebalance metadata load before decommission start save failed: {err}"
12662 )));
12663 }
12664 }
12665
12666 let current_pool_meta = {
12667 let pool_meta = self.pool_meta.read().await;
12668 pool_meta.clone()
12669 };
12670 let mut latest_pool_meta = PoolMeta::default();
12671 let replica_state = latest_pool_meta
12672 .load_no_lock_from_replicas_observing(self.pools.clone(), &mut save_guard)
12673 .await?;
12674 save_guard.observe_replicas(replica_state);
12675 save_guard.ensure_write_safe("decommission start failed")?;
12676 if latest_pool_meta.pools.is_empty() {
12677 latest_pool_meta = current_pool_meta;
12678 }
12679
12680 ensure_decommission_start_pool_states(&latest_pool_meta, indices)?;
12681 ensure_decommission_ledger_persistence_supported(&latest_pool_meta)?;
12682 ensure_decommission_capacity_writer_supported(&latest_pool_meta)?;
12683 let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
12684 activation_fence.ensure_held()?;
12685 let target_fence_proof = crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof();
12686
12687 let previous_pool_meta = latest_pool_meta.clone();
12688 let capacity_generation = next_decommission_capacity_generation(&latest_pool_meta)?;
12689 let first_idx = indices.first().copied();
12690 let now = OffsetDateTime::now_utc();
12691 for idx in indices.iter().copied() {
12692 let capacity = capacity_infos
12693 .iter()
12694 .find(|capacity| capacity.pool_index == idx)
12695 .ok_or_else(|| Error::DecommissionCapacity(format!("decommission capacity snapshot is missing pool {idx}")))?;
12696 latest_pool_meta.set_decommission_state_at(
12697 idx,
12698 capacity.space,
12699 Some(idx) != first_idx,
12700 now,
12701 Some(&rebalance_meta),
12702 )?;
12703 latest_pool_meta.queue_buckets(idx, decom_buckets.clone());
12704 }
12705 let model_version = select_decommission_capacity_model(&latest_pool_meta, target_fence_proof.is_some())?;
12706 if model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION {
12707 activation_fence.set_decommission_target_fence_proof(target_fence_proof);
12708 }
12709 reserve_decommission_start_target_capacity(
12710 &mut latest_pool_meta,
12711 indices,
12712 &capacity_infos,
12713 uuid::Uuid::new_v4(),
12714 capacity_generation,
12715 now,
12716 model_version,
12717 )?;
12718
12719 activation_fence.ensure_held()?;
12720 let outcome = latest_pool_meta
12721 .save_no_lock_with_activation_fence(self.pools.clone(), &mut save_guard, &activation_fence, indices)
12722 .await?;
12723 activation_fence.ensure_held()?;
12724 {
12725 let mut pool_meta = self.pool_meta.write().await;
12726 publish_pool_meta_updates(&mut pool_meta, &outcome.committed, indices);
12727 }
12728 activation_fence.ensure_held()?;
12729 outcome.disarm();
12730
12731 Ok(previous_pool_meta)
12732 }
12733
12734 async fn rollback_decommission_start_after_reload_failure(
12735 &self,
12736 movement_gate: &Arc<tokio::sync::RwLock<()>>,
12737 previous_pool_meta: &PoolMeta,
12738 indices: &[usize],
12739 ) -> Result<()> {
12740 let _movement_guard = movement_gate.write().await;
12741 let mut save_guard = self.pool_meta_save_gate.lock().await;
12742 let (pool_meta_guard, mut snapshot) = self
12743 .acquire_pool_meta_write_guard(&mut save_guard, "decommission start rollback failed")
12744 .await?;
12745 rollback_start_decommission_pool_meta(&mut snapshot, previous_pool_meta, indices);
12746 let outcome = snapshot
12747 .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), indices)
12748 .await?;
12749 let mut pool_meta = self.pool_meta.write().await;
12750 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission start rollback failed")?;
12751 publish_pool_meta_updates(&mut pool_meta, &outcome.committed, indices);
12752 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission start rollback failed")?;
12753 drop(pool_meta);
12754 outcome.disarm();
12755 self.ctx.advance_data_movement_operation_epoch();
12756 Ok(())
12757 }
12758
12759 async fn ensure_decommission_rebalance_idle_after_refresh(&self) -> Result<()> {
12760 self.load_rebalance_meta().await?;
12761 ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)
12762 }
12763
12764 async fn ensure_decommission_rebalance_idle_after_refresh_under_start_gate(&self) -> Result<()> {
12765 self.load_rebalance_meta_under_start_gate().await?;
12766 ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)
12767 }
12768
12769 pub async fn status(&self, idx: usize) -> Result<PoolStatus> {
12770 let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
12771 let space_info = capacity_infos
12772 .iter()
12773 .find(|capacity| capacity.pool_index == idx)
12774 .map(|capacity| capacity.space)
12775 .ok_or_else(|| invalid_decommission_pool_index_error(self.pools.len(), idx))?;
12776
12777 let pool_meta = self.pool_meta.read().await;
12778 let active_target_reservations = active_decommission_target_reservations(&pool_meta);
12779
12780 let mut pool_info = get_by_index(pool_meta.pools.as_slice(), idx, "fetch decommission status")?.clone();
12781 if let Some(info) = pool_info.decommission.as_mut() {
12782 observe_decommission_capacity_reservation(info, &capacity_infos, &active_target_reservations);
12783 }
12784 Ok(apply_decommission_status_space_info(pool_info, space_info))
12785 }
12786
12787 #[tracing::instrument(skip_all)]
12788 pub async fn refresh_pool_status_meta(&self) -> Result<()> {
12789 let movement_gate = self.ctx.data_movement_operation_gate();
12790 let _movement_guard = movement_gate.write().await;
12791 let persisted = self.load_runtime_pool_meta("refresh pool status metadata failed").await?;
12792
12793 let active_workers = {
12794 let cancelers = self.decommission_cancelers.read().await;
12795 cancelers
12796 .iter()
12797 .map(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
12798 .collect::<Vec<_>>()
12799 };
12800
12801 let mut pool_meta = self.pool_meta.write().await;
12802 if merge_pool_status_refresh(&mut pool_meta, persisted, &active_workers) {
12803 self.ctx.advance_data_movement_operation_epoch();
12804 }
12805 Ok(())
12806 }
12807
12808 async fn get_decommission_pool_capacity_info(
12809 &self,
12810 idx: usize,
12811 use_owned_usage: bool,
12812 ) -> Result<DecommissionPoolCapacityInfo> {
12813 if let Some(sets) = self.pools.get(idx) {
12814 let mut info = sets.storage_info_snapshot().await;
12815 info.backend = StorageAdminApi::backend_info(self).await;
12816
12817 let total = get_total_usable_capacity(&info.disks, &info);
12818 let free = get_total_usable_capacity_free(&info.disks, &info);
12819 let space = PoolSpaceInfo {
12820 free,
12821 total,
12822 used: total.saturating_sub(free),
12823 };
12824 let layout = DecommissionErasureLayout {
12825 data: info
12826 .backend
12827 .standard_sc_data
12828 .get(idx)
12829 .copied()
12830 .unwrap_or_else(|| sets.set_drive_count.saturating_sub(sets.parity_count)),
12831 parity: info
12832 .backend
12833 .standard_sc_parities
12834 .get(idx)
12835 .copied()
12836 .unwrap_or(sets.parity_count),
12837 };
12838 if !layout.is_valid() {
12839 return Err(Error::DecommissionCapacity(format!(
12840 "failed to read decommission capacity for pool {idx}: invalid erasure layout data={} parity={}",
12841 layout.data, layout.parity
12842 )));
12843 }
12844 let owned_physical_used = if use_owned_usage {
12845 Some(decommission_owned_physical_usage(sets, layout).await)
12846 } else {
12847 None
12848 };
12849 let (physical_total, physical_free, physical_used) =
12850 decommission_physical_pool_capacity(&info.disks, idx, layout, space, owned_physical_used.as_ref());
12851
12852 Ok(DecommissionPoolCapacityInfo {
12853 pool_index: idx,
12854 space,
12855 layout,
12856 physical_free,
12857 physical_total,
12858 physical_used,
12859 })
12860 } else {
12861 Err(invalid_decommission_pool_index_error(self.pools.len(), idx))
12862 }
12863 }
12864
12865 async fn get_decommission_all_pool_capacity_infos(&self) -> Result<Vec<DecommissionPoolCapacityInfo>> {
12866 #[cfg(any(test, feature = "test-util"))]
12867 if let Some(capacity_infos) = take_decommission_capacity_info_override_for_test(self.id) {
12868 return Ok(capacity_infos);
12869 }
12870
12871 let mut capacity_infos = Vec::with_capacity(self.pools.len());
12872 for idx in 0..self.pools.len() {
12873 capacity_infos.push(self.get_decommission_pool_capacity_info(idx, false).await?);
12874 }
12875 Ok(capacity_infos)
12876 }
12877
12878 async fn get_decommission_all_pool_capacity_infos_with_owned_usage(&self) -> Result<Vec<DecommissionPoolCapacityInfo>> {
12879 #[cfg(any(test, feature = "test-util"))]
12880 if let Some(capacity_infos) = take_decommission_capacity_info_override_for_test(self.id) {
12881 return Ok(capacity_infos);
12882 }
12883
12884 let mut capacity_infos = Vec::with_capacity(self.pools.len());
12885 for idx in 0..self.pools.len() {
12886 capacity_infos.push(self.get_decommission_pool_capacity_info(idx, true).await?);
12887 }
12888 Ok(capacity_infos)
12889 }
12890
12891 async fn ensure_decommission_runtime_capacity_available(&self, idx: usize, generation: OffsetDateTime) -> Result<()> {
12892 let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
12893 let pool_meta = self.pool_meta.read().await;
12894 ensure_decommission_generation(&pool_meta, idx, generation)?;
12895 #[cfg(test)]
12896 if pool_meta
12897 .pools
12898 .get(idx)
12899 .and_then(|pool| pool.decommission.as_ref())
12900 .is_some_and(|info| info.capacity_reservation.is_none())
12901 {
12902 return Ok(());
12903 }
12904 ensure_decommission_capacity_reservations_available(&pool_meta, &capacity_infos, "migration")
12905 }
12906
12907 async fn decommission_capacity_owner_for_worker(
12908 &self,
12909 idx: usize,
12910 generation: OffsetDateTime,
12911 ) -> Result<Option<DecommissionCapacityOwner>> {
12912 let pool_meta = self.pool_meta.read().await;
12913 ensure_decommission_generation(&pool_meta, idx, generation)?;
12914 let Some(reservation) = pool_meta
12915 .pools
12916 .get(idx)
12917 .and_then(|pool| pool.decommission.as_ref())
12918 .and_then(|info| info.capacity_reservation.as_ref())
12919 else {
12920 return Ok(None);
12921 };
12922 if !reservation.lease_active_at(OffsetDateTime::now_utc()) {
12923 return Err(decommission_capacity_blocked_error(
12924 "decommission capacity reservation lease is not active",
12925 ));
12926 }
12927 Ok(Some(DecommissionCapacityOwner {
12928 source_pool_index: idx,
12929 operation_id: reservation.operation_id,
12930 generation: reservation.generation,
12931 owner_nonce: reservation.owner_nonce,
12932 mutation_id: None,
12933 }))
12934 }
12935
12936 pub(crate) async fn is_decommission_capacity_target_reserved(
12937 &self,
12938 owner: DecommissionCapacityOwner,
12939 target_pool_index: usize,
12940 ) -> Result<bool> {
12941 let pool_meta = self.pool_meta.read().await;
12942 let reservation = pool_meta
12943 .pools
12944 .get(owner.source_pool_index)
12945 .and_then(|pool| pool.decommission.as_ref())
12946 .and_then(|info| info.capacity_reservation.as_ref())
12947 .filter(|reservation| reservation.admits_owner(owner, OffsetDateTime::now_utc()))
12948 .ok_or_else(|| decommission_capacity_blocked_error("decommission target selection reservation is stale"))?;
12949 Ok(reservation
12950 .targets
12951 .iter()
12952 .any(|target| target.pool_index == target_pool_index))
12953 }
12954
12955 pub(crate) async fn select_decommission_capacity_target_pool(
12956 &self,
12957 owner: DecommissionCapacityOwner,
12958 expected_data_bytes: usize,
12959 ) -> Result<usize> {
12960 let pool_meta = self.pool_meta.read().await;
12961 let reservation = pool_meta
12962 .pools
12963 .get(owner.source_pool_index)
12964 .and_then(|pool| pool.decommission.as_ref())
12965 .and_then(|info| info.capacity_reservation.as_ref())
12966 .filter(|reservation| reservation.admits_owner(owner, OffsetDateTime::now_utc()))
12967 .ok_or_else(|| decommission_capacity_blocked_error("decommission target selection reservation is stale"))?;
12968 let candidate = |target: &DecommissionCapacityTarget| {
12969 let expected_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target.layout).ok()?;
12970 let required_peak = expected_physical_bytes.saturating_mul(1usize.saturating_add(reservation.temporary_copies));
12971 let remaining = target.remaining_reserved_physical_bytes(reservation.temporary_copies);
12972 (required_peak <= remaining).then_some((target.pool_index, remaining))
12973 };
12974 if let Some(permitted_target_pool_index) = decommission_capacity_target_permit_index(self.id, owner) {
12975 if let Some((pool_index, _)) = reservation
12976 .targets
12977 .iter()
12978 .find(|target| target.pool_index == permitted_target_pool_index)
12979 .and_then(candidate)
12980 {
12981 return Ok(pool_index);
12982 }
12983 discard_decommission_capacity_target_permit_except(self.id, owner, None);
12987 }
12988 if let Some((pool_index, _)) = reservation
12989 .targets
12990 .iter()
12991 .filter_map(candidate)
12992 .max_by_key(|(_, remaining)| *remaining)
12993 {
12994 return Ok(pool_index);
12995 }
12996 if reservation.source_data_equivalent_bytes > 0 {
12997 return Err(decommission_capacity_blocked_error(format!(
12998 "source pool {} has no target allocation for {expected_data_bytes} data bytes",
12999 owner.source_pool_index
13000 )));
13001 }
13002 let capacities = self.get_decommission_all_pool_capacity_infos().await?;
13003 let active_sources = active_decommission_source_indices(&pool_meta);
13004 capacities
13005 .into_iter()
13006 .filter(|capacity| {
13007 capacity.pool_index != owner.source_pool_index
13008 && !active_sources.contains(&capacity.pool_index)
13009 && pool_meta
13010 .pools
13011 .get(capacity.pool_index)
13012 .is_some_and(is_decommission_start_active_pool)
13013 })
13014 .max_by_key(|capacity| capacity.physical_free)
13015 .map(|capacity| capacity.pool_index)
13016 .ok_or_else(|| {
13017 decommission_capacity_blocked_error(format!(
13018 "source pool {} has no target pool for {expected_data_bytes} data bytes",
13019 owner.source_pool_index
13020 ))
13021 })
13022 }
13023
13024 pub(crate) async fn next_scanner_data_movement_update(&self, now: OffsetDateTime) -> OffsetDateTime {
13025 let pool_meta = self.pool_meta.read().await;
13026 let rebalance_meta = self.rebalance_meta.read().await;
13027 pool_meta.next_scanner_data_movement_update(now, rebalance_meta.as_ref())
13028 }
13029
13030 #[tracing::instrument(skip(self))]
13031 pub async fn decommission_cancel(self: &Arc<Self>, idx: usize) -> Result<()> {
13032 self.decommission_cancel_with_owner(idx, None).await
13033 }
13034
13035 async fn decommission_cancel_for_operation(self: &Arc<Self>, idx: usize, owner: &DecommissionCanceler) -> Result<()> {
13036 self.decommission_cancel_with_owner(idx, Some(owner)).await
13037 }
13038
13039 #[cfg(test)]
13040 async fn decommission_cancel_with_owner_and_save<Save, SaveFuture>(
13041 self: &Arc<Self>,
13042 idx: usize,
13043 owner: Option<&DecommissionCanceler>,
13044 save_pool_meta: Save,
13045 ) -> Result<()>
13046 where
13047 Save: FnOnce(PoolMeta, Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> SaveFuture + Send + 'static,
13048 SaveFuture: Future<Output = Result<()>> + Send + 'static,
13049 {
13050 let store = self.clone();
13051 let owner = owner.cloned();
13052 tokio::spawn(async move { store.decommission_cancel_transaction(idx, owner, false, save_pool_meta).await })
13055 .await
13056 .map_err(|err| Error::other(format!("decommission cancel transaction task join error: {err}")))?
13057 }
13058
13059 async fn decommission_cancel_transaction<Save, SaveFuture>(
13060 &self,
13061 idx: usize,
13062 owner: Option<DecommissionCanceler>,
13063 acquire_runtime_fence: bool,
13064 save_pool_meta: Save,
13065 ) -> Result<()>
13066 where
13067 Save: FnOnce(PoolMeta, Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> SaveFuture,
13068 SaveFuture: Future<Output = Result<()>>,
13069 {
13070 let owner = owner.as_ref();
13071 ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?;
13072 #[cfg(test)]
13073 if acquire_runtime_fence {
13074 pause_decommission_cancel_before_start_gate(self.id).await;
13075 }
13076 let _start_guard = self.start_gate.lock().await;
13077 let terminal_fence_plan = if acquire_runtime_fence {
13081 let read_save_guard = self.pool_meta_save_gate.lock().await;
13082 let (read_guard, snapshot) = self
13083 .acquire_pool_meta_read_guard(&read_save_guard, "decommission cancel fence planning failed")
13084 .await?;
13085 let plan = decommission_capacity_terminal_fence_plan(&snapshot, idx)?;
13086 drop(read_guard);
13087 drop(read_save_guard);
13088 plan
13089 } else {
13090 None
13091 };
13092 let _capacity_target_guards = if acquire_runtime_fence {
13093 self.acquire_decommission_capacity_terminal_guards(idx, terminal_fence_plan.as_ref())
13094 .await?
13095 } else {
13096 Vec::new()
13097 };
13098 let mut save_guard = self.pool_meta_save_gate.lock().await;
13099 let (_pool_meta_guard, mut persisted_pool_meta) = if acquire_runtime_fence {
13100 let (guard, pool_meta) = self
13101 .acquire_pool_meta_write_guard(&mut save_guard, "decommission cancel failed")
13102 .await?;
13103 (Some(guard), Some(pool_meta))
13104 } else {
13105 save_guard.ensure_write_safe("decommission cancel failed")?;
13106 (None, None)
13107 };
13108 if let Some(persisted_pool_meta) = persisted_pool_meta.as_ref() {
13109 let committed_plan = decommission_capacity_terminal_fence_plan(persisted_pool_meta, idx)?;
13110 if committed_plan != terminal_fence_plan {
13111 return Err(decommission_capacity_blocked_error(
13112 "decommission capacity owner or target cohort changed while acquiring terminal fences",
13113 ));
13114 }
13115 }
13116 let pool_meta_fence = _pool_meta_guard
13117 .as_ref()
13118 .and_then(rustfs_lock::NamespaceLockGuard::lock_lost_signal);
13119
13120 let rebalance_meta = self.rebalance_meta.read().await.clone();
13125 let terminal_at = OffsetDateTime::now_utc();
13126 let mut cancelers = self.decommission_cancelers.write().await;
13127 let mut pool_meta = self.pool_meta.write().await;
13128 if acquire_runtime_fence {
13129 let local_plan = decommission_capacity_terminal_fence_plan(&pool_meta, idx)?;
13130 if local_plan != terminal_fence_plan {
13131 return Err(decommission_capacity_blocked_error(
13132 "local decommission capacity owner differs from the durable terminal fence plan",
13133 ));
13134 }
13135 }
13136 let (pending, should_reload_pool_meta, already_canceled, terminal_canceler, durable_movement_generation) = {
13137 let mut already_canceled = false;
13138 let (pool_present, decommission_present, terminal) = if let Some(pool) = pool_meta.pools.get(idx) {
13139 if let Some(info) = pool.decommission.as_ref() {
13140 already_canceled = info.canceled;
13141 (
13142 true,
13143 info.has_decommission_state(),
13144 should_reject_decommission_cancel_as_terminal(info.complete, info.failed),
13145 )
13146 } else {
13147 (true, false, false)
13148 }
13149 } else {
13150 (false, false, false)
13151 };
13152
13153 ensure_decommission_cancel_allowed(pool_present, decommission_present, terminal)?;
13154 let previous_pool = pool_meta
13155 .pools
13156 .get(idx)
13157 .cloned()
13158 .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), idx))?;
13159 let previous_decommission = previous_pool
13160 .decommission
13161 .as_ref()
13162 .ok_or_else(|| decommission_metadata_not_initialized_error("cancel decommission"))?;
13163 let mut snapshot = pool_meta.clone();
13164 let Some(changed) = update_decommission_for_operation(cancelers.as_slice(), &mut snapshot, idx, owner, |pool_meta| {
13165 pool_meta.decommission_cancel_at(idx, terminal_at, rebalance_meta.as_ref())
13166 }) else {
13167 return Ok(());
13168 };
13169 let pending = if changed {
13170 let canceled_pool = snapshot
13171 .pools
13172 .get(idx)
13173 .cloned()
13174 .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), idx))?;
13175 if let Some(persisted) = persisted_pool_meta.as_mut() {
13176 merge_pool_meta_updates_for_save(persisted, &snapshot, &[idx], "decommission cancel failed")?;
13177 snapshot = persisted.clone();
13178 }
13179 Some((
13180 snapshot,
13181 DecommissionCancelCommit {
13182 previous_start_time: previous_decommission.start_time,
13183 previous_queued: previous_decommission.queued,
13184 previous_last_update: previous_pool.last_update,
13185 canceled_pool,
13186 },
13187 ))
13188 } else {
13189 None
13190 };
13191 let terminal_canceler = if let Some(owner) = owner {
13192 Some(owner.clone())
13193 } else {
13194 cancelers.get(idx).and_then(Option::as_ref).cloned()
13195 };
13196 let durable_movement_generation = pending
13197 .as_ref()
13198 .map(|(_, commit)| crate::store::scanner_data_movement_timestamp_generation(commit.canceled_pool.last_update))
13199 .unwrap_or_default();
13200 (
13201 pending,
13202 should_retry_decommission_cancel_reload(changed, already_canceled),
13203 already_canceled,
13204 terminal_canceler,
13205 durable_movement_generation,
13206 )
13207 };
13208 let active_worker = terminal_canceler.as_ref().is_some_and(DecommissionCanceler::is_active);
13209 if !active_worker && !already_canceled {
13210 warn!(
13211 event = EVENT_DECOMMISSION_STATE,
13212 component = LOG_COMPONENT_ECSTORE,
13213 subsystem = LOG_SUBSYSTEM_POOLS,
13214 pool_index = idx,
13215 state = "cancel_skipped",
13216 reason = "no_active_canceler",
13217 "Decommission cancel skipped"
13218 );
13219 }
13220
13221 let changed = pending.is_some();
13222 let commit_result = if let Some((snapshot, commit)) = pending {
13223 if let Err(err) = save_pool_meta(snapshot, pool_meta_fence).await {
13224 save_guard.block_writes();
13225 return Err(err);
13226 }
13227 if let Some(pool_meta_guard) = _pool_meta_guard.as_ref()
13228 && let Err(err) = ensure_pool_meta_write_fence(pool_meta_guard, "decommission cancel failed")
13229 {
13230 save_guard.block_writes();
13231 return Err(err);
13232 }
13233 commit_decommission_cancel(&mut pool_meta, idx, commit)
13234 } else {
13235 Ok(())
13236 };
13237
13238 commit_result?;
13239 if let Some(pool_meta_guard) = _pool_meta_guard.as_ref()
13240 && let Err(err) = ensure_pool_meta_write_fence(pool_meta_guard, "decommission cancel failed")
13241 {
13242 save_guard.block_writes();
13243 return Err(err);
13244 }
13245 if let Some(canceler) = terminal_canceler.as_ref() {
13246 take_and_cancel_decommission_canceler_for_operation(cancelers.as_mut_slice(), idx, canceler);
13247 }
13248 drop(pool_meta);
13249 drop(cancelers);
13250 drop(_pool_meta_guard);
13251 drop(save_guard);
13252
13253 if changed {
13254 let movement_gate = self.ctx.data_movement_operation_gate();
13258 let _movement_guard = movement_gate.write().await;
13259 self.ctx
13260 .advance_data_movement_operation_epoch_to_durable_generation(durable_movement_generation);
13261 }
13262
13263 if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
13264 let stage = format!("decommission_cancel for pool {idx}");
13265 if let Err(err) =
13266 resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
13267 {
13268 warn!(
13269 event = EVENT_DECOMMISSION_STATE,
13270 component = LOG_COMPONENT_ECSTORE,
13271 subsystem = LOG_SUBSYSTEM_POOLS,
13272 pool_index = idx,
13273 state = "terminal_reload_failed",
13274 error = %err,
13275 "Decommission cancel saved locally but pool meta reload failed"
13276 );
13277 }
13278 }
13279
13280 Ok(())
13281 }
13282
13283 async fn release_decommission_canceler_slot(&self, idx: usize, owner: &DecommissionCanceler) {
13284 let mut cancelers = self.decommission_cancelers.write().await;
13285 take_and_cancel_decommission_canceler_for_operation(cancelers.as_mut_slice(), idx, owner);
13286 }
13287
13288 async fn decommission_terminal_retryable_for_operation(&self, idx: usize, owner: &DecommissionCanceler) -> bool {
13289 let _start_guard = self.start_gate.lock().await;
13290 let mut cancelers = self.decommission_cancelers.write().await;
13291 if !decommission_canceler_is_owned_by(cancelers.as_slice(), idx, owner) {
13292 owner.release();
13293 return false;
13294 }
13295
13296 let retryable = {
13297 let pool_meta = self.pool_meta.read().await;
13298 pool_meta
13299 .pools
13300 .get(idx)
13301 .and_then(|pool| pool.decommission.as_ref())
13302 .is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled)
13303 };
13304 if !retryable {
13305 take_and_cancel_decommission_canceler_for_operation(cancelers.as_mut_slice(), idx, owner);
13306 }
13307 retryable
13308 }
13309
13310 async fn retry_decommission_cancel_for_operation(self: &Arc<Self>, idx: usize, owner: &DecommissionCanceler) {
13311 if !self.decommission_terminal_retryable_for_operation(idx, owner).await {
13312 return;
13313 }
13314
13315 let mut attempt = 0usize;
13316 loop {
13317 if self
13318 .ensure_pool_meta_side_effects_safe("decommission cancel retry paused because pool metadata requires recovery")
13319 .await
13320 .is_err()
13321 {
13322 self.release_decommission_canceler_slot(idx, owner).await;
13323 return;
13324 }
13325 let Err(err) = self.decommission_cancel_for_operation(idx, owner).await else {
13326 return;
13327 };
13328 if !self.decommission_terminal_retryable_for_operation(idx, owner).await {
13329 return;
13330 }
13331 attempt += 1;
13332 warn!(
13333 event = EVENT_DECOMMISSION_STATE,
13334 component = LOG_COMPONENT_ECSTORE,
13335 subsystem = LOG_SUBSYSTEM_POOLS,
13336 pool_index = idx,
13337 state = "terminal_save_retry",
13338 terminal = "canceled",
13339 attempt,
13340 error = %err,
13341 "Decommission terminal save will be retried"
13342 );
13343 tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await;
13344 }
13345 }
13346
13347 async fn retry_decommission_failed_for_operation(&self, idx: usize, owner: &DecommissionCanceler) {
13348 let mut attempt = 0usize;
13349 loop {
13350 if self
13351 .ensure_pool_meta_side_effects_safe("decommission failure retry paused because pool metadata requires recovery")
13352 .await
13353 .is_err()
13354 {
13355 self.release_decommission_canceler_slot(idx, owner).await;
13356 return;
13357 }
13358 let Err(err) = self.decommission_failed_for_operation(idx, owner).await else {
13359 return;
13360 };
13361 if !self.decommission_terminal_retryable_for_operation(idx, owner).await {
13362 return;
13363 }
13364 attempt += 1;
13365 warn!(
13366 event = EVENT_DECOMMISSION_STATE,
13367 component = LOG_COMPONENT_ECSTORE,
13368 subsystem = LOG_SUBSYSTEM_POOLS,
13369 pool_index = idx,
13370 state = "terminal_save_retry",
13371 terminal = "failed",
13372 attempt,
13373 error = %err,
13374 "Decommission terminal save will be retried"
13375 );
13376 tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await;
13377 }
13378 }
13379
13380 async fn decommission_cancel_with_owner(self: &Arc<Self>, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> {
13381 let pools = self.pools.clone();
13382 let store = self.clone();
13383 let owner = owner.cloned();
13384 tokio::spawn(async move {
13385 store
13386 .decommission_cancel_transaction(idx, owner, true, move |snapshot, pool_meta_fence| async move {
13387 snapshot.save_no_lock_with_fence(pools, pool_meta_fence, &[idx]).await
13388 })
13389 .await
13390 })
13391 .await
13392 .map_err(|err| Error::other(format!("decommission cancel transaction task join error: {err}")))?
13393 }
13394
13395 #[cfg(test)]
13396 async fn clear_decommission_with_save<Save, SaveFuture>(self: &Arc<Self>, idx: usize, save_pool_meta: Save) -> Result<()>
13397 where
13398 Save: FnOnce() -> SaveFuture + Send + 'static,
13399 SaveFuture: Future<Output = Result<()>> + Send + 'static,
13400 {
13401 let store = self.clone();
13402 tokio::spawn(async move { store.clear_decommission_transaction(idx, save_pool_meta).await })
13403 .await
13404 .map_err(|err| Error::other(format!("clear decommission transaction task join error: {err}")))?
13405 }
13406
13407 #[tracing::instrument(skip(self))]
13408 pub async fn clear_decommission(self: &Arc<Self>, idx: usize) -> Result<()> {
13409 let store = self.clone();
13410 let save_store = store.clone();
13411 tokio::spawn(async move {
13414 store
13415 .clear_decommission_transaction(idx, move || async move { save_store.save_current_pool_meta(&[idx]).await })
13416 .await
13417 })
13418 .await
13419 .map_err(|err| Error::other(format!("clear decommission transaction task join error: {err}")))?
13420 }
13421
13422 async fn clear_decommission_transaction<Save, SaveFuture>(&self, idx: usize, save_pool_meta: Save) -> Result<()>
13423 where
13424 Save: FnOnce() -> SaveFuture,
13425 SaveFuture: Future<Output = Result<()>>,
13426 {
13427 ensure_decommission_terminal_operation_supported(self.single_pool(), "clear decommission")?;
13428 let _start_guard = self.start_gate.lock().await;
13429
13430 {
13431 let pool_meta = self.pool_meta.read().await;
13432 let pool_count = pool_meta.pools.len();
13433 ensure_valid_decommission_pool_index(pool_count, idx)?;
13434 let Some(pool) = pool_meta.pools.get(idx) else {
13435 return Err(invalid_decommission_pool_index_error(pool_count, idx));
13436 };
13437 let (decommission_present, complete, failed, canceled, unresolved_entries) = pool
13438 .decommission
13439 .as_ref()
13440 .map(|info| {
13441 (
13442 info.has_decommission_state(),
13443 info.complete,
13444 info.failed,
13445 info.canceled,
13446 info.unresolved_entries.len(),
13447 )
13448 })
13449 .unwrap_or((false, false, false, false, 0));
13450 ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled, unresolved_entries)?;
13451 }
13452 self.cancel_decommission_routines(&[idx]).await;
13455 let movement_gate = self.ctx.data_movement_operation_gate();
13456 let _movement_guard = movement_gate.write().await;
13457
13458 let (should_reload_pool_meta, previous_pool_meta) = {
13459 let rebalance_meta = self.rebalance_meta.read().await.clone();
13460 let terminal_at = OffsetDateTime::now_utc();
13461 let mut pool_meta = self.pool_meta.write().await;
13462 let previous_pool_meta = pool_meta.clone();
13463 let changed = pool_meta.clear_decommission_at(idx, terminal_at, rebalance_meta.as_ref())?;
13464 (changed, changed.then_some(previous_pool_meta))
13465 };
13466
13467 if should_reload_pool_meta && let Err(err) = save_pool_meta().await {
13468 if let Some(previous_pool_meta) = previous_pool_meta {
13469 let mut pool_meta = self.pool_meta.write().await;
13470 rollback_decommission_pool_meta(&mut pool_meta, &previous_pool_meta, &[idx]);
13471 }
13472 return Err(err);
13473 }
13474
13475 if should_reload_pool_meta {
13476 self.ctx.advance_data_movement_operation_epoch();
13477 }
13478 drop(_movement_guard);
13479
13480 if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
13481 let stage = format!("clear_decommission for pool {idx}");
13482 if let Err(err) =
13483 resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
13484 {
13485 warn!(
13486 event = EVENT_DECOMMISSION_STATE,
13487 component = LOG_COMPONENT_ECSTORE,
13488 subsystem = LOG_SUBSYSTEM_POOLS,
13489 pool_index = idx,
13490 state = "terminal_reload_failed",
13491 error = %err,
13492 "Decommission clear saved locally but pool meta reload failed"
13493 );
13494 }
13495 }
13496
13497 Ok(())
13498 }
13499
13500 async fn promote_queued_decommission(&self, idx: usize, owner: &DecommissionCanceler) -> Result<OffsetDateTime> {
13501 let (changed, generation, save_error) = {
13503 let _start_guard = self.start_gate.lock().await;
13504 let movement_gate = self.ctx.data_movement_operation_gate();
13505 let _movement_guard = movement_gate.write().await;
13506 let mut save_guard = self.pool_meta_save_gate.lock().await;
13507 let (pool_meta_guard, mut snapshot) = self
13508 .acquire_pool_meta_write_guard(&mut save_guard, "decommission promotion failed")
13509 .await?;
13510 let rebalance_meta = self.rebalance_meta.read().await.clone();
13511 let capacity_infos = if self.pools.is_empty() {
13512 Vec::new()
13513 } else {
13514 self.get_decommission_all_pool_capacity_infos_with_owned_usage().await?
13515 };
13516 let target_fence_proof = crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof();
13517 let mut pool_meta = self.pool_meta.write().await;
13518 if pool_meta.pools.get(idx).is_none() {
13519 return Err(Error::other("failed to start decommission: target pool was not found"));
13520 }
13521 let capacity_recovery_needed = active_decommission_source_indices(&pool_meta).into_iter().any(|source_idx| {
13522 pool_meta
13523 .pools
13524 .get(source_idx)
13525 .and_then(|pool| pool.decommission.as_ref())
13526 .and_then(|info| info.capacity_reservation.as_ref())
13527 .is_none_or(|reservation| !reservation.active())
13528 });
13529 let capacity_indices = if capacity_infos.is_empty() {
13530 Vec::new()
13531 } else {
13532 recover_decommission_capacity_reservations(
13533 &mut pool_meta,
13534 &capacity_infos,
13535 OffsetDateTime::now_utc(),
13536 target_fence_proof.is_some(),
13537 )?
13538 };
13539 let target_fence_proof_required = capacity_recovery_needed
13540 && active_decommission_capacity_model(&pool_meta)? == Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION);
13541 ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?;
13542 let reconciled = reconcile_decommission_meta_buckets(&mut pool_meta, idx);
13543 let promoted = pool_meta.promote_queued_decommission_at(idx, OffsetDateTime::now_utc(), rebalance_meta.as_ref());
13544 let mut changed_indices = capacity_indices;
13545 let changed = !changed_indices.is_empty() || reconciled || promoted;
13546 if changed {
13547 if !changed_indices.contains(&idx) {
13548 changed_indices.push(idx);
13549 }
13550 merge_pool_meta_updates_for_save(&mut snapshot, &pool_meta, &changed_indices, "decommission promotion failed")?;
13551 }
13552 drop(pool_meta);
13553
13554 let (save_outcome, save_error) = if changed {
13555 ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?;
13556 match snapshot
13557 .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &changed_indices)
13558 .await
13559 {
13560 Ok(outcome) => (Some(outcome), None),
13561 Err(err) => (None, Some(err)),
13562 }
13563 } else {
13564 (None, None)
13565 };
13566 let generation = self.active_decommission_generation(idx).await?;
13567 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission promotion failed")?;
13568 ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?;
13569 if let Some(outcome) = save_outcome {
13570 let mut pool_meta = self.pool_meta.write().await;
13571 pool_meta.version = pool_meta.version.max(outcome.committed.version);
13572 drop(pool_meta);
13573 outcome.disarm();
13574 }
13575 (changed, generation, save_error)
13576 };
13577
13578 if let Some(err) = save_error {
13579 resolve_decommission_terminal_mark_after_error_result(
13580 self.decommission_failed_for_operation(idx, owner).await,
13581 idx,
13582 &err,
13583 )?;
13584 return Err(err);
13585 }
13586
13587 if changed {
13588 self.ctx.advance_data_movement_operation_epoch();
13589 }
13590 if changed && let Some(notification_sys) = runtime_sources::notification_sys() {
13591 let stage = format!("promote_queued_decommission for pool {idx}");
13592 if let Err(err) =
13593 resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
13594 {
13595 resolve_decommission_terminal_mark_after_error_result(
13596 self.decommission_failed_for_operation(idx, owner).await,
13597 idx,
13598 &err,
13599 )?;
13600 return Err(err);
13601 }
13602 }
13603
13604 Ok(generation)
13605 }
13606
13607 #[cfg(test)]
13608 pub(crate) async fn promote_queued_decommission_for_test(&self, idx: usize) -> Result<()> {
13609 let owner = DecommissionCanceler::new(CancellationToken::new());
13610 self.promote_queued_decommission(idx, &owner).await?;
13611 let mut cancelers = self.decommission_cancelers.write().await;
13612 if let Some(slot) = cancelers.get_mut(idx)
13613 && let Some(previous) = slot.replace(owner)
13614 {
13615 previous.release();
13616 }
13617 Ok(())
13618 }
13619
13620 async fn record_decommission_terminal_reload_failure(&self, idx: usize, stage: &str, err: Error) -> Result<()> {
13621 let movement_gate = self.ctx.data_movement_operation_gate();
13622 let _movement_guard = movement_gate.write().await;
13623 let changed = {
13624 let rebalance_meta = self.rebalance_meta.read().await.clone();
13625 let mut pool_meta = self.pool_meta.write().await;
13626 pool_meta.record_decommission_terminal_reload_failure_at(
13627 idx,
13628 stage,
13629 err.to_string(),
13630 OffsetDateTime::now_utc(),
13631 rebalance_meta.as_ref(),
13632 )?
13633 };
13634
13635 if changed {
13636 self.save_current_pool_meta(&[idx]).await?;
13637 }
13638
13639 Ok(())
13640 }
13641
13642 async fn pause_decommission_for_capacity(&self, idx: usize, err: &Error) -> Result<()> {
13643 let mut save_guard = self.pool_meta_save_gate.lock().await;
13644 let (pool_meta_guard, mut snapshot) = self
13645 .acquire_pool_meta_write_guard(&mut save_guard, "decommission capacity pause failed")
13646 .await?;
13647 snapshot.mark_decommission_capacity_blocked(idx, err.to_string(), OffsetDateTime::now_utc())?;
13648 let outcome = snapshot
13649 .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx])
13650 .await?;
13651 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission capacity pause failed")?;
13652 {
13653 let mut pool_meta = self.pool_meta.write().await;
13654 publish_pool_meta_updates(&mut pool_meta, &outcome.committed, &[idx]);
13655 }
13656 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission capacity pause failed")?;
13657 outcome.disarm();
13658 Ok(())
13659 }
13660
13661 pub async fn is_decommission_running(&self) -> bool {
13662 {
13663 let cancelers = self.decommission_cancelers.read().await;
13664 if has_active_decommission_canceler(cancelers.as_slice()) {
13665 return true;
13666 }
13667 }
13668
13669 let pool_meta = self.pool_meta.read().await;
13670 for pool in pool_meta.pools.iter() {
13671 if let Some(ref info) = pool.decommission
13672 && info.has_decommission_state()
13673 && !info.complete
13674 && !info.failed
13675 && !info.canceled
13676 {
13677 return true;
13678 }
13679 }
13680
13681 false
13682 }
13683
13684 async fn decommission_cancel_requested(&self, idx: usize, rx: &CancellationToken) -> bool {
13685 let pool_meta = self.pool_meta.read().await;
13686 is_decommission_cancel_requested(rx.is_cancelled(), pool_meta.pools.get(idx))
13687 }
13688
13689 #[cfg(test)]
13690 async fn cancel_decommission_routines_and_wait(&self, indices: &[usize]) {
13691 self.cancel_decommission_routines(indices).await;
13692 let movement_gate = self.ctx.data_movement_operation_gate();
13693 let _movement_guard = movement_gate.write().await;
13694 }
13695
13696 async fn cancel_decommission_routines(&self, indices: &[usize]) {
13697 {
13698 let mut cancelers = self.decommission_cancelers.write().await;
13699 for idx in indices {
13700 take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), *idx);
13701 }
13702 }
13703 }
13704
13705 async fn quiesce_decommission_worker_after_join_error(&self, canceler: &DecommissionCanceler) {
13706 canceler.cancel();
13707 let movement_gate = self.ctx.data_movement_operation_gate();
13708 let _movement_guard = movement_gate.write().await;
13709 }
13710
13711 async fn reserve_decommission_routines(
13712 &self,
13713 rx: &CancellationToken,
13714 indices: &[usize],
13715 ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
13716 let indices = dedup_indices(indices);
13717 if indices.is_empty() {
13718 return Ok(Vec::new());
13719 }
13720
13721 let _start_guard = self.start_gate.lock().await;
13722 let save_guard = self.pool_meta_save_gate.lock().await;
13723 save_guard.ensure_write_safe("decommission cannot be scheduled while pool metadata requires recovery")?;
13724 let indices = {
13725 let pool_meta = self.pool_meta.read().await;
13726 let indices = resumable_decommission_queue_indices(&pool_meta)
13727 .into_iter()
13728 .filter(|idx| indices.contains(idx))
13729 .collect::<Vec<_>>();
13730 if !indices.is_empty() {
13731 ensure_decommission_ledger_persistence_supported(&pool_meta)?;
13732 }
13733 indices
13734 };
13735 if indices.is_empty() {
13736 return Ok(Vec::new());
13737 }
13738
13739 let index_cancelers = {
13740 let mut cancelers = self.decommission_cancelers.write().await;
13741 let missing = missing_decommission_worker_prefix(indices.as_slice(), cancelers.as_slice());
13742 if missing.is_empty() {
13743 return Ok(Vec::new());
13744 }
13745 let bound = bind_missing_decommission_cancelers(missing.as_slice(), rx, cancelers.as_mut_slice());
13746 let guards = guard_decommission_cancelers(bound);
13747 ensure_decommission_routines_scheduled(guards.len(), missing.len())?;
13748 guards
13749 };
13750 Ok(index_cancelers)
13751 }
13752
13753 async fn reserve_missing_local_decommission_routines(
13754 &self,
13755 rx: &CancellationToken,
13756 endpoints: &EndpointServerPools,
13757 ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
13758 let indices = {
13759 let pool_meta = self.pool_meta.read().await;
13760 resumable_decommission_queue_indices(&pool_meta)
13761 };
13762 let indices = local_decommission_queue_prefix(endpoints, &indices)?;
13763 self.reserve_decommission_routines(rx, indices.as_slice()).await
13764 }
13765
13766 pub async fn spawn_missing_local_decommission_routines(self: &Arc<Self>) -> Result<()> {
13767 self.spawn_missing_local_decommission_routines_with_token(CancellationToken::new())
13768 .await
13769 }
13770
13771 pub(crate) async fn has_active_local_decommission_worker(&self) -> bool {
13772 let cancelers = self.decommission_cancelers.read().await;
13773 has_active_decommission_canceler(cancelers.as_slice())
13774 }
13775
13776 pub(crate) async fn spawn_missing_local_decommission_routines_with_token(
13777 self: &Arc<Self>,
13778 rx: CancellationToken,
13779 ) -> Result<()> {
13780 #[cfg(test)]
13781 let endpoints = self.instance_endpoints().unwrap_or_else(|| self.endpoints());
13782 #[cfg(not(test))]
13783 let endpoints = self.endpoints();
13784 let index_cancelers = self.reserve_missing_local_decommission_routines(&rx, &endpoints).await?;
13785 if index_cancelers.is_empty() {
13786 return Ok(());
13787 }
13788
13789 drop(spawn_decommission_index_cancelers(
13790 self.clone(),
13791 rx,
13792 index_cancelers,
13793 Arc::new(Semaphore::new(decommission_entry_concurrency_limit())),
13794 ));
13795 Ok(())
13796 }
13797
13798 #[tracing::instrument(skip(self, rx))]
13799 pub async fn decommission(&self, rx: CancellationToken, indices: Vec<usize>) -> Result<()> {
13800 let indices = dedup_indices(&indices);
13801
13802 info!(
13803 event = EVENT_DECOMMISSION_STATE,
13804 component = LOG_COMPONENT_ECSTORE,
13805 subsystem = LOG_SUBSYSTEM_POOLS,
13806 pool_indices = ?indices,
13807 state = "requested",
13808 "Decommission requested"
13809 );
13810 validate_start_decommission_request(&indices, self.single_pool())?;
13811
13812 self.ensure_decommission_rebalance_idle_after_refresh().await?;
13813
13814 let store = require_decommission_store(runtime_sources::object_store_handle(), "start decommission")?;
13815 let local_indices = local_decommission_queue_prefix(&self.endpoints(), &indices)?;
13816 let index_cancelers = self
13817 .start_decommission_with_routines(indices, &rx, local_indices.as_slice())
13818 .await?;
13819 drop(spawn_decommission_index_cancelers(
13820 store,
13821 rx,
13822 index_cancelers,
13823 Arc::new(Semaphore::new(decommission_entry_concurrency_limit())),
13824 ));
13825
13826 Ok(())
13827 }
13828
13829 async fn active_decommission_generation(&self, idx: usize) -> Result<OffsetDateTime> {
13830 let pool_meta = self.pool_meta.read().await;
13831 let Some(pool) = pool_meta.pools.get(idx) else {
13832 return Err(invalid_decommission_pool_index_error(pool_meta.pools.len(), idx));
13833 };
13834 let Some(info) = pool.decommission.as_ref() else {
13835 return Err(decommission_metadata_not_initialized_error("load decommission generation"));
13836 };
13837 let Some(generation) = info.start_time else {
13838 return Err(Error::OperationCanceled);
13839 };
13840 ensure_decommission_generation(&pool_meta, idx, generation)?;
13841 Ok(generation)
13842 }
13843
13844 async fn ensure_decommission_generation_current(&self, idx: usize, generation: OffsetDateTime) -> Result<()> {
13845 let pool_meta = self.pool_meta.read().await;
13846 ensure_decommission_generation(&pool_meta, idx, generation)
13847 }
13848
13849 #[allow(clippy::too_many_arguments)]
13850 async fn decommission_entry_worker(
13851 self: Arc<Self>,
13852 rx: CancellationToken,
13853 idx: usize,
13854 set_idx: usize,
13855 generation: OffsetDateTime,
13856 bucket: String,
13857 set: Arc<SetDisks>,
13858 lifecycle_config: Option<BucketLifecycleConfiguration>,
13859 object_lock_config: Option<ObjectLockConfiguration>,
13860 replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
13861 expected_bucket_incarnation_id: Option<uuid::Uuid>,
13862 source_changed_exhaustions: Arc<AtomicUsize>,
13863 entry_budget: Arc<Semaphore>,
13864 queue: Arc<tokio::sync::Mutex<mpsc::Receiver<QueuedDecommissionEntry>>>,
13865 deferred: Arc<tokio::sync::Mutex<Vec<DeferredDecommissionEntry>>>,
13866 entry_error: Arc<tokio::sync::Mutex<Option<Error>>>,
13867 ) {
13868 loop {
13869 let queued = tokio::select! {
13870 biased;
13871 _ = rx.cancelled() => return,
13872 item = async {
13873 let mut queue = queue.lock().await;
13874 queue.recv().await
13875 } => item,
13876 };
13877 let Some(QueuedDecommissionEntry { entry, queue_permit }) = queued else {
13878 return;
13879 };
13880 if entry.is_dir() {
13883 continue;
13884 }
13885 let object_name = entry.name.clone();
13886 let mut attempts = 0;
13887
13888 if entry_error.lock().await.is_some() {
13889 drop(queue_permit);
13890 continue;
13891 }
13892
13893 if let Err(err) = self.ensure_decommission_generation_current(idx, generation).await {
13894 if matches!(err, Error::OperationCanceled) {
13895 rx.cancel();
13896 } else {
13897 record_decommission_entry_error(&entry_error, &rx, err).await;
13898 }
13899 return;
13900 }
13901
13902 if let Err(err) = backpressure::wait_for_data_movement_admission(DataMovementOperation::Decommission, idx, &rx).await
13903 {
13904 if matches!(err, Error::OperationCanceled) {
13905 return;
13906 }
13907 error!(
13908 event = EVENT_DECOMMISSION_ENTRY,
13909 component = LOG_COMPONENT_ECSTORE,
13910 subsystem = LOG_SUBSYSTEM_POOLS,
13911 pool_index = idx,
13912 set_index = set_idx,
13913 bucket = %bucket,
13914 object = %object_name,
13915 state = "entry_admission_failed",
13916 error = %err,
13917 "Decommission entry admission failed"
13918 );
13919 record_decommission_entry_error(&entry_error, &rx, err).await;
13920 return;
13921 }
13922
13923 let entry_budget_permit = match tokio::select! {
13924 biased;
13925 _ = rx.cancelled() => return,
13926 permit = entry_budget.clone().acquire_owned() => permit,
13927 } {
13928 Ok(permit) => permit,
13929 Err(err) => {
13930 let err = decommission_entry_budget_acquire_error(err);
13931 error!(
13932 event = EVENT_DECOMMISSION_ENTRY,
13933 component = LOG_COMPONENT_ECSTORE,
13934 subsystem = LOG_SUBSYSTEM_POOLS,
13935 pool_index = idx,
13936 set_index = set_idx,
13937 bucket = %bucket,
13938 object = %object_name,
13939 state = "entry_budget_acquire_failed",
13940 error = %err,
13941 "Decommission entry budget permit acquire failed"
13942 );
13943 record_decommission_entry_error(&entry_error, &rx, err).await;
13944 return;
13945 }
13946 };
13947
13948 let result = self
13949 .decommission_entry(
13950 rx.clone(),
13951 idx,
13952 generation,
13953 entry.clone(),
13954 bucket.clone(),
13955 set.clone(),
13956 lifecycle_config.clone(),
13957 object_lock_config.clone(),
13958 replication_config.clone(),
13959 expected_bucket_incarnation_id,
13960 Arc::clone(&source_changed_exhaustions),
13961 &mut attempts,
13962 )
13963 .await;
13964 drop(entry_budget_permit);
13965
13966 match result {
13967 Ok(DecommissionEntryOutcome::Complete) => drop(queue_permit),
13968 Ok(DecommissionEntryOutcome::Deferred(err)) => {
13969 drop(queue_permit);
13973 deferred.lock().await.push(DeferredDecommissionEntry {
13977 name: object_name,
13978 attempts,
13979 });
13980 info!(
13981 event = EVENT_DECOMMISSION_ENTRY,
13982 component = LOG_COMPONENT_ECSTORE,
13983 subsystem = LOG_SUBSYSTEM_POOLS,
13984 pool_index = idx,
13985 set_index = set_idx,
13986 bucket = %bucket,
13987 object = %entry.name,
13988 state = "entry_deferred_to_round_end",
13989 error = %err,
13990 "Decommission entry deferred to the end of the current round"
13991 );
13992 }
13993 Err(err) => {
13994 drop(queue_permit);
13995 error!(
13996 event = EVENT_DECOMMISSION_ENTRY,
13997 component = LOG_COMPONENT_ECSTORE,
13998 subsystem = LOG_SUBSYSTEM_POOLS,
13999 pool_index = idx,
14000 set_index = set_idx,
14001 bucket = %bucket,
14002 object = %object_name,
14003 state = "entry_failed",
14004 error = %err,
14005 "Decommission entry failed"
14006 );
14007 record_decommission_entry_error(&entry_error, &rx, err).await;
14008 return;
14009 }
14010 }
14011 }
14012 }
14013
14014 #[allow(clippy::too_many_arguments)]
14019 async fn replay_deferred_decommission_entry(
14020 self: Arc<Self>,
14021 rx: CancellationToken,
14022 idx: usize,
14023 set_idx: usize,
14024 generation: OffsetDateTime,
14025 mut deferred_entry: DeferredDecommissionEntry,
14026 bucket: String,
14027 set: Arc<SetDisks>,
14028 lifecycle_config: Option<BucketLifecycleConfiguration>,
14029 object_lock_config: Option<ObjectLockConfiguration>,
14030 replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
14031 expected_bucket_incarnation_id: Option<uuid::Uuid>,
14032 source_changed_exhaustions: Arc<AtomicUsize>,
14033 entry_budget: Arc<Semaphore>,
14034 entry_error: Arc<tokio::sync::Mutex<Option<Error>>>,
14035 ) -> Result<DecommissionEntryOutcome> {
14036 let object_name = deferred_entry.name;
14037 let entry = MetaCacheEntry {
14038 name: object_name.clone(),
14039 ..Default::default()
14040 };
14041
14042 if entry_error.lock().await.is_some() || rx.is_cancelled() {
14043 return Ok(DecommissionEntryOutcome::Complete);
14044 }
14045
14046 if let Err(err) = self.ensure_decommission_generation_current(idx, generation).await {
14047 if matches!(err, Error::OperationCanceled) {
14048 rx.cancel();
14049 } else {
14050 record_decommission_entry_error(&entry_error, &rx, err).await;
14051 }
14052 return Ok(DecommissionEntryOutcome::Complete);
14053 }
14054
14055 if let Err(err) = backpressure::wait_for_data_movement_admission(DataMovementOperation::Decommission, idx, &rx).await {
14056 if matches!(err, Error::OperationCanceled) {
14057 return Ok(DecommissionEntryOutcome::Complete);
14058 }
14059 error!(
14060 event = EVENT_DECOMMISSION_ENTRY,
14061 component = LOG_COMPONENT_ECSTORE,
14062 subsystem = LOG_SUBSYSTEM_POOLS,
14063 pool_index = idx,
14064 set_index = set_idx,
14065 bucket = %bucket,
14066 object = %object_name,
14067 state = "entry_admission_failed",
14068 error = %err,
14069 "Decommission entry admission failed"
14070 );
14071 record_decommission_entry_error(&entry_error, &rx, err).await;
14072 return Ok(DecommissionEntryOutcome::Complete);
14073 }
14074
14075 let entry_budget_permit = match tokio::select! {
14076 biased;
14077 _ = rx.cancelled() => return Ok(DecommissionEntryOutcome::Complete),
14078 permit = entry_budget.clone().acquire_owned() => permit,
14079 } {
14080 Ok(permit) => permit,
14081 Err(err) => {
14082 let err = decommission_entry_budget_acquire_error(err);
14083 record_decommission_entry_error(&entry_error, &rx, err).await;
14084 return Ok(DecommissionEntryOutcome::Complete);
14085 }
14086 };
14087
14088 let result = self
14089 .decommission_entry(
14090 rx.clone(),
14091 idx,
14092 generation,
14093 entry,
14094 bucket.clone(),
14095 set,
14096 lifecycle_config,
14097 object_lock_config,
14098 replication_config,
14099 expected_bucket_incarnation_id,
14100 source_changed_exhaustions,
14101 &mut deferred_entry.attempts,
14102 )
14103 .await;
14104 drop(entry_budget_permit);
14105
14106 let err = match result {
14107 Ok(DecommissionEntryOutcome::Complete) => return Ok(DecommissionEntryOutcome::Complete),
14108 Ok(outcome @ DecommissionEntryOutcome::Deferred(_)) => return Ok(outcome),
14109 Err(err) => err,
14110 };
14111 error!(
14112 event = EVENT_DECOMMISSION_ENTRY,
14113 component = LOG_COMPONENT_ECSTORE,
14114 subsystem = LOG_SUBSYSTEM_POOLS,
14115 pool_index = idx,
14116 set_index = set_idx,
14117 bucket = %bucket,
14118 object = %object_name,
14119 state = "entry_failed",
14120 error = %err,
14121 "Decommission entry failed"
14122 );
14123 record_decommission_entry_error(&entry_error, &rx, err).await;
14124 Ok(DecommissionEntryOutcome::Complete)
14125 }
14126
14127 #[allow(clippy::too_many_arguments)]
14128 async fn decommission_set(
14129 self: Arc<Self>,
14130 rx: CancellationToken,
14131 idx: usize,
14132 set_idx: usize,
14133 generation: OffsetDateTime,
14134 set: Arc<SetDisks>,
14135 bi: DecomBucketInfo,
14136 lifecycle_config: Option<BucketLifecycleConfiguration>,
14137 object_lock_config: Option<ObjectLockConfiguration>,
14138 replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
14139 expected_bucket_incarnation_id: Option<uuid::Uuid>,
14140 source_changed_exhaustions: Arc<AtomicUsize>,
14141 entry_budget: Arc<Semaphore>,
14142 entry_error: Arc<tokio::sync::Mutex<Option<Error>>>,
14143 ) -> Result<()> {
14144 let worker_count = DECOMMISSION_ENTRY_WORKERS_PER_SET;
14145 let queue_capacity = decommission_entry_queue_capacity(worker_count);
14146 let outstanding_capacity = queue_capacity.saturating_add(worker_count);
14147 let outstanding = Arc::new(Semaphore::new(outstanding_capacity));
14148 let (tx, rx_queue) = mpsc::channel(queue_capacity);
14149 let queue = Arc::new(tokio::sync::Mutex::new(rx_queue));
14150 let deferred: Arc<tokio::sync::Mutex<Vec<DeferredDecommissionEntry>>> = Arc::new(tokio::sync::Mutex::new(Vec::new()));
14151
14152 let mut entry_workers = tokio::task::JoinSet::new();
14153 for _ in 0..worker_count {
14154 let this = self.clone();
14155 let rx = rx.clone();
14156 let bucket = bi.name.clone();
14157 let set = set.clone();
14158 let lifecycle_config = lifecycle_config.clone();
14159 let object_lock_config = object_lock_config.clone();
14160 let replication_config = replication_config.clone();
14161 let source_changed_exhaustions = Arc::clone(&source_changed_exhaustions);
14162 let queue = queue.clone();
14163 let entry_budget = entry_budget.clone();
14164 let deferred = deferred.clone();
14165 let entry_error = entry_error.clone();
14166 entry_workers.spawn(async move {
14167 this.decommission_entry_worker(
14168 rx,
14169 idx,
14170 set_idx,
14171 generation,
14172 bucket,
14173 set,
14174 lifecycle_config,
14175 object_lock_config,
14176 replication_config,
14177 expected_bucket_incarnation_id,
14178 source_changed_exhaustions,
14179 entry_budget,
14180 queue,
14181 deferred,
14182 entry_error,
14183 )
14184 .await;
14185 });
14186 }
14187
14188 let callback: ListCallback = Arc::new({
14189 let tx = tx.clone();
14190 let outstanding = outstanding.clone();
14191 let callback_rx = rx.clone();
14192 let entry_error = entry_error.clone();
14193 let bucket = bi.name.clone();
14194 move |entry: MetaCacheEntry| {
14195 let tx = tx.clone();
14196 let outstanding = outstanding.clone();
14197 let callback_rx = callback_rx.clone();
14198 let entry_error = entry_error.clone();
14199 let bucket = bucket.clone();
14200 Box::pin(async move {
14201 if callback_rx.is_cancelled() || entry_error.lock().await.is_some() {
14202 return;
14203 }
14204
14205 if matches!(
14206 enqueue_decommission_entry(&callback_rx, &outstanding, &tx, entry).await,
14207 DecommissionEntryEnqueueResult::Closed
14208 ) {
14209 let err = Error::other("decommission entry queue closed");
14210 error!(
14211 event = EVENT_DECOMMISSION_ENTRY,
14212 component = LOG_COMPONENT_ECSTORE,
14213 subsystem = LOG_SUBSYSTEM_POOLS,
14214 pool_index = idx,
14215 set_index = set_idx,
14216 bucket = %bucket,
14217 state = "entry_queue_closed",
14218 error = %err,
14219 "Decommission entry queue closed"
14220 );
14221 record_decommission_entry_error(&entry_error, &callback_rx, err).await;
14222 }
14223 })
14224 }
14225 });
14226
14227 let list_set = set.clone();
14228 let list_rx = rx.clone();
14229 let list_rx_for_list = list_rx.clone();
14230 let list_rx_for_drain = list_rx.clone();
14231 let list_bi = bi.clone();
14232 let list_outstanding = outstanding.clone();
14233 let list_entry_error = entry_error.clone();
14234 let list_store = self.clone();
14235 let mut listing = tokio::spawn(async move {
14236 run_decommission_listing_with_retry_and_drain(
14237 list_rx.clone(),
14238 list_bi.name.clone(),
14239 callback,
14240 idx,
14241 set_idx,
14242 DECOMMISSION_LISTING_MAX_ATTEMPTS,
14243 move |callback| {
14244 let set = list_set.clone();
14245 let rx = list_rx_for_list.clone();
14246 let bucket = list_bi.clone();
14247 let entry_error = list_entry_error.clone();
14248 let store = list_store.clone();
14249 async move {
14250 set.list_objects_to_decommission(
14251 store,
14252 rx,
14253 bucket,
14254 callback,
14255 entry_error,
14256 idx,
14257 set_idx,
14258 generation,
14259 false,
14260 )
14261 .await
14262 }
14263 },
14264 move || {
14265 let rx = list_rx_for_drain.clone();
14266 let outstanding = list_outstanding.clone();
14267 async move { drain_decommission_entry_queue(&rx, &outstanding, outstanding_capacity).await }
14268 },
14269 )
14270 .await
14271 });
14272
14273 let mut listing_result = None;
14274 let mut workers_left = worker_count;
14275 let mut sender = Some(tx);
14276 while listing_result.is_none() || workers_left > 0 {
14277 tokio::select! {
14278 biased;
14279 result = &mut listing, if listing_result.is_none() => {
14280 let result = resolve_decommission_listing_worker_result(set_idx, result);
14281 if result.is_err() {
14282 rx.cancel();
14283 }
14284 listing_result = Some(result);
14285 drop(sender.take());
14286 }
14287 worker_result = entry_workers.join_next(), if workers_left > 0 => {
14288 workers_left -= 1;
14289 if let Some(Err(err)) = worker_result {
14290 let err = Error::other(format!("decommission entry worker {set_idx} task join error: {err}"));
14291 error!(
14292 event = EVENT_DECOMMISSION_ENTRY,
14293 component = LOG_COMPONENT_ECSTORE,
14294 subsystem = LOG_SUBSYSTEM_POOLS,
14295 pool_index = idx,
14296 set_index = set_idx,
14297 bucket = %bi.name,
14298 state = "entry_worker_join_failed",
14299 error = %err,
14300 "Decommission entry worker task failed"
14301 );
14302 record_decommission_entry_error(&entry_error, &rx, err).await;
14303 }
14304 }
14305 }
14306 }
14307
14308 let listing_result = listing_result.unwrap_or_else(|| Err(Error::other("decommission listing task did not complete")));
14309 if let Some(err) = entry_error.lock().await.clone() {
14310 return Err(err);
14311 }
14312 listing_result?;
14313
14314 let deferred_entries = std::mem::take(&mut *deferred.lock().await);
14318 let deferred_remaining = Arc::new(AtomicUsize::new(0));
14319 if !deferred_entries.is_empty() {
14320 futures::stream::iter(deferred_entries)
14323 .for_each_concurrent(Some(worker_count), |deferred_entry| {
14324 let this = self.clone();
14325 let rx = rx.clone();
14326 let bucket = bi.name.clone();
14327 let set = set.clone();
14328 let lifecycle_config = lifecycle_config.clone();
14329 let object_lock_config = object_lock_config.clone();
14330 let replication_config = replication_config.clone();
14331 let source_changed_exhaustions = Arc::clone(&source_changed_exhaustions);
14332 let entry_budget = entry_budget.clone();
14333 let entry_error = entry_error.clone();
14334 let deferred_remaining = Arc::clone(&deferred_remaining);
14335 async move {
14336 let outcome = this
14337 .replay_deferred_decommission_entry(
14338 rx,
14339 idx,
14340 set_idx,
14341 generation,
14342 deferred_entry,
14343 bucket,
14344 set,
14345 lifecycle_config,
14346 object_lock_config,
14347 replication_config,
14348 expected_bucket_incarnation_id,
14349 source_changed_exhaustions,
14350 entry_budget,
14351 entry_error,
14352 )
14353 .await;
14354 if matches!(outcome, Ok(DecommissionEntryOutcome::Deferred(_))) {
14355 deferred_remaining.fetch_add(1, Ordering::Relaxed);
14356 }
14357 }
14358 })
14359 .await;
14360 if let Some(err) = entry_error.lock().await.clone() {
14361 return Err(err);
14362 }
14363 }
14364 let deferred_remaining = deferred_remaining.load(Ordering::Relaxed);
14365 ensure_decommission_deferred_drained(bi.name.as_str(), set_idx, deferred_remaining)
14368 }
14369
14370 async fn track_decommission_entry_progress_stage(
14371 &self,
14372 idx: usize,
14373 generation: OffsetDateTime,
14374 bucket: &str,
14375 object: &str,
14376 stage: &'static str,
14377 ) -> Result<()> {
14378 {
14379 let mut pool_meta = self.pool_meta.write().await;
14380 ensure_decommission_generation(&pool_meta, idx, generation)?;
14381 track_decommission_current_object_stage(&mut pool_meta, idx, bucket, object, stage)
14382 .map_err(|err| with_decommission_entry_context(stage, bucket, object, err))?;
14383 }
14384
14385 Ok(())
14386 }
14387
14388 #[allow(clippy::too_many_arguments)]
14389 async fn wait_decommission_target_gate_retry(
14390 &self,
14391 rx: &CancellationToken,
14392 idx: usize,
14393 generation: OffsetDateTime,
14394 target_pool_index: usize,
14395 capacity_owner: Option<DecommissionCapacityOwner>,
14396 set: &SetDisks,
14397 entry: &MetaCacheEntry,
14398 bucket: &str,
14399 expected_version: &FileInfo,
14400 target_busy_attempt: usize,
14401 ) -> Result<Option<DecommissionCapacityTargetPermit>> {
14402 #[cfg(test)]
14403 notify_decommission_target_gate_retry(self.id);
14404 let mut wait_attempt = target_busy_attempt;
14405 let target_guard = loop {
14406 let retry_delay =
14407 decommission_capacity_gate_retry_delay(wait_attempt.min(DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS));
14408 if wait_decommission_retry_backoff(rx, retry_delay).await {
14409 decommission_cancel_signal_result(rx.is_cancelled())?;
14410 }
14411 if self.decommission_cancel_requested(idx, rx).await {
14412 rx.cancel();
14413 }
14414 decommission_cancel_signal_result(rx.is_cancelled())?;
14415 self.ensure_decommission_generation_current(idx, generation).await?;
14416
14417 match self.acquire_decommission_capacity_target_guard(target_pool_index).await {
14418 Ok(guard) => break guard,
14419 Err(err) if is_decommission_capacity_target_gate_busy(&err) => {
14420 if decommission_capacity_target_gate_retry_exhausted(wait_attempt) {
14421 warn!(
14422 event = EVENT_DECOMMISSION_STATE,
14423 component = LOG_COMPONENT_ECSTORE,
14424 subsystem = LOG_SUBSYSTEM_POOLS,
14425 pool_index = idx,
14426 target_pool_index,
14427 attempts = wait_attempt.saturating_add(1),
14428 state = "capacity_gate_retry_exhausted",
14429 error = %err,
14430 "Decommission target capacity gate remained busy; pausing for supervised retry"
14431 );
14432 return Err(err);
14433 }
14434 wait_attempt = wait_attempt.saturating_add(1);
14435 }
14436 Err(err) => return Err(err),
14437 }
14438 };
14439
14440 #[cfg(test)]
14441 notify_decommission_target_gate_exact_reload(self.id);
14442 let current = load_decommission_entry_exact_versions(set, entry, bucket, "target_gate_retry").await?;
14443 ensure_decommission_capacity_target_fence(&target_guard, target_pool_index, "source identity revalidation")?;
14444 let expected_identity = data_movement::source_cleanup_version_identity(expected_version);
14445 let identity_current = current
14446 .versions
14447 .iter()
14448 .any(|version| data_movement::source_cleanup_version_identity(version) == expected_identity);
14449 if !identity_current {
14450 return Ok(None);
14451 }
14452 let owner = capacity_owner.ok_or_else(|| Error::other("target-gate retry is missing its decommission capacity owner"))?;
14453 let owner = owner.with_mutation_id(decommission_capacity_version_mutation_id(owner, bucket, expected_version));
14454 install_decommission_capacity_target_permit(self.id, target_pool_index, owner, target_guard).map(Some)
14455 }
14456
14457 #[cfg(feature = "test-util")]
14458 pub async fn prepare_scanner_pause_backlog_retirement_for_test(
14459 &self,
14460 source_pool_index: usize,
14461 source_bytes: usize,
14462 ) -> Result<()> {
14463 let source_bytes = source_bytes.max(1);
14464 let total = source_bytes.saturating_mul(8).saturating_add(64 * 1024);
14465 let capacities = (0..self.pools.len())
14466 .map(|pool_index| {
14467 DecommissionPoolCapacityInfo::for_test(
14468 pool_index,
14469 DecommissionErasureLayout { data: 1, parity: 0 },
14470 if pool_index == source_pool_index { 0 } else { total },
14471 total,
14472 if pool_index == source_pool_index { source_bytes } else { 0 },
14473 )
14474 })
14475 .collect();
14476 set_decommission_capacity_info_overrides_for_test(self.id, vec![capacities]);
14477 self.save_current_pool_meta_for_decommission_start(&[source_pool_index], Vec::new())
14478 .await
14479 .map(|_| ())
14480 }
14481
14482 #[cfg(feature = "test-util")]
14483 pub async fn retire_scanner_pause_backlog_for_test(
14484 self: &Arc<Self>,
14485 source_pool_index: usize,
14486 source_set_index: usize,
14487 ) -> Result<()> {
14488 let set = self
14489 .pools
14490 .get(source_pool_index)
14491 .and_then(|pool| pool.disk_set.get(source_set_index))
14492 .cloned()
14493 .ok_or_else(|| Error::other("scanner retirement test requested an unknown set"))?;
14494 let generation = self.active_decommission_generation(source_pool_index).await?;
14495 let owner = self
14496 .decommission_capacity_owner_for_worker(source_pool_index, generation)
14497 .await?;
14498 let expected = set
14499 .load_file_info_versions_exact(RUSTFS_META_BUCKET, data_movement::scanner_backlog::SCANNER_PAUSE_BACKLOG_PATH)
14500 .await?
14501 .unwrap_or_default();
14502 match self
14503 .retire_scanner_pause_backlog_entry(CancellationToken::new(), source_pool_index, generation, set, expected, owner)
14504 .await?
14505 {
14506 DecommissionEntryAttemptOutcome::Complete => Ok(()),
14507 DecommissionEntryAttemptOutcome::SourceChanged => Err(Error::other("scanner retirement source changed")),
14508 }
14509 }
14510
14511 #[cfg(feature = "test-util")]
14512 pub async fn stage_scanner_pause_backlog_retirement_intent_for_test(
14513 &self,
14514 source_pool_index: usize,
14515 source_set_index: usize,
14516 ) -> Result<()> {
14517 let generation = self.active_decommission_generation(source_pool_index).await?;
14518 let owner = self
14519 .decommission_capacity_owner_for_worker(source_pool_index, generation)
14520 .await?
14521 .ok_or_else(|| Error::other("scanner retirement test has no capacity owner"))?;
14522 let versions = self.pools[source_pool_index].disk_set[source_set_index]
14523 .load_file_info_versions_exact(RUSTFS_META_BUCKET, data_movement::scanner_backlog::SCANNER_PAUSE_BACKLOG_PATH)
14524 .await?
14525 .ok_or_else(|| Error::other("scanner retirement test source is missing"))?;
14526 let version = versions
14527 .versions
14528 .first()
14529 .ok_or_else(|| Error::other("scanner retirement test source is empty"))?;
14530 let owner = owner.with_mutation_id(decommission_capacity_version_mutation_id(owner, RUSTFS_META_BUCKET, version));
14531 let size = usize::try_from(version.size).map_err(|_| Error::other("scanner retirement test source size is invalid"))?;
14532 let target = self.select_decommission_capacity_target_pool(owner, size).await?;
14533 let failed: Result<()> = self
14534 .run_decommission_capacity_admitted_mutation(target, Some(owner), Some(size), || async {
14535 Err(Error::other("injected scanner retirement target failure"))
14536 })
14537 .await;
14538 match failed {
14539 Err(err) if err.to_string().contains("injected scanner retirement target failure") => Ok(()),
14540 Err(err) => Err(err),
14541 Ok(()) => Err(Error::other("scanner retirement test did not retain its target intent")),
14542 }
14543 }
14544
14545 #[allow(clippy::too_many_arguments)]
14546 async fn retire_scanner_pause_backlog_entry(
14547 self: &Arc<Self>,
14548 rx: CancellationToken,
14549 idx: usize,
14550 generation: OffsetDateTime,
14551 set: Arc<SetDisks>,
14552 expected: FileInfoVersions,
14553 capacity_owner: Option<DecommissionCapacityOwner>,
14554 ) -> Result<DecommissionEntryAttemptOutcome> {
14555 let store = Arc::clone(self);
14556 tokio::spawn(async move {
14559 let operation_gate = store.ctx.data_movement_operation_gate();
14560 store
14561 .run_guarded_decommission_side_effect(&rx, &operation_gate, || {
14562 store.retire_scanner_pause_backlog_entry_inner(idx, generation, set, &expected, capacity_owner)
14563 })
14564 .await
14565 })
14566 .await
14567 .map_err(Error::from)?
14568 }
14569
14570 async fn retire_scanner_pause_backlog_entry_inner(
14571 &self,
14572 idx: usize,
14573 generation: OffsetDateTime,
14574 set: Arc<SetDisks>,
14575 expected: &FileInfoVersions,
14576 capacity_owner: Option<DecommissionCapacityOwner>,
14577 ) -> Result<DecommissionEntryAttemptOutcome> {
14578 use data_movement::scanner_backlog::{
14579 SCANNER_PAUSE_BACKLOG_PATH, persist_native_scanner_pause_backlog_replica, plan_scanner_pause_backlog_retirement,
14580 read_scanner_pause_backlog_retirement_replicas,
14581 };
14582
14583 if expected.versions.is_empty() && expected.free_versions.is_empty() {
14584 return Ok(DecommissionEntryAttemptOutcome::Complete);
14587 }
14588 let [version] = expected.versions.as_slice() else {
14589 return Err(Error::other("scanner pause backlog retirement requires exactly one source version"));
14590 };
14591 if version.version_id.is_some_and(|version| !version.is_nil())
14592 || version.deleted
14593 || version.tier_free_version()
14594 || version.is_remote()
14595 || !expected.free_versions.is_empty()
14596 {
14597 return Err(Error::other(
14598 "scanner pause backlog retirement requires an unversioned local source record",
14599 ));
14600 }
14601 let object_fence = self
14602 .acquire_decommission_source_cleanup_fence(RUSTFS_META_BUCKET, SCANNER_PAUSE_BACKLOG_PATH, set.as_ref())
14603 .await?;
14604 let save_guard = self.pool_meta_save_gate.lock().await;
14607 let (pool_meta_guard, snapshot) = self
14608 .acquire_pool_meta_read_guard(&save_guard, "scanner pause backlog retirement admission failed")
14609 .await?;
14610 ensure_decommission_generation(&snapshot, idx, generation)?;
14611 let owner = capacity_owner.ok_or_else(|| {
14612 decommission_capacity_blocked_error("scanner pause backlog retirement has no active capacity owner")
14613 })?;
14614 let reservation = snapshot.pools[idx]
14615 .decommission
14616 .as_ref()
14617 .and_then(|info| info.capacity_reservation.as_ref())
14618 .filter(|reservation| reservation.admits_owner(owner, OffsetDateTime::now_utc()))
14619 .ok_or_else(|| decommission_capacity_blocked_error("scanner pause backlog retirement reservation is stale"))?;
14620 let mutation_id = decommission_capacity_version_mutation_id(owner, RUSTFS_META_BUCKET, version);
14621 if reservation.targets.iter().any(|target| {
14622 target.pending_mutation_id == Some(mutation_id)
14623 || target
14624 .temporary_mutations
14625 .iter()
14626 .any(|mutation| mutation.mutation_id == mutation_id)
14627 }) {
14628 return Err(decommission_capacity_blocked_error(
14629 "scanner pause backlog retirement has an unresolved target capacity intent",
14630 ));
14631 }
14632 if snapshot.scanner_pause_backlog_pool_writable(idx) {
14633 return Err(Error::other("scanner pause backlog source still belongs to writable membership"));
14634 }
14635 let sets: Vec<_> = self
14636 .pools
14637 .iter()
14638 .enumerate()
14639 .filter(|(pool_index, _)| *pool_index == idx || snapshot.scanner_pause_backlog_pool_writable(*pool_index))
14640 .flat_map(|(_, pool)| pool.disk_set.iter().cloned())
14641 .collect();
14642 let mut replicas = read_scanner_pause_backlog_retirement_replicas(idx, set.set_index, sets.clone()).await?;
14643 if let Some(plan) = plan_scanner_pause_backlog_retirement(idx, &replicas)? {
14644 let expected_stable_record = plan.stable_record.clone();
14645 let mut phases = Vec::with_capacity(3);
14646 if let Some(record) = plan.seed_record {
14647 phases.push(("seed", record));
14648 }
14649 phases.push(("commit", plan.commit_record));
14650 phases.push(("stabilize", plan.stable_record));
14651 for (phase, record) in phases {
14652 if object_fence.is_lock_lost()
14653 || pool_meta_guard.is_lock_lost()
14654 || !reservation.admits_owner(owner, OffsetDateTime::now_utc())
14655 {
14656 return Err(decommission_capacity_blocked_error(
14657 "scanner pause backlog retirement fence expired during native repair",
14658 ));
14659 }
14660 for read in replicas.iter().filter(|read| read.replica.pool_index != idx) {
14661 ensure_external_decommission_target_admission(
14662 &snapshot,
14663 read.replica.pool_index,
14664 DecommissionCapacityAdmission::ScannerBacklog,
14665 )?;
14666 }
14667 let results = join_all(replicas.iter().filter(|read| read.replica.pool_index != idx).map(|read| {
14668 let target = Arc::clone(&self.pools[read.replica.pool_index].disk_set[read.replica.set_index]);
14669 let record = record.clone();
14670 let object_fence = &object_fence;
14671 let pool_meta_guard = &pool_meta_guard;
14672 async move {
14673 let mut opts = ObjectOptions {
14674 no_lock: self.pools[0].disk_set[0].shares_namespace_lock_domain(&target).await,
14675 ..Default::default()
14676 };
14677 object_fence.add_namespace_lock_fence(&mut opts);
14678 opts.add_namespace_lock_guard(pool_meta_guard);
14679 persist_native_scanner_pause_backlog_replica(target, record, read.preconditions(), opts, phase).await
14680 }
14681 }))
14682 .await;
14683 for result in results {
14686 result?;
14687 }
14688 replicas = read_scanner_pause_backlog_retirement_replicas(idx, set.set_index, sets.clone()).await?;
14689 if replicas
14690 .iter()
14691 .filter(|read| read.replica.pool_index != idx)
14692 .any(|read| read.replica.data.as_deref() != Some(record.as_slice()))
14693 {
14694 return Err(Error::other(
14695 "scanner pause backlog native repair did not persist every surviving replica",
14696 ));
14697 }
14698 if let Some(next) = plan_scanner_pause_backlog_retirement(idx, &replicas)?
14699 && next.stable_record != expected_stable_record
14700 {
14701 return Err(Error::other("scanner pause backlog native authority changed during repair"));
14702 }
14703 }
14704 if plan_scanner_pause_backlog_retirement(idx, &replicas)?.is_some() {
14705 return Err(Error::other("scanner pause backlog native repair is not fully committed and stable"));
14706 }
14707 }
14708 if object_fence.is_lock_lost()
14709 || pool_meta_guard.is_lock_lost()
14710 || !reservation.admits_owner(owner, OffsetDateTime::now_utc())
14711 {
14712 return Err(decommission_capacity_blocked_error(
14713 "scanner pause backlog retirement fence expired before cleanup",
14714 ));
14715 }
14716 let result = data_movement::cleanup_source_entry_if_unchanged(
14717 set,
14718 RUSTFS_META_BUCKET,
14719 SCANNER_PAUSE_BACKLOG_PATH,
14720 expected,
14721 &[],
14722 data_movement::SourceCleanupBucketFence {
14723 expected_incarnation_id: None,
14724 lifecycle_guard: None,
14725 namespace_lock_lost_signal: pool_meta_guard.lock_lost_signal(),
14726 object_mutation_fence: Some(&object_fence),
14727 },
14728 "scanner pause backlog retirement",
14729 )
14730 .await;
14731 match result {
14732 Ok(_) => Ok(DecommissionEntryAttemptOutcome::Complete),
14733 Err(data_movement::SourceCleanupError::SourceChanged) => Ok(DecommissionEntryAttemptOutcome::SourceChanged),
14734 Err(data_movement::SourceCleanupError::Storage(err)) => Err(err),
14735 }
14736 }
14737
14738 #[allow(clippy::too_many_arguments)]
14739 #[tracing::instrument(
14740 level = "trace",
14741 skip_all,
14742 fields(
14743 event = EVENT_DECOMMISSION_ENTRY,
14744 component = LOG_COMPONENT_ECSTORE,
14745 subsystem = LOG_SUBSYSTEM_POOLS,
14746 state = "processing",
14747 pool_index = idx,
14748 bucket = %bucket,
14749 object = %entry.name,
14750 generation = %generation,
14751 )
14752 )]
14753 async fn decommission_entry(
14754 self: &Arc<Self>,
14755 rx: CancellationToken,
14756 idx: usize,
14757 generation: OffsetDateTime,
14758 entry: MetaCacheEntry,
14759 bucket: String,
14760 set: Arc<SetDisks>,
14761 lifecycle_config: Option<BucketLifecycleConfiguration>,
14762 object_lock_config: Option<ObjectLockConfiguration>,
14763 replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
14764 expected_bucket_incarnation_id: Option<uuid::Uuid>,
14765 source_changed_exhaustions: Arc<AtomicUsize>,
14766 attempts: &mut usize,
14767 ) -> Result<DecommissionEntryOutcome> {
14768 let mut counted_versions = HashSet::new();
14769
14770 for entry_attempt in 1..=DECOMMISSION_ENTRY_MAX_ATTEMPTS {
14771 let attempt_result = {
14772 let mut conflict_attempt = 0;
14773 let mut scanner_backlog_handoff_attempt = 0;
14774 loop {
14775 let result = self
14776 .decommission_entry_attempt(
14777 rx.clone(),
14778 idx,
14779 generation,
14780 entry.clone(),
14781 bucket.clone(),
14782 Arc::clone(&set),
14783 lifecycle_config.clone(),
14784 object_lock_config.clone(),
14785 replication_config.clone(),
14786 expected_bucket_incarnation_id,
14787 entry_attempt,
14788 scanner_backlog_handoff_attempt,
14789 source_changed_exhaustions.as_ref(),
14790 &mut counted_versions,
14791 attempts,
14792 )
14793 .await;
14794 let retry = result
14795 .as_ref()
14796 .err()
14797 .and_then(|err| decommission_capacity_retry_kind(err, conflict_attempt));
14798 if let Some(DecommissionCapacityRetryKind::IntentConflict) = retry {
14799 conflict_attempt += 1;
14800 let retry_delay =
14801 decommission_retry_backoff_delay(DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, conflict_attempt);
14802 if wait_decommission_retry_backoff(&rx, retry_delay).await {
14803 decommission_cancel_signal_result(rx.is_cancelled())?;
14804 }
14805 continue;
14806 }
14807 let scanner_retry = result
14812 .as_ref()
14813 .err()
14814 .and_then(|err| decommission_scanner_backlog_retry_kind(err, scanner_backlog_handoff_attempt));
14815 if let Some(DecommissionScannerBacklogRetryKind::HandoffConflict) = scanner_retry {
14816 scanner_backlog_handoff_attempt += 1;
14817 let retry_delay = decommission_retry_backoff_delay(
14818 DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY,
14819 scanner_backlog_handoff_attempt,
14820 );
14821 info!(
14822 event = EVENT_DECOMMISSION_ENTRY,
14823 component = LOG_COMPONENT_ECSTORE,
14824 subsystem = LOG_SUBSYSTEM_POOLS,
14825 state = "scanner_backlog_handoff_retry",
14826 pool_index = idx,
14827 bucket = %bucket,
14828 object = %entry.name,
14829 attempt = scanner_backlog_handoff_attempt,
14830 max_attempts = DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS,
14831 retry_delay_ms = retry_delay.as_millis(),
14832 "Decommission scanner pause backlog handoff conflict; retrying entry"
14833 );
14834 if wait_decommission_retry_backoff(&rx, retry_delay).await {
14835 decommission_cancel_signal_result(rx.is_cancelled())?;
14836 }
14837 continue;
14838 }
14839 break result;
14840 }
14841 };
14842 match attempt_result {
14843 Ok(DecommissionEntryAttemptOutcome::Complete) => return Ok(DecommissionEntryOutcome::Complete),
14844 Ok(DecommissionEntryAttemptOutcome::SourceChanged) => {
14845 let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, entry_attempt);
14846 warn!(
14847 event = EVENT_DECOMMISSION_ENTRY,
14848 component = LOG_COMPONENT_ECSTORE,
14849 subsystem = LOG_SUBSYSTEM_POOLS,
14850 state = "source_changed_retry",
14851 pool_index = idx,
14852 bucket = %bucket,
14853 object = %entry.name,
14854 attempt = entry_attempt,
14855 max_attempts = DECOMMISSION_ENTRY_MAX_ATTEMPTS,
14856 retry_delay_ms = retry_delay.as_millis(),
14857 "Decommission source changed during cleanup preflight; retrying entry"
14858 );
14859 if wait_decommission_retry_backoff(&rx, retry_delay).await {
14860 decommission_cancel_signal_result(rx.is_cancelled())?;
14861 }
14862 }
14863 Err(err) => {
14864 return match resolve_decommission_entry_failure(err) {
14865 Ok(DecommissionEntryOutcome::Deferred(err)) => {
14866 warn!(
14871 event = EVENT_DECOMMISSION_ENTRY,
14872 component = LOG_COMPONENT_ECSTORE,
14873 subsystem = LOG_SUBSYSTEM_POOLS,
14874 state = "entry_deferred",
14875 pool_index = idx,
14876 bucket = %bucket,
14877 object = %entry.name,
14878 attempts = DECOMMISSION_ENTRY_MAX_ATTEMPTS,
14879 error = %err,
14880 "Decommission entry deferred after exhausting its attempts on target capacity contention"
14881 );
14882 Ok(DecommissionEntryOutcome::Deferred(err))
14883 }
14884 Ok(DecommissionEntryOutcome::Complete) => {
14885 warn!(
14886 event = EVENT_DECOMMISSION_ENTRY,
14887 component = LOG_COMPONENT_ECSTORE,
14888 subsystem = LOG_SUBSYSTEM_POOLS,
14889 state = "entry_ignored",
14890 pool_index = idx,
14891 bucket = %bucket,
14892 object = %entry.name,
14893 "Decommission entry ignored because its mutation is no longer needed"
14894 );
14895 Ok(DecommissionEntryOutcome::Complete)
14896 }
14897 Err(err) => Err(err),
14898 };
14899 }
14900 }
14901 }
14902
14903 Err(Error::other(format!(
14904 "decommission entry retry loop ended without a terminal result for {bucket}/{}",
14905 entry.name
14906 )))
14907 }
14908
14909 #[allow(unused_assignments, clippy::too_many_arguments)]
14910 async fn decommission_entry_attempt(
14911 self: &Arc<Self>,
14912 rx: CancellationToken,
14913 idx: usize,
14914 generation: OffsetDateTime,
14915 entry: MetaCacheEntry,
14916 bucket: String,
14917 set: Arc<SetDisks>,
14918 lifecycle_config: Option<BucketLifecycleConfiguration>,
14919 object_lock_config: Option<ObjectLockConfiguration>,
14920 replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
14921 expected_bucket_incarnation_id: Option<uuid::Uuid>,
14922 entry_attempt: usize,
14923 scanner_backlog_handoff_attempt: usize,
14924 source_changed_exhaustions: &AtomicUsize,
14925 counted_versions: &mut HashSet<(Option<uuid::Uuid>, bool)>,
14926 attempts: &mut usize,
14927 ) -> Result<DecommissionEntryAttemptOutcome> {
14928 record_decommission_object_attempt(attempts);
14929 debug!(
14930 event = EVENT_DECOMMISSION_ENTRY,
14931 component = LOG_COMPONENT_ECSTORE,
14932 subsystem = LOG_SUBSYSTEM_POOLS,
14933 state = "started",
14934 pool_index = idx,
14935 bucket = %bucket,
14936 object = %entry.name,
14937 attempt = entry_attempt,
14938 max_attempts = DECOMMISSION_ENTRY_MAX_ATTEMPTS,
14939 "Decommission entry started"
14940 );
14941 if is_decommission_set_local_usage_cache(&bucket, &entry.name) {
14945 return Ok(DecommissionEntryAttemptOutcome::Complete);
14946 }
14947 let durable_ilm_record = if bucket == RUSTFS_META_BUCKET {
14948 classify_durable_ilm_record(&entry.name)
14949 .map_err(|err| with_decommission_entry_context("durable_ilm_namespace", &bucket, &entry.name, err))?
14950 } else {
14951 None
14952 };
14953 if self.decommission_cancel_requested(idx, &rx).await {
14954 rx.cancel();
14955 }
14956 decommission_cancel_signal_result(rx.is_cancelled())?;
14957 self.ensure_decommission_generation_current(idx, generation).await?;
14958 self.ensure_decommission_runtime_capacity_available(idx, generation).await?;
14959 let capacity_owner = self.decommission_capacity_owner_for_worker(idx, generation).await?;
14960 let operation_gate = self.ctx.data_movement_operation_gate();
14961
14962 let bucket_incarnation_fence = match expected_bucket_incarnation_id {
14963 Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
14964 None => None,
14965 };
14966
14967 let mut fivs = load_decommission_entry_exact_versions(&set, &entry, &bucket, "file_info_versions").await?;
14968
14969 if data_movement::scanner_backlog::is_scanner_pause_backlog(&bucket, &entry.name) {
14970 let outcome = self
14971 .retire_scanner_pause_backlog_entry(rx, idx, generation, Arc::clone(&set), fivs.clone(), capacity_owner)
14972 .await;
14973 return match outcome {
14974 Ok(DecommissionEntryAttemptOutcome::Complete) => {
14975 let mut pool_meta = self.pool_meta.write().await;
14976 ensure_decommission_generation(&pool_meta, idx, generation)?;
14977 if let Some(version) = fivs.versions.first()
14978 && counted_versions.insert((version.version_id, false))
14979 {
14980 count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), false)?;
14981 }
14982 track_decommission_current_object(&mut pool_meta, idx, &bucket, &entry.name)?;
14983 drop(pool_meta);
14984 self.track_decommission_entry_progress_stage(
14985 idx,
14986 generation,
14987 &bucket,
14988 &entry.name,
14989 DECOMMISSION_STAGE_ENTRY_FINISHED,
14990 )
14991 .await?;
14992 Ok(DecommissionEntryAttemptOutcome::Complete)
14993 }
14994 Ok(outcome) => Ok(outcome),
14995 Err(err) => {
14996 let retryable_handoff = data_movement::data_movement_stage_source_as::<
15002 data_movement::scanner_backlog::ScannerPauseBacklogRetirementError,
15003 >(&err)
15004 .is_some_and(
15005 data_movement::scanner_backlog::ScannerPauseBacklogRetirementError::is_retryable_handoff_conflict,
15006 );
15007 let terminal = !retryable_handoff
15008 || scanner_backlog_handoff_attempt >= DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS;
15009 if terminal {
15010 let mut pool_meta = self.pool_meta.write().await;
15011 ensure_decommission_generation(&pool_meta, idx, generation)?;
15012 if let Some(version) = fivs.versions.first()
15013 && counted_versions.insert((version.version_id, true))
15014 {
15015 count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), true)?;
15016 }
15017 track_decommission_current_object(&mut pool_meta, idx, &bucket, &entry.name)?;
15018 }
15019 Err(err)
15020 }
15021 };
15022 }
15023
15024 let pending_mutations = if let Some(owner) = capacity_owner {
15025 self.pool_meta
15026 .read()
15027 .await
15028 .pools
15029 .get(owner.source_pool_index)
15030 .and_then(|pool| pool.decommission.as_ref())
15031 .and_then(|info| info.capacity_reservation.as_ref())
15032 .filter(|reservation| reservation.admits_cleanup_owner(owner))
15033 .map(|reservation| {
15034 reservation
15035 .targets
15036 .iter()
15037 .filter_map(|target| target.pending_mutation_id)
15038 .collect::<HashSet<_>>()
15039 })
15040 .unwrap_or_default()
15041 } else {
15042 HashSet::new()
15043 };
15044 fivs.versions.sort_by_key(|version| {
15045 let mutation_id = capacity_owner.map(|owner| decommission_capacity_version_mutation_id(owner, &bucket, version));
15046 (
15047 mutation_id.is_none_or(|mutation_id| !pending_mutations.contains(&mutation_id)),
15048 version.mod_time.is_none(),
15049 std::cmp::Reverse(version.mod_time),
15050 )
15051 });
15052
15053 let mut decommissioned: usize = 0;
15054 let mut expired: usize = 0;
15055 let mut free_version_disposition = DecommissionFreeVersionDisposition::default();
15056 let mut cleanup_preflight_allowed_missing = Vec::new();
15057 let mut entry_blocked = false;
15058
15059 for version in fivs.versions.iter() {
15060 if self.decommission_cancel_requested(idx, &rx).await {
15061 rx.cancel();
15062 }
15063 decommission_cancel_signal_result(rx.is_cancelled())?;
15064
15065 if version.tier_free_version() {
15066 let version_id = version.version_id.map(|v| v.to_string());
15067 let mut migration_error = None;
15068 let mut migrated = false;
15069 let mut consumed = false;
15070 let mut capacity_failure = false;
15071 let mut version_attempt = 1;
15072 let mut target_busy_attempt: usize = 0;
15073 let mut target_permit = None;
15074 while version_attempt <= DECOMMISSION_VERSION_COPY_ATTEMPTS {
15075 if version_attempt > 1 || target_busy_attempt > 0 {
15076 record_decommission_object_attempt(attempts);
15077 }
15078 let result = self
15079 .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
15080 self.decommission_tiered_object(
15081 bucket.as_str(),
15082 &version.name,
15083 version,
15084 &decommission_capacity_owned_opts(
15085 decommission_remote_tiered_opts(
15086 version,
15087 version_id.clone(),
15088 idx,
15089 expected_bucket_incarnation_id,
15090 ),
15091 capacity_owner,
15092 ),
15093 )
15094 .await
15095 })
15096 .await;
15097 drop(target_permit.take());
15098 if let Some(target_pool_index) = result.as_ref().err().and_then(decommission_capacity_target_gate_busy_index)
15099 {
15100 target_busy_attempt = target_busy_attempt.saturating_add(1);
15101 let Some(permit) = self
15102 .wait_decommission_target_gate_retry(
15103 &rx,
15104 idx,
15105 generation,
15106 target_pool_index,
15107 capacity_owner,
15108 set.as_ref(),
15109 &entry,
15110 &bucket,
15111 version,
15112 target_busy_attempt,
15113 )
15114 .await?
15115 else {
15116 return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
15117 };
15118 target_permit = Some(permit);
15119 continue;
15120 }
15121
15122 match classify_decommission_free_version_attempt(result) {
15123 DecommissionFreeVersionAttempt::Migrated => {
15124 migrated = true;
15125 migration_error = None;
15126 break;
15127 }
15128 DecommissionFreeVersionAttempt::Consumed => {
15129 consumed = true;
15130 migration_error = None;
15131 break;
15132 }
15133 DecommissionFreeVersionAttempt::CapacityFailure(err) => {
15134 capacity_failure = true;
15135 migration_error = Some(err);
15136 break;
15137 }
15138 DecommissionFreeVersionAttempt::Retry(err) => {
15139 migration_error = Some(err);
15140 version_attempt += 1;
15141 }
15142 }
15143 }
15144
15145 if counted_versions.insert((version.version_id, version.deleted)) {
15146 let mut pool_meta = self.pool_meta.write().await;
15147 ensure_decommission_generation(&pool_meta, idx, generation)?;
15148 if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, !migrated && !consumed) {
15149 return Err(with_decommission_entry_context(
15150 "count_decommission_item",
15151 bucket.as_str(),
15152 entry.name.as_str(),
15153 err,
15154 ));
15155 }
15156 }
15157
15158 if migrated || consumed {
15159 decommissioned += 1;
15160 cleanup_preflight_allowed_missing.push(data_movement::source_cleanup_version_identity(version));
15161 }
15162 if migrated {
15163 free_version_disposition.record_migrated();
15164 } else if consumed {
15165 free_version_disposition.record_consumed();
15166 } else {
15167 free_version_disposition.record_retained();
15168 }
15169
15170 debug!(
15171 event = EVENT_DECOMMISSION_ENTRY,
15172 component = LOG_COMPONENT_ECSTORE,
15173 subsystem = LOG_SUBSYSTEM_POOLS,
15174 pool_index = idx,
15175 bucket = %bucket,
15176 object = %version.name,
15177 version_id = ?version_id,
15178 result = ?migration_error,
15179 reason = if migrated {
15180 DECOMMISSION_FREE_VERSION_MIGRATED_REASON
15181 } else if consumed {
15182 DECOMMISSION_FREE_VERSION_CONSUMED_REASON
15183 } else {
15184 DECOMMISSION_FREE_VERSION_RETAINED_REASON
15185 },
15186 state = if migrated {
15187 "free_version_migrated"
15188 } else if consumed {
15189 "free_version_consumed"
15190 } else {
15191 "free_version_retained"
15192 },
15193 "Decommission free-version disposition recorded"
15194 );
15195
15196 if capacity_failure {
15197 return Err(with_decommission_entry_context(
15198 "decommission_tier_free_version",
15199 bucket.as_str(),
15200 version.name.as_str(),
15201 migration_error.expect("capacity failure must retain its error"),
15202 ));
15203 }
15204
15205 if !migrated && !consumed {
15206 break;
15207 }
15208 continue;
15209 }
15210
15211 if self
15212 .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
15213 should_skip_lifecycle_for_data_movement(
15214 self.clone(),
15215 &bucket,
15216 version,
15217 lifecycle_config.as_ref(),
15218 object_lock_config.as_ref(),
15219 true,
15220 &LcEventSrc::Decom,
15221 None,
15222 )
15223 .await
15224 })
15225 .await
15226 .map_err(|err| with_decommission_entry_context("lifecycle_expiry", bucket.as_str(), version.name.as_str(), err))?
15227 {
15228 expired += 1;
15229 cleanup_preflight_allowed_missing.push(data_movement::source_cleanup_version_identity(version));
15230 continue;
15231 }
15232
15233 let remaining_versions = decommission_remaining_version_count(&fivs.versions, expired);
15234 if should_skip_decommission_delete_marker(version, remaining_versions, replication_config.is_some()) {
15235 decommissioned += 1;
15237 debug!(
15238 event = EVENT_DECOMMISSION_ENTRY,
15239 component = LOG_COMPONENT_ECSTORE,
15240 subsystem = LOG_SUBSYSTEM_POOLS,
15241 pool_index = idx,
15242 bucket = %bucket,
15243 object = %version.name,
15244 state = "skipped_delete_marker",
15245 "Decommission delete marker skipped"
15246 );
15247 continue;
15248 }
15249
15250 let version_id = version.version_id.map(|v| v.to_string());
15251
15252 let mut ignore = false;
15253 let mut cleanup_ignored = false;
15254 let mut failure = false;
15255 let mut error = None;
15256 if version.deleted {
15257 let mut version_attempt = 1;
15258 let mut target_busy_attempt: usize = 0;
15259 let mut target_permit = None;
15260 while version_attempt <= DECOMMISSION_VERSION_COPY_ATTEMPTS {
15261 if version_attempt > 1 || target_busy_attempt > 0 {
15262 record_decommission_object_attempt(attempts);
15263 }
15264 let result = self
15265 .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
15266 self.delete_object(
15267 bucket.as_str(),
15268 &version.name,
15269 decommission_capacity_owned_opts(
15270 decommission_delete_marker_opts(
15271 version,
15272 version_id.clone(),
15273 idx,
15274 expected_bucket_incarnation_id,
15275 ),
15276 capacity_owner,
15277 ),
15278 )
15279 .await
15280 })
15281 .await;
15282 drop(target_permit.take());
15283 if let Some(target_pool_index) = result.as_ref().err().and_then(decommission_capacity_target_gate_busy_index)
15284 {
15285 target_busy_attempt = target_busy_attempt.saturating_add(1);
15286 let Some(permit) = self
15287 .wait_decommission_target_gate_retry(
15288 &rx,
15289 idx,
15290 generation,
15291 target_pool_index,
15292 capacity_owner,
15293 set.as_ref(),
15294 &entry,
15295 &bucket,
15296 version,
15297 target_busy_attempt,
15298 )
15299 .await?
15300 else {
15301 return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
15302 };
15303 target_permit = Some(permit);
15304 continue;
15305 }
15306 #[cfg(test)]
15307 let result = decommission_test_wrap_result(
15308 "delete_marker_copy",
15309 bucket.as_str(),
15310 version.name.as_str(),
15311 version_attempt,
15312 result,
15313 );
15314
15315 match result {
15316 Ok(_) => {
15317 failure = false;
15318 error = None;
15319 break;
15320 }
15321 Err(err) if is_decommission_copy_cleanup_safe_error(&err) => {
15322 warn!(
15323 event = EVENT_DECOMMISSION_ENTRY,
15324 component = LOG_COMPONENT_ECSTORE,
15325 subsystem = LOG_SUBSYSTEM_POOLS,
15326 state = "ignored_delete_marker_copy",
15327 pool_index = idx,
15328 bucket = %bucket,
15329 object = %version.name,
15330 version_id = ?version_id,
15331 error = ?err,
15332 "Decommission delete marker copy ignored"
15333 );
15334 ignore = true;
15335 cleanup_ignored = true;
15336 break;
15337 }
15338 Err(err) if is_decommission_target_capacity_error(&err) => {
15339 return Err(with_decommission_entry_context(
15340 "delete_marker_copy",
15341 bucket.as_str(),
15342 version.name.as_str(),
15343 err,
15344 ));
15345 }
15346 Err(err) => {
15347 failure = true;
15348 if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS {
15349 error!(
15350 event = EVENT_DECOMMISSION_ENTRY,
15351 component = LOG_COMPONENT_ECSTORE,
15352 subsystem = LOG_SUBSYSTEM_POOLS,
15353 state = "delete_marker_copy_failed",
15354 pool_index = idx,
15355 bucket = %bucket,
15356 object = %version.name,
15357 version_id = ?version_id,
15358 attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
15359 error = ?err,
15360 "Decommission delete marker copy failed"
15361 );
15362 error = Some(err);
15363 break;
15364 }
15365
15366 let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt);
15367 warn!(
15368 event = EVENT_DECOMMISSION_ENTRY,
15369 component = LOG_COMPONENT_ECSTORE,
15370 subsystem = LOG_SUBSYSTEM_POOLS,
15371 state = "delete_marker_copy_retry",
15372 pool_index = idx,
15373 bucket = %bucket,
15374 object = %version.name,
15375 version_id = ?version_id,
15376 attempt = version_attempt,
15377 max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
15378 retry_delay_ms = retry_delay.as_millis(),
15379 error = ?err,
15380 "Decommission delete marker copy failed; retrying"
15381 );
15382 error = Some(err);
15383 if wait_decommission_retry_backoff(&rx, retry_delay).await {
15384 decommission_cancel_signal_result(rx.is_cancelled())?;
15385 }
15386 version_attempt += 1;
15387 }
15388 }
15389 }
15390
15391 if ignore {
15392 if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
15393 decommissioned += 1;
15394 }
15395 debug!(
15396 event = EVENT_DECOMMISSION_ENTRY,
15397 component = LOG_COMPONENT_ECSTORE,
15398 subsystem = LOG_SUBSYSTEM_POOLS,
15399 pool_index = idx,
15400 bucket = %bucket,
15401 object = %version.name,
15402 state = "ignored",
15403 "Decommission entry ignored"
15404 );
15405 continue;
15406 }
15407
15408 if counted_versions.insert((version.version_id, version.deleted)) {
15409 let mut pool_meta = self.pool_meta.write().await;
15410 ensure_decommission_generation(&pool_meta, idx, generation)?;
15411 if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, failure) {
15412 return Err(with_decommission_entry_context(
15413 "count_decommission_item",
15414 bucket.as_str(),
15415 entry.name.as_str(),
15416 err,
15417 ));
15418 }
15419 }
15420
15421 if !failure {
15422 decommissioned += 1;
15423 }
15424
15425 if !failure {
15426 debug!(
15427 event = EVENT_DECOMMISSION_ENTRY,
15428 component = LOG_COMPONENT_ECSTORE,
15429 subsystem = LOG_SUBSYSTEM_POOLS,
15430 state = "delete_marker_copied",
15431 pool_index = idx,
15432 bucket = %bucket,
15433 object = %version.name,
15434 version_id = ?version_id,
15435 result = ?error,
15436 "Decommission delete marker copied"
15437 );
15438 }
15439 continue;
15440 }
15441
15442 let mut version_attempt = 1;
15443 let mut target_busy_attempt: usize = 0;
15444 let mut target_permit = None;
15445 while version_attempt <= DECOMMISSION_VERSION_COPY_ATTEMPTS {
15446 if version_attempt > 1 || target_busy_attempt > 0 {
15447 record_decommission_object_attempt(attempts);
15448 }
15449 if version.is_remote() {
15450 let result = self
15451 .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
15452 self.decommission_tiered_object(
15453 bucket.as_str(),
15454 &version.name,
15455 version,
15456 &decommission_capacity_owned_opts(
15457 decommission_remote_tiered_opts(
15458 version,
15459 version_id.clone(),
15460 idx,
15461 expected_bucket_incarnation_id,
15462 ),
15463 capacity_owner,
15464 ),
15465 )
15466 .await
15467 })
15468 .await;
15469 drop(target_permit.take());
15470 if let Some(target_pool_index) = result.as_ref().err().and_then(decommission_capacity_target_gate_busy_index)
15471 {
15472 target_busy_attempt = target_busy_attempt.saturating_add(1);
15473 let Some(permit) = self
15474 .wait_decommission_target_gate_retry(
15475 &rx,
15476 idx,
15477 generation,
15478 target_pool_index,
15479 capacity_owner,
15480 set.as_ref(),
15481 &entry,
15482 &bucket,
15483 version,
15484 target_busy_attempt,
15485 )
15486 .await?
15487 else {
15488 return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
15489 };
15490 target_permit = Some(permit);
15491 continue;
15492 }
15493 #[cfg(test)]
15494 let result = decommission_test_wrap_result(
15495 "decommission_tiered_object",
15496 bucket.as_str(),
15497 version.name.as_str(),
15498 version_attempt,
15499 result,
15500 );
15501
15502 match result {
15503 Ok(_) => {
15504 failure = false;
15505 error = None;
15506 }
15507 Err(err) if is_decommission_copy_cleanup_safe_error(&err) => {
15508 ignore = true;
15509 cleanup_ignored = true;
15510 }
15511 Err(err) if is_decommission_target_capacity_error(&err) => {
15512 return Err(with_decommission_entry_context(
15513 "decommission_tiered_object",
15514 bucket.as_str(),
15515 version.name.as_str(),
15516 err,
15517 ));
15518 }
15519 Err(err) => {
15520 failure = true;
15521 if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS {
15522 error!(
15523 event = EVENT_DECOMMISSION_ENTRY,
15524 component = LOG_COMPONENT_ECSTORE,
15525 subsystem = LOG_SUBSYSTEM_POOLS,
15526 state = "tiered_copy_failed",
15527 pool_index = idx,
15528 bucket = %bucket,
15529 object = %version.name,
15530 version_id = ?version_id,
15531 attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
15532 error = ?err,
15533 "Decommission tiered version copy failed"
15534 );
15535 error = Some(err);
15536 } else {
15537 let retry_delay =
15538 decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt);
15539 warn!(
15540 event = EVENT_DECOMMISSION_ENTRY,
15541 component = LOG_COMPONENT_ECSTORE,
15542 subsystem = LOG_SUBSYSTEM_POOLS,
15543 state = "tiered_copy_retry",
15544 pool_index = idx,
15545 bucket = %bucket,
15546 object = %version.name,
15547 version_id = ?version_id,
15548 attempt = version_attempt,
15549 max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
15550 retry_delay_ms = retry_delay.as_millis(),
15551 error = ?err,
15552 "Decommission tiered version copy failed; retrying"
15553 );
15554 error = Some(err);
15555 if wait_decommission_retry_backoff(&rx, retry_delay).await {
15556 decommission_cancel_signal_result(rx.is_cancelled())?;
15557 }
15558 version_attempt += 1;
15559 continue;
15560 }
15561 }
15562 }
15563 break;
15564 }
15565
15566 let bucket = bucket.clone();
15567
15568 let read_result = set
15569 .get_object_reader(
15570 bucket.as_str(),
15571 &encode_dir_object(&version.name),
15572 None,
15573 HeaderMap::new(),
15574 &decommission_object_migration_read_opts(version_id.clone()),
15575 )
15576 .await;
15577 #[cfg(test)]
15578 let read_result =
15579 decommission_test_wrap_result("object_read", &bucket, &version.name, version_attempt, read_result);
15580 let rd = match read_result {
15581 Ok(rd) => rd,
15582 Err(err) => {
15583 if is_err_object_not_found(&err) || is_err_version_not_found(&err) {
15584 ignore = true;
15585 cleanup_ignored = true;
15586 break;
15587 }
15588
15589 failure = true;
15590 if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS {
15591 error!(
15592 event = EVENT_DECOMMISSION_ENTRY,
15593 component = LOG_COMPONENT_ECSTORE,
15594 subsystem = LOG_SUBSYSTEM_POOLS,
15595 state = "object_read_failed",
15596 pool_index = idx,
15597 bucket = %bucket,
15598 object = %version.name,
15599 version_id = ?version_id,
15600 attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
15601 error = ?err,
15602 "Decommission source object read failed"
15603 );
15604 error = Some(err);
15605 break;
15606 }
15607
15608 let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt);
15609 warn!(
15610 event = EVENT_DECOMMISSION_ENTRY,
15611 component = LOG_COMPONENT_ECSTORE,
15612 subsystem = LOG_SUBSYSTEM_POOLS,
15613 state = "object_read_retry",
15614 pool_index = idx,
15615 bucket = %bucket,
15616 object = %version.name,
15617 version_id = ?version_id,
15618 attempt = version_attempt,
15619 max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
15620 retry_delay_ms = retry_delay.as_millis(),
15621 error = ?err,
15622 "Decommission source object read failed; retrying"
15623 );
15624 error = Some(err);
15625 if wait_decommission_retry_backoff(&rx, retry_delay).await {
15626 decommission_cancel_signal_result(rx.is_cancelled())?;
15627 }
15628 version_attempt += 1;
15629 continue;
15630 }
15631 };
15632
15633 let bucket_name = bucket.clone();
15634 let object_name = rd.object_info.name.clone();
15635
15636 self.track_decommission_entry_progress_stage(
15637 idx,
15638 generation,
15639 bucket_name.as_str(),
15640 object_name.as_str(),
15641 DECOMMISSION_STAGE_MIGRATE_OBJECT,
15642 )
15643 .await?;
15644
15645 let migrate_result = self
15646 .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
15647 self.clone()
15648 .decommission_object(idx, bucket, rd, expected_bucket_incarnation_id, capacity_owner)
15649 .await
15650 })
15651 .await;
15652 drop(target_permit.take());
15653 if let Some(target_pool_index) = migrate_result
15654 .as_ref()
15655 .err()
15656 .and_then(decommission_capacity_target_gate_busy_index)
15657 {
15658 target_busy_attempt = target_busy_attempt.saturating_add(1);
15659 let Some(permit) = self
15660 .wait_decommission_target_gate_retry(
15661 &rx,
15662 idx,
15663 generation,
15664 target_pool_index,
15665 capacity_owner,
15666 set.as_ref(),
15667 &entry,
15668 &bucket_name,
15669 version,
15670 target_busy_attempt,
15671 )
15672 .await?
15673 else {
15674 return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
15675 };
15676 target_permit = Some(permit);
15677 continue;
15678 }
15679 #[cfg(test)]
15680 let migrate_result = decommission_test_wrap_result(
15681 DECOMMISSION_STAGE_MIGRATE_OBJECT,
15682 bucket_name.as_str(),
15683 object_name.as_str(),
15684 version_attempt,
15685 migrate_result,
15686 );
15687
15688 if let Err(err) = migrate_result {
15689 if is_decommission_copy_cleanup_safe_error(&err) {
15690 ignore = true;
15691 cleanup_ignored = true;
15692 break;
15693 }
15694
15695 if is_decommission_target_capacity_error(&err) {
15696 return Err(with_decommission_entry_context(
15697 DECOMMISSION_STAGE_MIGRATE_OBJECT,
15698 bucket_name.as_str(),
15699 object_name.as_str(),
15700 err,
15701 ));
15702 }
15703
15704 failure = true;
15705 if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS {
15706 error!(
15707 event = EVENT_DECOMMISSION_ENTRY,
15708 component = LOG_COMPONENT_ECSTORE,
15709 subsystem = LOG_SUBSYSTEM_POOLS,
15710 state = "object_migration_failed",
15711 pool_index = idx,
15712 bucket = %bucket_name,
15713 object = %object_name,
15714 version = %version.name,
15715 attempt = version_attempt,
15716 max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
15717 error = ?err,
15718 "Decommission object migration failed"
15719 );
15720 error = Some(err);
15721 break;
15722 }
15723
15724 let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt);
15725 warn!(
15726 event = EVENT_DECOMMISSION_ENTRY,
15727 component = LOG_COMPONENT_ECSTORE,
15728 subsystem = LOG_SUBSYSTEM_POOLS,
15729 state = "object_migration_retry",
15730 pool_index = idx,
15731 bucket = %bucket_name,
15732 object = %object_name,
15733 version = %version.name,
15734 attempt = version_attempt,
15735 max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
15736 retry_delay_ms = retry_delay.as_millis(),
15737 error = ?err,
15738 "Decommission object migration failed; retrying"
15739 );
15740 error = Some(err);
15741 if wait_decommission_retry_backoff(&rx, retry_delay).await {
15742 decommission_cancel_signal_result(rx.is_cancelled())?;
15743 }
15744 version_attempt += 1;
15745 continue;
15746 }
15747
15748 warn!(
15749 event = EVENT_DECOMMISSION_ENTRY,
15750 component = LOG_COMPONENT_ECSTORE,
15751 subsystem = LOG_SUBSYSTEM_POOLS,
15752 pool_index = idx,
15753 bucket = %bucket_name,
15754 object = %object_name,
15755 version = %version.name,
15756 state = "object_migrated",
15757 "Decommission object migrated"
15758 );
15759
15760 failure = false;
15761 break;
15762 }
15763
15764 if ignore {
15765 if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
15766 decommissioned += 1;
15767 }
15768 debug!(
15769 event = EVENT_DECOMMISSION_ENTRY,
15770 component = LOG_COMPONENT_ECSTORE,
15771 subsystem = LOG_SUBSYSTEM_POOLS,
15772 pool_index = idx,
15773 bucket = %bucket,
15774 object = %version.name,
15775 state = "ignored",
15776 "Decommission entry ignored"
15777 );
15778 continue;
15779 }
15780
15781 if counted_versions.insert((version.version_id, version.deleted)) {
15782 let mut pool_meta = self.pool_meta.write().await;
15783 ensure_decommission_generation(&pool_meta, idx, generation)?;
15784 if let Err(err) = count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), failure) {
15785 return Err(with_decommission_entry_context(
15786 "count_decommission_item",
15787 bucket.as_str(),
15788 entry.name.as_str(),
15789 err,
15790 ));
15791 }
15792 }
15793
15794 if failure {
15795 break;
15796 }
15797
15798 if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
15799 decommissioned += 1;
15800 }
15801 }
15802
15803 if free_version_disposition.total() > 0 {
15804 debug!(
15805 event = EVENT_DECOMMISSION_ENTRY,
15806 component = LOG_COMPONENT_ECSTORE,
15807 subsystem = LOG_SUBSYSTEM_POOLS,
15808 pool_index = idx,
15809 bucket = %bucket,
15810 object = %entry.name,
15811 free_versions_migrated = free_version_disposition.migrated,
15812 free_versions_consumed = free_version_disposition.consumed,
15813 free_versions_retained = free_version_disposition.retained,
15814 free_versions_total = free_version_disposition.total(),
15815 reason = DECOMMISSION_FREE_VERSION_DISPOSITION_REASON,
15816 state = "free_version_disposition",
15817 "Decommission free-version disposition summary"
15818 );
15819 }
15820
15821 if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) && durable_ilm_record.is_none()
15822 {
15823 if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
15824 return Err(Error::other("decommission bucket incarnation fence was lost before source cleanup"));
15825 }
15826 decommission_cancel_signal_result(rx.is_cancelled())?;
15827 self.ensure_decommission_generation_current(idx, generation).await?;
15828
15829 self.track_decommission_entry_progress_stage(
15830 idx,
15831 generation,
15832 bucket.as_str(),
15833 entry.name.as_str(),
15834 DECOMMISSION_STAGE_CLEANUP_PREFLIGHT,
15835 )
15836 .await?;
15837
15838 self.track_decommission_entry_progress_stage(
15839 idx,
15840 generation,
15841 bucket.as_str(),
15842 entry.name.as_str(),
15843 DECOMMISSION_STAGE_SOURCE_CLEANUP,
15844 )
15845 .await?;
15846
15847 #[cfg(test)]
15848 run_decommission_cleanup_mutation_hook(bucket.as_str(), entry.name.as_str(), entry_attempt).await;
15849
15850 let source_cleanup_mutation_fence = self
15851 .acquire_decommission_source_cleanup_fence(bucket.as_str(), entry.name.as_str(), set.as_ref())
15852 .await?;
15853 let cleanup_result = self
15854 .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
15855 data_movement::cleanup_source_entry_if_unchanged(
15856 set.clone(),
15857 bucket.as_str(),
15858 entry.name.as_str(),
15859 &fivs,
15860 &cleanup_preflight_allowed_missing,
15861 data_movement::SourceCleanupBucketFence {
15862 expected_incarnation_id: expected_bucket_incarnation_id,
15863 lifecycle_guard: bucket_incarnation_fence
15864 .as_ref()
15865 .and_then(|guard| guard.namespace_lock_guard()),
15866 namespace_lock_lost_signal: None,
15867 object_mutation_fence: Some(&source_cleanup_mutation_fence),
15868 },
15869 "decommission",
15870 )
15871 .await
15872 })
15873 .await;
15874 match cleanup_result {
15875 Ok(_) => {}
15876 Err(data_movement::SourceCleanupError::Storage(err)) => {
15877 resolve_decommission_entry_cleanup_delete_result(
15878 Err::<(), Error>(err),
15879 bucket.as_str(),
15880 entry.name.as_str(),
15881 )?;
15882 }
15883 Err(data_movement::SourceCleanupError::SourceChanged) if entry_attempt < DECOMMISSION_ENTRY_MAX_ATTEMPTS => {
15884 return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
15885 }
15886 Err(data_movement::SourceCleanupError::SourceChanged) => {
15887 let exhausted_entries = source_changed_exhaustions.fetch_add(1, Ordering::Relaxed) + 1;
15888 if should_fail_decommission_pool_after_exhausted_source_changed(exhausted_entries) {
15889 return Err(Error::other(format!(
15890 "decommission source cleanup retries exhausted for {}/{} on all {} attempts; exhausted entries exceed pool limit {}",
15891 bucket, entry.name, DECOMMISSION_ENTRY_MAX_ATTEMPTS, DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT
15892 )));
15893 }
15894
15895 {
15896 let mut pool_meta = self.pool_meta.write().await;
15897 ensure_decommission_generation(&pool_meta, idx, generation)?;
15898 count_decommission_item(&mut pool_meta, idx, 0, true).map_err(|err| {
15899 with_decommission_entry_context(
15900 "count_source_changed_exhaustion",
15901 bucket.as_str(),
15902 entry.name.as_str(),
15903 err,
15904 )
15905 })?;
15906 }
15907
15908 error!(
15909 event = EVENT_DECOMMISSION_ENTRY,
15910 component = LOG_COMPONENT_ECSTORE,
15911 subsystem = LOG_SUBSYSTEM_POOLS,
15912 state = "source_cleanup_exhausted",
15913 pool_index = idx,
15914 bucket = %bucket,
15915 object = %entry.name,
15916 attempts = DECOMMISSION_ENTRY_MAX_ATTEMPTS,
15917 exhausted_entries,
15918 exhaustion_limit = DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT,
15919 "Decommission source cleanup retries exhausted; source retained and entry marked failed"
15920 );
15921 entry_blocked = true;
15922 }
15923 }
15924 } else if durable_ilm_record.is_some() {
15925 debug!(
15926 event = EVENT_DECOMMISSION_ENTRY,
15927 component = LOG_COMPONENT_ECSTORE,
15928 subsystem = LOG_SUBSYSTEM_POOLS,
15929 pool_index = idx,
15930 bucket = %bucket,
15931 object = %entry.name,
15932 state = "retained_for_final_verification",
15933 "Decommission durable ILM source retained for final verification"
15934 );
15935 } else if decommissioned != fivs.versions.len() || expired > 0 {
15936 warn!(
15937 event = EVENT_DECOMMISSION_ENTRY,
15938 component = LOG_COMPONENT_ECSTORE,
15939 subsystem = LOG_SUBSYSTEM_POOLS,
15940 pool_index = idx,
15941 bucket = %bucket,
15942 object = %entry.name,
15943 decommissioned,
15944 total_versions = fivs.versions.len(),
15945 expired,
15946 state = "source_retained",
15947 "Decommission source object retained"
15948 );
15949 }
15950
15951 let should_save_progress = {
15952 let mut pool_meta = self.pool_meta.write().await;
15953 ensure_decommission_generation(&pool_meta, idx, generation)?;
15954
15955 if let Err(err) = track_decommission_current_object(&mut pool_meta, idx, bucket.as_str(), entry.name.as_str()) {
15956 return Err(with_decommission_entry_context(
15957 "track_decommission_current_object",
15958 bucket.as_str(),
15959 entry.name.as_str(),
15960 err,
15961 ));
15962 }
15963
15964 match resolve_decommission_update_after_result(pool_meta.update_after(idx, DECOMMISSION_PROGRESS_SAVE_INTERVAL)) {
15965 Ok(ok) => ok,
15966 Err(err) => {
15967 return Err(with_decommission_entry_context("update_after", bucket.as_str(), entry.name.as_str(), err));
15968 }
15969 }
15970 };
15971
15972 self.track_decommission_entry_progress_stage(
15973 idx,
15974 generation,
15975 bucket.as_str(),
15976 entry.name.as_str(),
15977 DECOMMISSION_STAGE_ENTRY_FINISHED,
15978 )
15979 .await?;
15980
15981 if should_save_progress {
15982 match self.save_decommission_progress_checkpoint(idx, generation).await {
15983 Ok(true) => {
15984 if let Some(notification_sys) = runtime_sources::notification_sys()
15985 && let Err(err) = resolve_decommission_entry_reload_result(
15986 notification_sys.reload_pool_meta().await,
15987 bucket.as_str(),
15988 entry.name.as_str(),
15989 )
15990 {
15991 warn!("{err}");
15992 }
15993 }
15994 Ok(false) => {}
15995 Err(err) => {
15996 if let Some(err) = resolve_decommission_progress_save_result(Err(err)) {
15997 warn!(
15998 event = EVENT_DECOMMISSION_ENTRY,
15999 component = LOG_COMPONENT_ECSTORE,
16000 subsystem = LOG_SUBSYSTEM_POOLS,
16001 pool_index = idx,
16002 bucket = %bucket,
16003 object = %entry.name,
16004 state = "progress_save_failed",
16005 error = %err,
16006 "Decommission progress save failed; continuing and will retry at the next checkpoint"
16007 );
16008 }
16009 }
16010 }
16011 }
16012
16013 debug!(
16014 event = EVENT_DECOMMISSION_ENTRY,
16015 component = LOG_COMPONENT_ECSTORE,
16016 subsystem = LOG_SUBSYSTEM_POOLS,
16017 state = if entry_blocked { "blocked" } else { "completed" },
16018 pool_index = idx,
16019 bucket = %bucket,
16020 object = %entry.name,
16021 "Decommission entry finished"
16022 );
16023 Ok(DecommissionEntryAttemptOutcome::Complete)
16024 }
16025
16026 #[cfg(test)]
16027 pub(crate) async fn decommission_entry_for_test(
16028 self: &Arc<Self>,
16029 idx: usize,
16030 entry: MetaCacheEntry,
16031 bucket: String,
16032 set: Arc<SetDisks>,
16033 ) -> Result<()> {
16034 self.decommission_entry_with_retry_state_for_test(
16035 CancellationToken::new(),
16036 idx,
16037 entry,
16038 bucket,
16039 set,
16040 None,
16041 Arc::new(AtomicUsize::new(0)),
16042 )
16043 .await
16044 }
16045
16046 #[cfg(test)]
16047 #[allow(clippy::too_many_arguments)]
16048 pub(crate) async fn decommission_entry_with_retry_state_for_test(
16049 self: &Arc<Self>,
16050 rx: CancellationToken,
16051 idx: usize,
16052 entry: MetaCacheEntry,
16053 bucket: String,
16054 set: Arc<SetDisks>,
16055 expected_bucket_incarnation_id: Option<uuid::Uuid>,
16056 source_changed_exhaustions: Arc<AtomicUsize>,
16057 ) -> Result<()> {
16058 {
16059 let mut cancelers = self.decommission_cancelers.write().await;
16060 if cancelers.get(idx).and_then(Option::as_ref).is_none()
16061 && let Some(slot) = cancelers.get_mut(idx)
16062 {
16063 *slot = Some(DecommissionCanceler::new(CancellationToken::new()));
16064 }
16065 }
16066 let needs_capacity_reservation = {
16067 let pool_meta = self.pool_meta.read().await;
16068 pool_meta
16069 .pools
16070 .get(idx)
16071 .and_then(|pool| pool.decommission.as_ref())
16072 .and_then(|info| info.capacity_reservation.as_ref())
16073 .is_some_and(|reservation| !reservation.active())
16074 };
16075 if needs_capacity_reservation {
16076 let capacity_infos = self.get_decommission_all_pool_capacity_infos_with_owned_usage().await?;
16077 let mut pool_meta = self.pool_meta.write().await;
16078 let version = pool_meta.version;
16079 pool_meta.version = POOL_META_VERSION;
16080 recover_decommission_capacity_reservations(
16081 &mut pool_meta,
16082 &capacity_infos,
16083 OffsetDateTime::now_utc(),
16084 crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof().is_some(),
16085 )?;
16086 pool_meta.version = version;
16087 }
16088 let generation = self.active_decommission_generation(idx).await?;
16089 resolve_decommission_entry_test_outcome(
16090 self.decommission_entry(
16091 rx,
16092 idx,
16093 generation,
16094 entry,
16095 bucket,
16096 set,
16097 None,
16098 None,
16099 None,
16100 expected_bucket_incarnation_id,
16101 source_changed_exhaustions,
16102 &mut 0,
16103 )
16104 .await,
16105 )
16106 }
16107
16108 #[cfg(all(test, feature = "test-util"))]
16109 pub(crate) async fn decommission_entry_for_test_with_bucket_incarnation(
16110 self: &Arc<Self>,
16111 idx: usize,
16112 entry: MetaCacheEntry,
16113 bucket: String,
16114 set: Arc<SetDisks>,
16115 ) -> Result<()> {
16116 let expected_bucket_incarnation_id = if is_meta_bucketname(&bucket) {
16117 None
16118 } else {
16119 Some(self.bucket_incarnation_id_from_disk(&bucket).await?)
16120 };
16121 let generation = self.active_decommission_generation(idx).await?;
16122 resolve_decommission_entry_test_outcome(
16123 self.decommission_entry(
16124 CancellationToken::new(),
16125 idx,
16126 generation,
16127 entry,
16128 bucket,
16129 set,
16130 None,
16131 None,
16132 None,
16133 expected_bucket_incarnation_id,
16134 Arc::new(AtomicUsize::new(0)),
16135 &mut 0,
16136 )
16137 .await,
16138 )
16139 }
16140
16141 #[tracing::instrument(skip(self, rx))]
16142 async fn decommission_pool(
16143 self: &Arc<Self>,
16144 rx: CancellationToken,
16145 idx: usize,
16146 pool: Arc<Sets>,
16147 bi: DecomBucketInfo,
16148 entry_budget: Arc<Semaphore>,
16149 source_changed_exhaustions: Arc<AtomicUsize>,
16150 ) -> Result<()> {
16151 let entry_error = Arc::new(tokio::sync::Mutex::new(None::<Error>));
16152 let generation = self.active_decommission_generation(idx).await?;
16153 let mut listing_workers = Vec::with_capacity(pool.disk_set.len());
16154
16155 let mut lifecycle_config = None;
16156 let mut object_lock_config = None;
16157 let mut replication_config = None;
16158 let expected_bucket_incarnation_id = if bi.name == RUSTFS_META_BUCKET {
16159 None
16160 } else {
16161 Some(self.bucket_incarnation_id_from_disk(&bi.name).await?)
16162 };
16163
16164 if bi.name != RUSTFS_META_BUCKET {
16165 let _ = resolve_decommission_optional_bucket_config_result(
16166 &bi.name,
16167 "versioning",
16168 BucketVersioningSys::get_in(&self.ctx, &bi.name).await,
16169 )?;
16170 let expiry_configs = get_expiry_configs(self, &bi.name).await?;
16171 lifecycle_config = expiry_configs.lifecycle.map(|config| (*config).clone());
16172 object_lock_config = expiry_configs.object_lock.map(|config| (*config).clone());
16173 replication_config = resolve_decommission_optional_bucket_config_result(
16174 &bi.name,
16175 "replication",
16176 metadata_sys::get_replication_config_in(&self.ctx, &bi.name).await,
16177 )?;
16178 }
16179
16180 for (set_idx, set) in pool.disk_set.iter().enumerate() {
16181 debug!(
16182 event = EVENT_DECOMMISSION_BUCKET,
16183 component = LOG_COMPONENT_ECSTORE,
16184 subsystem = LOG_SUBSYSTEM_POOLS,
16185 pool_index = idx,
16186 set_index = set_idx,
16187 bucket = %bi.name,
16188 state = "listing_worker_started",
16189 "Decommission listing worker started"
16190 );
16191
16192 let set = set.clone();
16193 let store = Arc::clone(self);
16194 let rx_clone = rx.clone();
16195 let bi_clone = bi.clone();
16196 let lifecycle_config = lifecycle_config.clone();
16197 let object_lock_config = object_lock_config.clone();
16198 let replication_config = replication_config.clone();
16199 let entry_budget = entry_budget.clone();
16200 let source_changed_exhaustions = Arc::clone(&source_changed_exhaustions);
16201 let entry_error = entry_error.clone();
16202 let worker = tokio::spawn(async move {
16203 store
16204 .decommission_set(
16205 rx_clone,
16206 idx,
16207 set_idx,
16208 generation,
16209 set,
16210 bi_clone,
16211 lifecycle_config,
16212 object_lock_config,
16213 replication_config,
16214 expected_bucket_incarnation_id,
16215 source_changed_exhaustions,
16216 entry_budget,
16217 entry_error,
16218 )
16219 .await
16220 });
16221 listing_workers.push((set_idx, worker));
16222 }
16223
16224 debug!(
16225 event = EVENT_DECOMMISSION_BUCKET,
16226 component = LOG_COMPONENT_ECSTORE,
16227 subsystem = LOG_SUBSYSTEM_POOLS,
16228 pool_index = idx,
16229 bucket = %bi.name,
16230 state = "waiting_for_workers",
16231 "Decommission waiting for workers"
16232 );
16233
16234 let mut listing_worker_error = None;
16235 for (set_id, worker) in listing_workers {
16236 if let Err(err) = resolve_decommission_listing_worker_result(set_id, worker.await) {
16237 rx.cancel();
16238 if listing_worker_error.is_none() {
16239 listing_worker_error = Some(err);
16240 }
16241 }
16242 }
16243
16244 if let Some(err) = listing_worker_error {
16245 return Err(err);
16246 }
16247
16248 if let Some(err) = entry_error.lock().await.clone() {
16249 return Err(err);
16250 }
16251
16252 if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) {
16253 warn!(
16254 event = EVENT_DECOMMISSION_BUCKET,
16255 component = LOG_COMPONENT_ECSTORE,
16256 subsystem = LOG_SUBSYSTEM_POOLS,
16257 pool_index = idx,
16258 bucket = %bi.name,
16259 state = "cancelled_after_wait",
16260 error = %err,
16261 "Decommission bucket cancelled after wait"
16262 );
16263 return Err(err);
16264 }
16265
16266 debug!(
16267 event = EVENT_DECOMMISSION_BUCKET,
16268 component = LOG_COMPONENT_ECSTORE,
16269 subsystem = LOG_SUBSYSTEM_POOLS,
16270 pool_index = idx,
16271 bucket = %bi.name,
16272 state = "completed",
16273 "Decommission bucket completed"
16274 );
16275
16276 Ok(())
16277 }
16278
16279 #[cfg(all(test, feature = "test-util"))]
16280 pub(crate) async fn decommission_pool_for_test(
16281 self: &Arc<Self>,
16282 rx: CancellationToken,
16283 idx: usize,
16284 pool: Arc<Sets>,
16285 bucket: DecomBucketInfo,
16286 ) -> Result<()> {
16287 self.decommission_pool(
16288 rx,
16289 idx,
16290 pool,
16291 bucket,
16292 Arc::new(Semaphore::new(decommission_entry_concurrency_limit())),
16293 Arc::new(AtomicUsize::new(0)),
16294 )
16295 .await
16296 }
16297
16298 #[tracing::instrument(skip(self, canceler))]
16299 pub async fn do_decommission_in_routine(
16300 self: &Arc<Self>,
16301 canceler: DecommissionCanceler,
16302 idx: usize,
16303 entry_budget: Arc<Semaphore>,
16304 ) -> Result<()> {
16305 let rx = canceler.token().clone();
16306 self.run_decommission_in_routine(rx, idx, &canceler, entry_budget).await
16307 }
16308
16309 async fn run_decommission_in_routine(
16310 self: &Arc<Self>,
16311 rx: CancellationToken,
16312 idx: usize,
16313 canceler: &DecommissionCanceler,
16314 entry_budget: Arc<Semaphore>,
16315 ) -> Result<()> {
16316 self.ensure_pool_meta_side_effects_safe("decommission cannot run while pool metadata requires recovery")
16317 .await?;
16318 let generation = match self.promote_queued_decommission(idx, canceler).await {
16319 Ok(generation) => generation,
16320 Err(Error::OperationCanceled) => return Ok(()),
16321 Err(err) if is_decommission_capacity_blocked_error(&err) || is_decommission_target_capacity_error(&err) => {
16322 if let Err(pause_err) = self.pause_decommission_for_capacity(idx, &err).await {
16323 return Err(decommission_capacity_blocked_error(format!(
16324 "failed to persist paused state for pool {idx}: {pause_err}"
16325 )));
16326 }
16327 return Ok(());
16328 }
16329 Err(err) => {
16330 resolve_decommission_terminal_mark_after_error_result(
16331 self.decommission_failed_for_operation(idx, canceler).await,
16332 idx,
16333 &err,
16334 )?;
16335 return Err(err);
16336 }
16337 };
16338 if rx.is_cancelled() {
16339 let already_canceled = {
16340 let pool_meta = self.pool_meta.read().await;
16341 should_skip_canceled_decommission_routine(true, pool_meta.pools.get(idx))
16342 };
16343 if already_canceled {
16344 warn!(
16345 event = EVENT_DECOMMISSION_STATE,
16346 component = LOG_COMPONENT_ECSTORE,
16347 subsystem = LOG_SUBSYSTEM_POOLS,
16348 pool_index = idx,
16349 state = "canceled_preserved",
16350 "Decommission routine skipped because pool is already canceled"
16351 );
16352 return Ok(());
16353 }
16354 if let Err(err) = self.decommission_cancel_for_operation(idx, canceler).await {
16355 resolve_decommission_terminal_mark_after_error_result(
16356 self.decommission_failed_for_operation(idx, canceler).await,
16357 idx,
16358 &err,
16359 )?;
16360 return Err(err);
16361 }
16362 return Ok(());
16363 }
16364 let result = self
16365 .decommission_in_background(rx.clone(), idx, generation, entry_budget)
16366 .await;
16367
16368 if let Err(err) = &result
16369 && (is_decommission_capacity_blocked_error(err) || is_decommission_target_capacity_error(err))
16370 {
16371 if let Err(pause_err) = self.pause_decommission_for_capacity(idx, err).await {
16372 return Err(decommission_capacity_blocked_error(format!(
16373 "failed to persist paused state for pool {idx}: {pause_err}"
16374 )));
16375 }
16376 return Ok(());
16377 }
16378
16379 let (final_state, canceled, cmd_line) = {
16380 let pool_meta = self.pool_meta.read().await;
16381 let Some(pool) = pool_meta.pools.get(idx) else {
16382 error!(
16383 event = EVENT_DECOMMISSION_STATE,
16384 component = LOG_COMPONENT_ECSTORE,
16385 subsystem = LOG_SUBSYSTEM_POOLS,
16386 pool_index = idx,
16387 state = "pool_metadata_missing",
16388 "Decommission pool metadata missing"
16389 );
16390 return Err(Error::other(format!(
16391 "failed to resolve decommission final state: pool metadata missing for idx {idx}"
16392 )));
16393 };
16394
16395 let (final_state, canceled) = if let Some(info) = &pool.decommission {
16396 (
16397 determine_decommission_final_state(info.items_decommission_failed, info.canceled),
16398 info.canceled,
16399 )
16400 } else {
16401 (DecommissionFinalState::Failed, false)
16402 };
16403 let cmd_line = pool.cmd_line.clone();
16404 (final_state, canceled, cmd_line)
16405 };
16406
16407 if let Err(err) = result {
16408 error!(
16409 event = EVENT_DECOMMISSION_STATE,
16410 component = LOG_COMPONENT_ECSTORE,
16411 subsystem = LOG_SUBSYSTEM_POOLS,
16412 pool_index = idx,
16413 state = "background_failed",
16414 error = ?err,
16415 "Decommission background routine failed"
16416 );
16417
16418 if should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
16419 warn!(
16420 event = EVENT_DECOMMISSION_STATE,
16421 component = LOG_COMPONENT_ECSTORE,
16422 subsystem = LOG_SUBSYSTEM_POOLS,
16423 pool_index = idx,
16424 cmd_line = %cmd_line,
16425 state = "cancelled_preserved",
16426 "Decommission cancelled; preserving canceled state"
16427 );
16428 return Ok(());
16429 }
16430
16431 resolve_decommission_terminal_mark_after_error_result(
16432 self.decommission_failed_for_operation(idx, canceler).await,
16433 idx,
16434 &err,
16435 )?;
16436 warn!(
16437 event = EVENT_DECOMMISSION_STATE,
16438 component = LOG_COMPONENT_ECSTORE,
16439 subsystem = LOG_SUBSYSTEM_POOLS,
16440 pool_index = idx,
16441 state = "marked_failed",
16442 "Decommission marked failed"
16443 );
16444
16445 return Ok(());
16446 }
16447
16448 debug!(
16449 event = EVENT_DECOMMISSION_STATE,
16450 component = LOG_COMPONENT_ECSTORE,
16451 subsystem = LOG_SUBSYSTEM_POOLS,
16452 pool_index = idx,
16453 state = "background_complete",
16454 "Decommission background routine completed"
16455 );
16456
16457 if should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
16458 warn!(
16459 event = EVENT_DECOMMISSION_STATE,
16460 component = LOG_COMPONENT_ECSTORE,
16461 subsystem = LOG_SUBSYSTEM_POOLS,
16462 pool_index = idx,
16463 cmd_line = %cmd_line,
16464 state = "terminal_state_preserved",
16465 "Decommission terminal state preserved after cancellation"
16466 );
16467 return Ok(());
16468 }
16469
16470 match final_state {
16471 DecommissionFinalState::Complete => {
16472 debug!(
16473 event = EVENT_DECOMMISSION_STATE,
16474 component = LOG_COMPONENT_ECSTORE,
16475 subsystem = LOG_SUBSYSTEM_POOLS,
16476 pool_index = idx,
16477 cmd_line = %cmd_line,
16478 state = "verifying_completion",
16479 "Decommission completion verification started"
16480 );
16481 let verified_unresolved_entries = match self.check_after_decommission(idx, &rx, generation).await {
16482 Ok(verified_entries) => verified_entries,
16483 Err(err) => {
16484 if is_err_operation_canceled(&err) {
16485 return Err(err);
16486 }
16487 resolve_decommission_terminal_mark_result(
16488 self.decommission_failed_for_operation(idx, canceler).await,
16489 "failed",
16490 &cmd_line,
16491 )?;
16492 return Err(Error::other(format!(
16493 "failed to finalize decommission for pool {cmd_line}: post-check failed: {err}"
16494 )));
16495 }
16496 };
16497 info!(
16498 event = EVENT_DECOMMISSION_STATE,
16499 component = LOG_COMPONENT_ECSTORE,
16500 subsystem = LOG_SUBSYSTEM_POOLS,
16501 pool_index = idx,
16502 cmd_line = %cmd_line,
16503 state = "marking_completed",
16504 "Decommission marking completed state"
16505 );
16506 if let Err(err) = self
16507 .complete_decommission_for_operation(idx, canceler, generation, verified_unresolved_entries)
16508 .await
16509 {
16510 resolve_decommission_terminal_mark_result(
16511 self.decommission_failed_for_operation(idx, canceler).await,
16512 "failed",
16513 &cmd_line,
16514 )?;
16515 return Err(Error::other(format!("failed to finalize decommission for pool {cmd_line}: {err}")));
16516 }
16517 }
16518 DecommissionFinalState::Failed => {
16519 warn!(
16520 event = EVENT_DECOMMISSION_STATE,
16521 component = LOG_COMPONENT_ECSTORE,
16522 subsystem = LOG_SUBSYSTEM_POOLS,
16523 pool_index = idx,
16524 cmd_line = %cmd_line,
16525 state = "marking_failed",
16526 "Decommission marking failed state"
16527 );
16528 resolve_decommission_terminal_mark_result(
16529 self.decommission_failed_for_operation(idx, canceler).await,
16530 "failed",
16531 &cmd_line,
16532 )?;
16533 }
16534 }
16535
16536 info!(
16537 event = EVENT_DECOMMISSION_STATE,
16538 component = LOG_COMPONENT_ECSTORE,
16539 subsystem = LOG_SUBSYSTEM_POOLS,
16540 pool_index = idx,
16541 cmd_line = %cmd_line,
16542 state = "completed",
16543 "Decommission completed"
16544 );
16545 Ok(())
16546 }
16547
16548 #[tracing::instrument(skip(self))]
16549 pub async fn decommission_failed(&self, idx: usize) -> Result<()> {
16550 self.decommission_failed_with_owner(idx, None).await
16551 }
16552
16553 async fn decommission_failed_for_operation(&self, idx: usize, owner: &DecommissionCanceler) -> Result<()> {
16554 self.decommission_failed_with_owner(idx, Some(owner)).await
16555 }
16556
16557 async fn decommission_failed_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> {
16558 self.decommission_failed_with_owner_and_save(idx, owner, self.save_current_pool_meta(&[idx]))
16559 .await
16560 }
16561
16562 async fn decommission_failed_with_owner_and_save<SaveFuture>(
16563 &self,
16564 idx: usize,
16565 owner: Option<&DecommissionCanceler>,
16566 save_pool_meta: SaveFuture,
16567 ) -> Result<()>
16568 where
16569 SaveFuture: Future<Output = Result<()>>,
16570 {
16571 ensure_decommission_terminal_operation_supported(self.single_pool(), "mark decommission failed")?;
16572 let _start_guard = self.start_gate.lock().await;
16573 let movement_gate = self.ctx.data_movement_operation_gate();
16574 let _movement_guard = movement_gate.write().await;
16575
16576 let rebalance_meta = self.rebalance_meta.read().await.clone();
16580 let terminal_at = OffsetDateTime::now_utc();
16581 let (should_reload_pool_meta, previous_pool_meta, terminal_canceler) = {
16582 let cancelers = self.decommission_cancelers.read().await;
16583 let mut pool_meta = self.pool_meta.write().await;
16584 let previous_pool_meta = pool_meta.clone();
16585 let Some(changed) =
16586 update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, idx, owner, |pool_meta| {
16587 pool_meta.decommission_failed_at(idx, terminal_at, rebalance_meta.as_ref())
16588 })
16589 else {
16590 return Ok(());
16591 };
16592 let terminal_canceler = if let Some(owner) = owner {
16593 Some(owner.clone())
16594 } else {
16595 cancelers.get(idx).and_then(Option::as_ref).cloned()
16596 };
16597 (changed, changed.then_some(previous_pool_meta), terminal_canceler)
16598 };
16599
16600 if should_reload_pool_meta && let Err(err) = save_pool_meta.await {
16601 if let Some(previous_pool_meta) = previous_pool_meta {
16602 let mut pool_meta = self.pool_meta.write().await;
16603 rollback_decommission_pool_meta(&mut pool_meta, &previous_pool_meta, &[idx]);
16604 }
16605 return Err(err);
16606 }
16607 if should_reload_pool_meta {
16608 {
16609 let mut pool_meta = self.pool_meta.write().await;
16610 pool_meta.mark_decommission_progress_saved();
16611 }
16612 }
16613 if let Some(canceler) = terminal_canceler.as_ref() {
16614 self.release_decommission_canceler_slot(idx, canceler).await;
16615 }
16616
16617 if should_reload_pool_meta {
16618 self.ctx.advance_data_movement_operation_epoch();
16619 }
16620 drop(_movement_guard);
16621
16622 if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
16623 let stage = format!("decommission_failed for pool {idx}");
16624 if let Some(err) = observe_decommission_terminal_reload_result(
16625 resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()),
16626 stage.as_str(),
16627 ) {
16628 if let Err(record_err) = self
16629 .record_decommission_terminal_reload_failure(idx, stage.as_str(), err.clone())
16630 .await
16631 {
16632 warn!(
16633 event = EVENT_DECOMMISSION_STATE,
16634 component = LOG_COMPONENT_ECSTORE,
16635 subsystem = LOG_SUBSYSTEM_POOLS,
16636 pool_index = idx,
16637 state = "terminal_reload_record_failed",
16638 error = %record_err,
16639 original_error = %err,
16640 "Decommission terminal reload failure record failed"
16641 );
16642 }
16643 warn!(
16644 event = EVENT_DECOMMISSION_STATE,
16645 component = LOG_COMPONENT_ECSTORE,
16646 subsystem = LOG_SUBSYSTEM_POOLS,
16647 pool_index = idx,
16648 state = "terminal_reload_failed",
16649 error = %err,
16650 "Decommission terminal state saved but pool meta reload failed"
16651 );
16652 }
16653 }
16654
16655 Ok(())
16656 }
16657
16658 #[tracing::instrument(skip(self))]
16659 pub async fn complete_decommission(&self, idx: usize) -> Result<()> {
16660 self.complete_decommission_with_owner(idx, None, None, None).await
16661 }
16662
16663 async fn complete_decommission_for_operation(
16664 &self,
16665 idx: usize,
16666 owner: &DecommissionCanceler,
16667 verified_generation: OffsetDateTime,
16668 verified_unresolved_entries: Vec<DecommissionUnresolvedEntry>,
16669 ) -> Result<()> {
16670 self.complete_decommission_with_owner(idx, Some(owner), Some(verified_generation), Some(verified_unresolved_entries))
16671 .await
16672 }
16673
16674 async fn complete_decommission_with_owner(
16675 &self,
16676 idx: usize,
16677 owner: Option<&DecommissionCanceler>,
16678 verified_generation: Option<OffsetDateTime>,
16679 verified_unresolved_entries: Option<Vec<DecommissionUnresolvedEntry>>,
16680 ) -> Result<()> {
16681 ensure_decommission_terminal_operation_supported(self.single_pool(), "complete decommission")?;
16682 ensure_valid_decommission_pool_index(self.pools.len(), idx)?;
16683 if let Some(owner) = owner {
16684 let cancelers = self.decommission_cancelers.read().await;
16685 if !decommission_canceler_is_owned_by(cancelers.as_slice(), idx, owner) {
16686 owner.release();
16687 return Ok(());
16688 }
16689 }
16690 self.verify_decommission_durable_ilm_receipts(idx).await?;
16691 let _start_guard = self.start_gate.lock().await;
16692 let movement_gate = self.ctx.data_movement_operation_gate();
16693 let _movement_guard = movement_gate.write().await;
16694
16695 let rebalance_meta = self.rebalance_meta.read().await.clone();
16699 let terminal_at = OffsetDateTime::now_utc();
16700 let (should_reload_pool_meta, completed, previous_pool_meta, terminal_canceler) = {
16701 let cancelers = self.decommission_cancelers.read().await;
16702 let mut pool_meta = self.pool_meta.write().await;
16703 let previous_pool_meta = pool_meta.clone();
16704 let Some(changed) =
16705 update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, idx, owner, |pool_meta| {
16706 reconcile_decommission_unresolved_entries_for_completion(
16707 pool_meta,
16708 idx,
16709 verified_generation,
16710 verified_unresolved_entries.as_deref(),
16711 )?;
16712 Ok::<bool, Error>(pool_meta.decommission_complete_at(idx, terminal_at, rebalance_meta.as_ref()))
16713 })
16714 else {
16715 return Ok(());
16716 };
16717 let changed = changed?;
16718 let completed = pool_meta
16719 .pools
16720 .get(idx)
16721 .and_then(|pool| pool.decommission.as_ref())
16722 .is_some_and(|decommission| decommission.complete);
16723 let terminal_canceler = if let Some(owner) = owner {
16724 Some(owner.clone())
16725 } else {
16726 cancelers.get(idx).and_then(Option::as_ref).cloned()
16727 };
16728 (changed, completed, changed.then_some(previous_pool_meta), terminal_canceler)
16729 };
16730
16731 if should_reload_pool_meta && let Err(err) = self.save_current_pool_meta(&[idx]).await {
16732 if let Some(previous_pool_meta) = previous_pool_meta {
16733 let mut pool_meta = self.pool_meta.write().await;
16734 rollback_decommission_pool_meta(&mut pool_meta, &previous_pool_meta, &[idx]);
16735 }
16736 return Err(err);
16737 }
16738 if should_reload_pool_meta {
16739 {
16740 let mut pool_meta = self.pool_meta.write().await;
16741 pool_meta.mark_decommission_progress_saved();
16742 }
16743 }
16744 if let Some(canceler) = terminal_canceler.as_ref() {
16745 self.release_decommission_canceler_slot(idx, canceler).await;
16746 }
16747
16748 if should_reload_pool_meta {
16749 self.ctx.advance_data_movement_operation_epoch();
16750 }
16751 drop(_movement_guard);
16752
16753 if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
16754 let stage = format!("complete_decommission for pool {idx}");
16755 if let Some(err) = observe_decommission_terminal_reload_result(
16756 resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()),
16757 stage.as_str(),
16758 ) {
16759 if let Err(record_err) = self
16760 .record_decommission_terminal_reload_failure(idx, stage.as_str(), err.clone())
16761 .await
16762 {
16763 warn!(
16764 event = EVENT_DECOMMISSION_STATE,
16765 component = LOG_COMPONENT_ECSTORE,
16766 subsystem = LOG_SUBSYSTEM_POOLS,
16767 pool_index = idx,
16768 state = "terminal_reload_record_failed",
16769 error = %record_err,
16770 original_error = %err,
16771 "Decommission terminal reload failure record failed"
16772 );
16773 }
16774 warn!(
16775 event = EVENT_DECOMMISSION_STATE,
16776 component = LOG_COMPONENT_ECSTORE,
16777 subsystem = LOG_SUBSYSTEM_POOLS,
16778 pool_index = idx,
16779 state = "terminal_reload_failed",
16780 error = %err,
16781 "Decommission terminal state saved but pool meta reload failed"
16782 );
16783 }
16784 }
16785
16786 if completed && let Err(err) = self.cleanup_decommission_durable_ilm_receipts(idx).await {
16787 warn!(
16788 event = EVENT_DECOMMISSION_STATE,
16789 component = LOG_COMPONENT_ECSTORE,
16790 subsystem = LOG_SUBSYSTEM_POOLS,
16791 pool_index = idx,
16792 state = "receipt_cleanup_failed",
16793 error = %err,
16794 "Decommission durable ILM receipt cleanup failed"
16795 );
16796 }
16797
16798 Ok(())
16799 }
16800
16801 async fn decommission_pending_bucket(
16802 self: &Arc<Self>,
16803 rx: CancellationToken,
16804 idx: usize,
16805 generation: OffsetDateTime,
16806 bucket: DecomBucketInfo,
16807 entry_budget: Arc<Semaphore>,
16808 source_changed_exhaustions: Arc<AtomicUsize>,
16809 ) -> Result<()> {
16810 let is_decommissioned = {
16811 let pool_meta = self.pool_meta.read().await;
16812 ensure_decommission_generation(&pool_meta, idx, generation)?;
16813 resolve_decommission_bucket_state(&pool_meta, idx, &bucket)?
16814 };
16815
16816 if is_decommissioned {
16817 warn!("decommission: already done, moving on {}", bucket.to_string());
16818
16819 self.mark_decommission_bucket_done_and_save(idx, generation, &bucket).await?;
16820 return Ok(());
16821 }
16822
16823 warn!("decommission: currently on bucket {}", &bucket.name);
16824
16825 let pool = get_by_index(self.pools.as_slice(), idx, "load decommission background pool")?.clone();
16826 if let Err(err) = self
16827 .decommission_pool(rx.clone(), idx, pool, bucket.clone(), entry_budget, source_changed_exhaustions)
16828 .await
16829 {
16830 error!("decommission: decommission_pool err {:?}", &err);
16831 return Err(err);
16832 } else {
16833 warn!("decommission: decommission_pool done {}", &bucket.name);
16834 }
16835
16836 if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) {
16837 warn!("decommission: cancellation observed after decommission_pool {}", &bucket.name);
16838 return Err(err);
16839 }
16840
16841 self.mark_decommission_bucket_done_and_save(idx, generation, &bucket).await?;
16842
16843 warn!("decommission: decommission_pool bucket_done {}", &bucket.name);
16844
16845 Ok(())
16846 }
16847
16848 #[tracing::instrument(skip(self, rx))]
16849 async fn decommission_in_background(
16850 self: &Arc<Self>,
16851 rx: CancellationToken,
16852 idx: usize,
16853 generation: OffsetDateTime,
16854 entry_budget: Arc<Semaphore>,
16855 ) -> Result<()> {
16856 self.ensure_decommission_runtime_capacity_available(idx, generation).await?;
16857 let pending = {
16858 let pool_meta = self.pool_meta.read().await;
16859 pool_meta.pending_buckets(idx)
16860 };
16861 let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
16862 let bucket_concurrency = decommission_bucket_concurrency_limit();
16863 let (regular_buckets, meta_buckets) = split_decommission_buckets(pending);
16864 let store = Arc::clone(self);
16865 run_decommission_phases(rx.clone(), regular_buckets, meta_buckets, bucket_concurrency, move |bucket, rx| {
16866 let store = Arc::clone(&store);
16867 let entry_budget = entry_budget.clone();
16868 let source_changed_exhaustions = Arc::clone(&source_changed_exhaustions);
16869 Box::pin(async move {
16870 store
16871 .decommission_pending_bucket(rx, idx, generation, bucket, entry_budget, source_changed_exhaustions)
16872 .await
16873 })
16874 })
16875 .await
16876 }
16877
16878 #[tracing::instrument(skip(self))]
16879 pub async fn start_decommission(&self, indices: Vec<usize>) -> Result<()> {
16880 self.start_decommission_inner(indices, None).await.map(|_| ())
16881 }
16882
16883 async fn start_decommission_with_routines(
16884 &self,
16885 indices: Vec<usize>,
16886 rx: &CancellationToken,
16887 local_indices: &[usize],
16888 ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
16889 self.start_decommission_inner(indices, Some((rx, local_indices))).await
16890 }
16891
16892 async fn start_decommission_inner(
16893 &self,
16894 indices: Vec<usize>,
16895 reservation: Option<(&CancellationToken, &[usize])>,
16896 ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
16897 let indices = dedup_indices(&indices);
16898 validate_start_decommission_request(&indices, self.single_pool())?;
16899
16900 self.ensure_decommission_rebalance_idle_after_refresh().await?;
16901 #[cfg(test)]
16902 let endpoints = self.instance_endpoints().unwrap_or_else(|| self.endpoints());
16903 #[cfg(not(test))]
16904 let endpoints = self.endpoints();
16905 ensure_decommission_start_local_leader(&endpoints, &indices)?;
16906
16907 for idx in indices.iter().copied() {
16908 ensure_valid_decommission_pool_index(self.pools.len(), idx)?;
16909 }
16910
16911 {
16912 let pool_meta = self.pool_meta.read().await;
16913 ensure_decommission_start_pool_states(&pool_meta, &indices)?;
16914 ensure_decommission_ledger_persistence_supported(&pool_meta)?;
16915 ensure_decommission_capacity_writer_supported(&pool_meta)?;
16916 }
16917
16918 #[cfg(test)]
16919 observe_pool_activation_preflight_side_effect_attempt(PoolActivationStartKind::Decommission);
16920 let decom_buckets = self.get_buckets_to_decommission().await?;
16921
16922 let mut healed_buckets = HashSet::with_capacity(decom_buckets.len());
16923 for bk in decom_buckets.iter() {
16924 if healed_buckets.insert(bk.name.as_str()) {
16925 resolve_decommission_preflight_heal_result(&bk.name, self.heal_bucket(&bk.name, &HealOpts::default()).await)?;
16926 }
16927 }
16928
16929 let meta_bucket_opts = decommission_meta_bucket_options();
16930 for prefix in DECOMMISSION_META_PREFIXES {
16931 let bk = path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(prefix)]);
16932 if let Err(err) = self
16933 .make_bucket(bk.to_string_lossy().to_string().as_str(), &meta_bucket_opts)
16934 .await
16935 && !is_err_bucket_exists(&err)
16936 {
16937 error!("decommission: make bucket failed: {err}");
16938 return Err(err);
16939 }
16940 }
16941
16942 let _start_guard = self.start_gate.lock().await;
16943 self.ensure_decommission_rebalance_idle_after_refresh_under_start_gate()
16944 .await?;
16945
16946 let all_capacity_infos = self.get_decommission_all_pool_capacity_infos_with_owned_usage().await?;
16947 let target_fence_proof_available =
16948 crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof().is_some();
16949 self.cancel_decommission_routines(&indices).await;
16952 let movement_gate = self.ctx.data_movement_operation_gate();
16953 let _movement_guard = movement_gate.write().await;
16954
16955 let index_cancelers = if let Some((rx, local_indices)) = reservation {
16956 let mut cancelers = self.decommission_cancelers.write().await;
16960 let pool_meta = self.pool_meta.read().await;
16961 ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_capacity_infos, target_fence_proof_available)?;
16962 reserve_decommission_start_cancelers(&pool_meta, &indices, local_indices, rx, cancelers.as_mut_slice())?
16963 } else {
16964 let pool_meta = self.pool_meta.read().await;
16965 ensure_decommission_start_pool_states(&pool_meta, &indices)?;
16966 ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_capacity_infos, target_fence_proof_available)?;
16967 Vec::new()
16968 };
16969
16970 let previous_pool_meta = self
16971 .save_current_pool_meta_for_decommission_start(&indices, decom_buckets)
16972 .await?;
16973 self.ctx.advance_data_movement_operation_epoch();
16974 drop(_movement_guard);
16977
16978 if let Some(notification_sys) = runtime_sources::notification_sys()
16979 && let Err(err) = resolve_start_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await)
16980 {
16981 warn!(
16982 event = EVENT_DECOMMISSION_STATE,
16983 component = LOG_COMPONENT_ECSTORE,
16984 subsystem = LOG_SUBSYSTEM_POOLS,
16985 state = "start_failed",
16986 stage = "reload_pool_meta",
16987 error = %err,
16988 "Decommission start failed after pool metadata save"
16989 );
16990
16991 let rollback_result = self
16992 .rollback_decommission_start_after_reload_failure(&movement_gate, &previous_pool_meta, &indices)
16993 .await;
16994 if let Err(rollback_save_err) = rollback_result {
16995 error!(
16996 event = EVENT_DECOMMISSION_STATE,
16997 component = LOG_COMPONENT_ECSTORE,
16998 subsystem = LOG_SUBSYSTEM_POOLS,
16999 state = "rollback_failed",
17000 stage = "save_pool_meta",
17001 error = %rollback_save_err,
17002 original_error = %err,
17003 "Decommission rollback failed after pool metadata reload failure"
17004 );
17005 return Err(Error::other(format!(
17006 "{err}; decommission start rollback save failed: {rollback_save_err}"
17007 )));
17008 }
17009
17010 if let Err(rollback_reload_err) = resolve_decommission_pool_meta_reload_result(
17011 notification_sys.reload_pool_meta().await,
17012 "start_decommission_rollback",
17013 ) {
17014 error!(
17015 event = EVENT_DECOMMISSION_STATE,
17016 component = LOG_COMPONENT_ECSTORE,
17017 subsystem = LOG_SUBSYSTEM_POOLS,
17018 state = "rollback_partial",
17019 stage = "reload_pool_meta",
17020 error = %rollback_reload_err,
17021 original_error = %err,
17022 "Decommission rollback metadata reload failed after local rollback save"
17023 );
17024 return Err(Error::other(format!(
17025 "{err}; decommission start rollback saved locally but peer reload failed: {rollback_reload_err}"
17026 )));
17027 }
17028
17029 warn!(
17030 event = EVENT_DECOMMISSION_STATE,
17031 component = LOG_COMPONENT_ECSTORE,
17032 subsystem = LOG_SUBSYSTEM_POOLS,
17033 state = "rollback_success",
17034 original_error = %err,
17035 "Decommission start rolled back after pool metadata reload failure"
17036 );
17037 return Err(Error::other(format!("{err}; decommission start rollback succeeded")));
17038 }
17039
17040 Ok(index_cancelers)
17041 }
17042
17043 async fn get_buckets_to_decommission(&self) -> Result<Vec<DecomBucketInfo>> {
17044 let buckets = self.list_bucket(&BucketOptions::default()).await?;
17045
17046 let mut ret: Vec<DecomBucketInfo> = buckets
17047 .iter()
17048 .map(|v| DecomBucketInfo {
17049 name: v.name.clone(),
17050 ..Default::default()
17051 })
17052 .collect();
17053
17054 ret.extend(decommission_meta_buckets());
17055
17056 Ok(ret)
17057 }
17058
17059 async fn durable_ilm_receipt_run_token(&self, source_pool_idx: usize) -> Result<String> {
17060 let pool_meta = self.pool_meta.read().await;
17061 let pool = pool_meta
17062 .pools
17063 .get(source_pool_idx)
17064 .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), source_pool_idx))?;
17065 let start_time = pool
17066 .decommission
17067 .as_ref()
17068 .and_then(|info| info.start_time)
17069 .ok_or_else(|| Error::other(format!("decommission run identity is missing for pool {source_pool_idx}")))?;
17070 Ok(decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time))
17071 }
17072
17073 async fn durable_ilm_receipt_run_token_for_generation(
17074 &self,
17075 source_pool_idx: usize,
17076 generation: OffsetDateTime,
17077 ) -> Result<String> {
17078 let pool_meta = self.pool_meta.read().await;
17079 ensure_decommission_generation(&pool_meta, source_pool_idx, generation)?;
17080 let pool = pool_meta
17081 .pools
17082 .get(source_pool_idx)
17083 .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), source_pool_idx))?;
17084 Ok(decommission_durable_ilm_receipt_run_token(&pool.cmd_line, generation))
17085 }
17086
17087 async fn load_decommissioned_durable_ilm_target(
17088 &self,
17089 source_pool_idx: usize,
17090 path: &str,
17091 max_record_size: usize,
17092 record_context: &str,
17093 ) -> Result<Option<(usize, Vec<u8>)>> {
17094 let mut target = None::<(usize, Vec<u8>)>;
17095 let mut first_read_error = None;
17096 for (target_pool_idx, pool) in self.pools.iter().enumerate() {
17097 if target_pool_idx == source_pool_idx {
17098 continue;
17099 }
17100 match read_config_limited_preserve_empty(pool.clone(), path, max_record_size).await {
17101 Ok(data) => {
17102 if let Some((existing_pool_idx, existing)) = target.as_ref()
17103 && existing != &data
17104 {
17105 return Err(Error::other(format!(
17106 "divergent target durable ILM records at path `{path}` {record_context} in pools {existing_pool_idx} and {target_pool_idx}"
17107 )));
17108 }
17109 target = Some((target_pool_idx, data));
17110 }
17111 Err(err)
17112 if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17113 || is_err_object_not_found(&err)
17114 || is_err_version_not_found(&err) => {}
17115 Err(err) => {
17116 first_read_error.get_or_insert_with(|| {
17117 Error::other(format!(
17118 "failed to read target durable ILM record at path `{path}` {record_context} from pool {target_pool_idx}: {err}"
17119 ))
17120 });
17121 }
17122 }
17123 }
17124
17125 if let Some(err) = first_read_error {
17126 return Err(err);
17127 }
17128 Ok(target)
17129 }
17130
17131 async fn list_decommission_durable_ilm_receipt_paths_in_pool(&self, pool_idx: usize, prefix: &str) -> Result<Vec<String>> {
17132 let pool = self
17133 .pools
17134 .get(pool_idx)
17135 .ok_or_else(|| invalid_decommission_pool_index_error(self.pools.len(), pool_idx))?;
17136 let mut receipts = Vec::new();
17137 let mut continuation = None;
17138 loop {
17139 let page = pool
17140 .clone()
17141 .list_objects_v2(RUSTFS_META_BUCKET, prefix, continuation, None, 1000, false, None, false)
17142 .await
17143 .map_err(|err| {
17144 Error::other(format!(
17145 "failed to list durable ILM decommission receipts under `{prefix}` in pool {pool_idx}: {err}"
17146 ))
17147 })?;
17148 receipts.extend(page.objects.into_iter().map(|object| object.name));
17149 if !page.is_truncated {
17150 break;
17151 }
17152 continuation = Some(page.next_continuation_token.ok_or_else(|| {
17153 Error::other(format!(
17154 "durable ILM decommission receipt listing under `{prefix}` in pool {pool_idx} was truncated without a continuation token"
17155 ))
17156 })?);
17157 }
17158 Ok(receipts)
17159 }
17160
17161 async fn list_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<Vec<(usize, String)>> {
17162 let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
17163 self.list_decommission_durable_ilm_receipts_for_run(source_pool_idx, &run_token)
17164 .await
17165 }
17166
17167 async fn list_decommission_durable_ilm_receipts_for_run(
17168 &self,
17169 source_pool_idx: usize,
17170 run_token: &str,
17171 ) -> Result<Vec<(usize, String)>> {
17172 let prefix = decommission_durable_ilm_receipt_run_prefix(run_token);
17173 let mut receipts = Vec::new();
17174 for pool_idx in 0..self.pools.len() {
17175 if pool_idx == source_pool_idx {
17176 continue;
17177 }
17178 for receipt_path in self
17179 .list_decommission_durable_ilm_receipt_paths_in_pool(pool_idx, &prefix)
17180 .await?
17181 {
17182 let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
17183 if locator.run_token != run_token {
17184 return Err(Error::other(format!(
17185 "durable ILM receipt path `{receipt_path}` has an unexpected run token"
17186 )));
17187 }
17188 receipts.push((pool_idx, receipt_path));
17189 }
17190 }
17191 Ok(receipts)
17192 }
17193
17194 async fn list_decommission_durable_ilm_manifest_receipts(&self, source_pool_idx: usize) -> Result<Vec<String>> {
17195 let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
17196 self.list_decommission_durable_ilm_manifest_receipts_for_run(source_pool_idx, &run_token)
17197 .await
17198 }
17199
17200 async fn list_decommission_durable_ilm_manifest_receipts_for_run(
17201 &self,
17202 source_pool_idx: usize,
17203 run_token: &str,
17204 ) -> Result<Vec<String>> {
17205 let prefix = decommission_durable_ilm_receipt_run_prefix(run_token);
17206 let receipt_paths = self
17207 .list_decommission_durable_ilm_receipt_paths_in_pool(source_pool_idx, &prefix)
17208 .await?;
17209 for receipt_path in &receipt_paths {
17210 let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?;
17211 if locator.run_token != run_token {
17212 return Err(Error::other(format!(
17213 "durable ILM expected manifest receipt path `{receipt_path}` has an unexpected run token"
17214 )));
17215 }
17216 }
17217 Ok(receipt_paths)
17218 }
17219
17220 #[cfg(all(test, feature = "test-util"))]
17221 async fn persist_decommission_durable_ilm_manifest(&self, source_pool_idx: usize) -> Result<()> {
17222 let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
17223 self.persist_decommission_durable_ilm_manifest_for_run(source_pool_idx, &run_token)
17224 .await
17225 }
17226
17227 async fn persist_decommission_durable_ilm_manifest_for_run(&self, source_pool_idx: usize, run_token: &str) -> Result<()> {
17228 let receipt_paths = self
17229 .list_decommission_durable_ilm_manifest_receipts_for_run(source_pool_idx, run_token)
17230 .await?;
17231 for receipt_path in &receipt_paths {
17232 self.read_decommission_durable_ilm_receipt(source_pool_idx, receipt_path)
17233 .await?;
17234 }
17235 let manifest = DecommissionDurableIlmManifest::new(run_token, &receipt_paths)?;
17236 let manifest_path = decommission_durable_ilm_manifest_path(run_token);
17237 let encoded = manifest.encode()?;
17238 let mut attempt = 1;
17239 loop {
17240 match read_config_limited_preserve_empty(
17241 self.pools[source_pool_idx].clone(),
17242 &manifest_path,
17243 DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE,
17244 )
17245 .await
17246 {
17247 Ok(existing) => {
17248 DecommissionDurableIlmManifest::decode(&existing, run_token, &receipt_paths).map_err(|err| {
17249 Error::other(format!(
17250 "durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx} is invalid: {err}"
17251 ))
17252 })?;
17253 return Ok(());
17254 }
17255 Err(err)
17256 if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17257 || is_err_object_not_found(&err)
17258 || is_err_version_not_found(&err) => {}
17259 Err(err) => {
17260 return Err(Error::other(format!(
17261 "failed to read durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}"
17262 )));
17263 }
17264 }
17265 match save_config_with_opts(
17266 self.pools[source_pool_idx].clone(),
17267 &manifest_path,
17268 encoded.clone(),
17269 &ObjectOptions {
17270 max_parity: true,
17271 write_completion: crate::object_api::WriteCompletion::TailDrained,
17272 http_preconditions: Some(HTTPPreconditions {
17273 if_none_match: Some("*".to_string()),
17274 ..Default::default()
17275 }),
17276 ..Default::default()
17277 },
17278 )
17279 .await
17280 {
17281 Ok(()) => return Ok(()),
17282 Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => {
17283 attempt += 1;
17284 }
17285 Err(Error::PreconditionFailed) => {
17286 return Err(Error::other(format!(
17287 "failed to persist durable ILM expected manifest `{manifest_path}` after concurrent updates"
17288 )));
17289 }
17290 Err(err) => {
17291 return Err(Error::other(format!(
17292 "failed to persist durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx}: {err}"
17293 )));
17294 }
17295 }
17296 }
17297 }
17298
17299 async fn load_decommission_durable_ilm_manifest_for_run(
17300 &self,
17301 source_pool_idx: usize,
17302 run_token: &str,
17303 ) -> Result<HashMap<String, DecommissionDurableIlmReceipt>> {
17304 let receipt_paths = self
17305 .list_decommission_durable_ilm_manifest_receipts_for_run(source_pool_idx, run_token)
17306 .await?;
17307 let manifest_path = decommission_durable_ilm_manifest_path(run_token);
17308 let data = read_config_limited_preserve_empty(
17309 self.pools[source_pool_idx].clone(),
17310 &manifest_path,
17311 DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE,
17312 )
17313 .await
17314 .map_err(|err| {
17315 Error::other(format!(
17316 "failed to read durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}"
17317 ))
17318 })?;
17319 DecommissionDurableIlmManifest::decode(&data, run_token, &receipt_paths).map_err(|err| {
17320 Error::other(format!(
17321 "durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx} is invalid: {err}"
17322 ))
17323 })?;
17324
17325 let mut receipts = HashMap::with_capacity(receipt_paths.len());
17326 for receipt_path in receipt_paths {
17327 let receipt = self
17328 .read_decommission_durable_ilm_receipt(source_pool_idx, &receipt_path)
17329 .await?;
17330 if receipts.insert(receipt_path.clone(), receipt).is_some() {
17331 return Err(Error::other(format!(
17332 "durable ILM expected manifest contains duplicate receipt path `{receipt_path}`"
17333 )));
17334 }
17335 }
17336 Ok(receipts)
17337 }
17338
17339 #[cfg(all(test, feature = "test-util"))]
17340 async fn persist_decommission_durable_ilm_receipt(
17341 &self,
17342 source_pool_idx: usize,
17343 target_pool_idx: usize,
17344 receipt: &DecommissionDurableIlmReceipt,
17345 ) -> Result<()> {
17346 let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
17347 self.persist_decommission_durable_ilm_receipt_for_run(target_pool_idx, receipt, &run_token)
17348 .await
17349 }
17350
17351 async fn persist_decommission_durable_ilm_receipt_for_run(
17352 &self,
17353 target_pool_idx: usize,
17354 receipt: &DecommissionDurableIlmReceipt,
17355 run_token: &str,
17356 ) -> Result<()> {
17357 let receipt_path = decommission_durable_ilm_receipt_path(run_token, &receipt.source_path, &receipt.id_kind, &receipt.id);
17358 let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
17359 let mut attempt = 1;
17360 loop {
17361 let (merged, http_preconditions) = match read_config_limited_preserve_empty_with_metadata(
17362 self.pools[target_pool_idx].clone(),
17363 &receipt_path,
17364 DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
17365 )
17366 .await
17367 {
17368 Ok((existing_data, metadata)) => {
17369 let existing = DecommissionDurableIlmReceipt::decode(&existing_data).map_err(|err| {
17370 Error::other(format!(
17371 "durable ILM decommission receipt `{receipt_path}` in pool {target_pool_idx} for {} is invalid: {err}",
17372 locator.context()
17373 ))
17374 })?;
17375 Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &existing)?;
17376 let merged = merge_decommission_durable_ilm_receipts(&existing, receipt)?;
17377 if merged == existing {
17378 return Ok(());
17379 }
17380 let etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| {
17381 Error::other(format!(
17382 "durable ILM decommission receipt `{receipt_path}` in pool {target_pool_idx} is missing an ETag"
17383 ))
17384 })?;
17385 (
17386 merged,
17387 HTTPPreconditions {
17388 if_match: Some(etag),
17389 ..Default::default()
17390 },
17391 )
17392 }
17393 Err(err)
17394 if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17395 || is_err_object_not_found(&err)
17396 || is_err_version_not_found(&err) =>
17397 {
17398 (
17399 receipt.clone(),
17400 HTTPPreconditions {
17401 if_none_match: Some("*".to_string()),
17402 ..Default::default()
17403 },
17404 )
17405 }
17406 Err(err) => {
17407 return Err(Error::other(format!(
17408 "failed to read durable ILM decommission receipt `{receipt_path}` from pool {target_pool_idx} for {}: {err}",
17409 locator.context()
17410 )));
17411 }
17412 };
17413 let encoded = merged.encode().map_err(|err| {
17414 Error::other(format!(
17415 "failed to encode durable ILM decommission receipt `{receipt_path}` for source path `{}` {}: {err}",
17416 receipt.source_path,
17417 receipt.context()
17418 ))
17419 })?;
17420 match save_config_with_opts(
17421 self.pools[target_pool_idx].clone(),
17422 &receipt_path,
17423 encoded,
17424 &ObjectOptions {
17425 max_parity: true,
17426 write_completion: crate::object_api::WriteCompletion::TailDrained,
17427 http_preconditions: Some(http_preconditions),
17428 ..Default::default()
17429 },
17430 )
17431 .await
17432 {
17433 Ok(()) => return Ok(()),
17434 Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => {
17435 attempt += 1;
17436 }
17437 Err(Error::PreconditionFailed) => {
17438 return Err(Error::other(format!(
17439 "failed to persist durable ILM decommission receipt `{receipt_path}` for {} after concurrent updates",
17440 locator.context()
17441 )));
17442 }
17443 Err(err) => {
17444 return Err(Error::other(format!(
17445 "failed to persist durable ILM decommission receipt `{receipt_path}` for {}: {err}",
17446 locator.context()
17447 )));
17448 }
17449 }
17450 }
17451 }
17452
17453 fn validate_decommission_durable_ilm_receipt_locator(
17454 receipt_path: &str,
17455 locator: &DecommissionDurableIlmReceiptLocator,
17456 receipt: &DecommissionDurableIlmReceipt,
17457 ) -> Result<()> {
17458 if locator.source_path != receipt.source_path || locator.id_kind != receipt.id_kind || locator.id != receipt.id {
17459 return Err(Error::other(format!(
17460 "durable ILM decommission receipt path `{receipt_path}` identity {} does not match receipt {}",
17461 locator.context(),
17462 receipt.context()
17463 )));
17464 }
17465 Ok(())
17466 }
17467
17468 async fn read_decommission_durable_ilm_receipt(
17469 &self,
17470 receipt_pool_idx: usize,
17471 receipt_path: &str,
17472 ) -> Result<DecommissionDurableIlmReceipt> {
17473 let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?;
17474 let data = read_config_limited_preserve_empty(
17475 self.pools[receipt_pool_idx].clone(),
17476 receipt_path,
17477 DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
17478 )
17479 .await
17480 .map_err(|err| {
17481 Error::other(format!(
17482 "failed to read durable ILM decommission receipt `{receipt_path}` from pool {receipt_pool_idx} for {}: {err}",
17483 locator.context()
17484 ))
17485 })?;
17486 let receipt = DecommissionDurableIlmReceipt::decode(&data).map_err(|err| {
17487 Error::other(format!(
17488 "durable ILM decommission receipt `{receipt_path}` in pool {receipt_pool_idx} for {} is invalid: {err}",
17489 locator.context()
17490 ))
17491 })?;
17492 Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?;
17493 Ok(receipt)
17494 }
17495
17496 async fn load_decommission_durable_ilm_terminal_receipt_for_run(
17497 &self,
17498 source_pool_idx: usize,
17499 path: &str,
17500 source_record: &ValidatedDurableIlmRecord,
17501 run_token: &str,
17502 ) -> Result<Option<DecommissionDurableIlmReceipt>> {
17503 let receipt_path = decommission_durable_ilm_receipt_path(run_token, path, source_record.id_kind, &source_record.id);
17504 let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
17505 let mut proof = None::<DecommissionDurableIlmReceipt>;
17506 let mut nonterminal_receipt_found = false;
17507 for pool_idx in 0..self.pools.len() {
17508 if pool_idx == source_pool_idx {
17509 continue;
17510 }
17511 let data = match read_config_limited_preserve_empty(
17512 self.pools[pool_idx].clone(),
17513 &receipt_path,
17514 DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
17515 )
17516 .await
17517 {
17518 Ok(data) => data,
17519 Err(err)
17520 if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17521 || is_err_object_not_found(&err)
17522 || is_err_version_not_found(&err) =>
17523 {
17524 continue;
17525 }
17526 Err(err) => {
17527 return Err(Error::other(format!(
17528 "failed to read terminal durable ILM decommission receipt `{receipt_path}` from pool {pool_idx} for {}: {err}",
17529 source_record.context()
17530 )));
17531 }
17532 };
17533 let receipt = DecommissionDurableIlmReceipt::decode(&data).map_err(|err| {
17534 Error::other(format!(
17535 "terminal durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} for {} is invalid: {err}",
17536 source_record.context()
17537 ))
17538 })?;
17539 Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &receipt)?;
17540 if receipt.namespace != source_record.namespace
17541 || receipt.id_kind != source_record.id_kind
17542 || receipt.id != source_record.id
17543 {
17544 return Err(Error::other(format!(
17545 "terminal durable ILM decommission receipt identity mismatch at path `{path}` {}; receipt {}",
17546 source_record.context(),
17547 receipt.context()
17548 )));
17549 }
17550 if let Some(terminal_checkpoint) = receipt.terminal_checkpoint.as_ref() {
17551 if !source_record.checkpoint.is_predecessor_of_terminal(terminal_checkpoint) {
17552 return Err(Error::other_with_context(
17553 "terminal durable ILM decommission receipt does not cover source",
17554 format!("path `{path}` {}", source_record.context()),
17555 ));
17556 }
17557 proof = Some(match proof {
17558 Some(existing) => merge_decommission_durable_ilm_receipts(&existing, &receipt)?,
17559 None => receipt,
17560 });
17561 } else {
17562 source_record
17563 .checkpoint
17564 .validate_successor(&receipt.checkpoint)
17565 .map_err(|err| {
17566 Error::other_with_context(
17567 "durable ILM decommission receipt does not cover source",
17568 format!("path `{path}` {}: {err}", source_record.context()),
17569 )
17570 })?;
17571 nonterminal_receipt_found = true;
17572 }
17573 }
17574 if nonterminal_receipt_found { Ok(None) } else { Ok(proof) }
17579 }
17580
17581 async fn verify_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> {
17582 let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
17583 self.verify_decommission_durable_ilm_receipts_for_run(source_pool_idx, &run_token)
17584 .await
17585 }
17586
17587 async fn verify_decommission_durable_ilm_receipts_for_run(&self, source_pool_idx: usize, run_token: &str) -> Result<()> {
17588 let expected_receipts = self
17589 .load_decommission_durable_ilm_manifest_for_run(source_pool_idx, run_token)
17590 .await?;
17591 let receipt_paths = self
17592 .list_decommission_durable_ilm_receipts_for_run(source_pool_idx, run_token)
17593 .await?;
17594 let present_receipt_paths = receipt_paths
17595 .iter()
17596 .map(|(_, receipt_path)| receipt_path.as_str())
17597 .collect::<HashSet<_>>();
17598 for (expected_path, expected) in &expected_receipts {
17599 if !present_receipt_paths.contains(expected_path.as_str()) {
17600 return Err(Error::other(format!(
17601 "durable ILM decommission receipt is missing at `{expected_path}` for source path `{}` {}",
17602 expected.source_path,
17603 expected.context()
17604 )));
17605 }
17606 }
17607
17608 for (receipt_pool_idx, receipt_path) in receipt_paths {
17609 let expected = expected_receipts.get(&receipt_path).ok_or_else(|| {
17610 Error::other(format!(
17611 "durable ILM decommission receipt `{receipt_path}` in pool {receipt_pool_idx} is absent from the expected manifest"
17612 ))
17613 })?;
17614 let receipt = self
17615 .read_decommission_durable_ilm_receipt(receipt_pool_idx, &receipt_path)
17616 .await?;
17617 if receipt.source_path != expected.source_path
17618 || receipt.namespace != expected.namespace
17619 || receipt.id_kind != expected.id_kind
17620 || receipt.id != expected.id
17621 {
17622 return Err(Error::other(format!(
17623 "durable ILM decommission receipt identity mismatch at `{receipt_path}` for source path `{}` {}; decoded {}",
17624 expected.source_path,
17625 expected.context(),
17626 receipt.context()
17627 )));
17628 }
17629 expected.checkpoint.validate_successor(&receipt.checkpoint).map_err(|err| {
17630 Error::other(format!(
17631 "durable ILM decommission receipt generation mismatch at `{receipt_path}` for source path `{}` {}: {err}",
17632 expected.source_path,
17633 expected.context()
17634 ))
17635 })?;
17636 match (&expected.terminal_checkpoint, &receipt.terminal_checkpoint) {
17637 (Some(expected_terminal), Some(receipt_terminal)) => {
17638 expected_terminal.validate_successor(receipt_terminal).map_err(|err| {
17639 Error::other(format!(
17640 "durable ILM decommission terminal receipt generation mismatch at `{receipt_path}` for source path `{}` {}: {err}",
17641 expected.source_path,
17642 expected.context()
17643 ))
17644 })?;
17645 }
17646 (Some(_), None) => {
17647 return Err(Error::other(format!(
17648 "durable ILM decommission terminal receipt is missing at `{receipt_path}` for source path `{}` {}",
17649 expected.source_path,
17650 expected.context()
17651 )));
17652 }
17653 (None, _) => {}
17654 }
17655 let namespace = classify_durable_ilm_record(&receipt.source_path)?
17656 .ok_or_else(|| Error::other(format!("path `{}` is not a durable ILM record", receipt.source_path)))?;
17657 let target = self
17658 .load_decommissioned_durable_ilm_target(
17659 source_pool_idx,
17660 &receipt.source_path,
17661 namespace.max_record_size,
17662 &receipt.context(),
17663 )
17664 .await?;
17665 if let Some((_, target)) = target {
17666 let target_record = validate_durable_ilm_record(&receipt.source_path, &target).map_err(|err| {
17667 Error::other(format!(
17668 "target durable ILM record is invalid at path `{}` {}: {err}",
17669 receipt.source_path,
17670 receipt.context()
17671 ))
17672 })?;
17673 let identity_matches = target_record.namespace == receipt.namespace
17674 && target_record.id_kind == receipt.id_kind
17675 && target_record.id == receipt.id;
17676 let reused_manual_scope = receipt.terminal_checkpoint.is_some()
17677 && matches!(
17678 (&receipt.checkpoint, &target_record.checkpoint),
17679 (
17680 DurableIlmRecordCheckpoint::ManualTransitionScope { .. },
17681 DurableIlmRecordCheckpoint::ManualTransitionScope { .. }
17682 )
17683 );
17684 if !identity_matches && !reused_manual_scope {
17685 return Err(Error::other(format!(
17686 "target durable ILM record identity mismatch at path `{}` {}; decoded {}",
17687 receipt.source_path,
17688 receipt.context(),
17689 target_record.context()
17690 )));
17691 }
17692 if identity_matches {
17693 receipt
17694 .terminal_checkpoint
17695 .as_ref()
17696 .unwrap_or(&receipt.checkpoint)
17697 .validate_successor(&target_record.checkpoint)
17698 .map_err(|err| {
17699 Error::other(format!(
17700 "target durable ILM record generation mismatch at path `{}` {}: {err}",
17701 receipt.source_path,
17702 receipt.context()
17703 ))
17704 })?;
17705 }
17706 } else if receipt.terminal_checkpoint.is_none() {
17707 return Err(Error::other(format!(
17708 "target durable ILM record is missing at path `{}` {} without a recovery terminal checkpoint",
17709 receipt.source_path,
17710 receipt.context()
17711 )));
17712 }
17713 }
17714 Ok(())
17715 }
17716
17717 async fn advance_durable_ilm_decommission_receipt(
17718 &self,
17719 pool_idx: usize,
17720 receipt_path: &str,
17721 record: &ValidatedDurableIlmRecord,
17722 fleet_topology_generation: Option<&str>,
17723 terminal: bool,
17724 ) -> Result<bool> {
17725 let stage = if terminal { "terminal" } else { "progress" };
17726 let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?;
17727 let mut attempt = 1;
17728 loop {
17729 let (receipt_data, metadata) = match read_config_limited_preserve_empty_with_metadata(
17730 self.pools[pool_idx].clone(),
17731 receipt_path,
17732 DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
17733 )
17734 .await
17735 {
17736 Ok(receipt) => receipt,
17737 Err(err)
17738 if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17739 || is_err_object_not_found(&err)
17740 || is_err_version_not_found(&err) =>
17741 {
17742 return Ok(false);
17743 }
17744 Err(err) => {
17745 return Err(Error::other(format!(
17746 "failed to read durable ILM decommission receipt `{receipt_path}` from pool {pool_idx} for {}: {err}",
17747 locator.context()
17748 )));
17749 }
17750 };
17751 let mut receipt = DecommissionDurableIlmReceipt::decode(&receipt_data).map_err(|err| {
17752 Error::other(format!(
17753 "durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} for {} is invalid: {err}",
17754 locator.context()
17755 ))
17756 })?;
17757 Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?;
17758 if receipt.fleet_topology_generation.as_deref() != fleet_topology_generation {
17759 return Err(Error::other_with_context(
17760 "durable ILM decommission receipt fleet topology mismatch",
17761 format!("path `{}` {}", receipt.source_path, receipt.context()),
17762 ));
17763 }
17764 receipt.checkpoint.validate_successor(&record.checkpoint).map_err(|err| {
17765 Error::other(format!(
17766 "{stage} durable ILM record generation mismatch at path `{}` {}: {err}",
17767 receipt.source_path,
17768 receipt.context()
17769 ))
17770 })?;
17771
17772 if terminal {
17773 if let Some(existing) = &receipt.terminal_checkpoint {
17774 if existing == &record.checkpoint || record.checkpoint.validate_successor(existing).is_ok() {
17775 return Ok(true);
17776 }
17777 existing.validate_successor(&record.checkpoint).map_err(|err| {
17778 Error::other(format!(
17779 "terminal durable ILM record checkpoint conflicts at path `{}` {}: {err}",
17780 receipt.source_path,
17781 receipt.context()
17782 ))
17783 })?;
17784 }
17785 receipt.terminal_checkpoint = Some(record.checkpoint.clone());
17786 } else {
17787 if let Some(existing) = &receipt.terminal_checkpoint {
17788 if existing == &record.checkpoint {
17789 return Ok(true);
17790 }
17791 existing.validate_successor(&record.checkpoint).map_err(|err| {
17792 Error::other(format!(
17793 "progress durable ILM record conflicts with terminal checkpoint at path `{}` {}: {err}",
17794 receipt.source_path,
17795 receipt.context()
17796 ))
17797 })?;
17798 receipt.terminal_checkpoint = None;
17799 }
17800 if receipt.checkpoint == record.checkpoint {
17801 return Ok(true);
17802 }
17803 receipt.checkpoint = record.checkpoint.clone();
17804 }
17805
17806 let etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| {
17807 Error::other(format!(
17808 "durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} is missing an ETag"
17809 ))
17810 })?;
17811 let encoded = receipt.encode()?;
17812 match save_config_with_opts(
17813 self.pools[pool_idx].clone(),
17814 receipt_path,
17815 encoded,
17816 &ObjectOptions {
17817 max_parity: true,
17818 write_completion: crate::object_api::WriteCompletion::TailDrained,
17819 http_preconditions: Some(HTTPPreconditions {
17820 if_match: Some(etag),
17821 ..Default::default()
17822 }),
17823 ..Default::default()
17824 },
17825 )
17826 .await
17827 {
17828 Ok(()) => return Ok(true),
17829 Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => {
17830 attempt += 1;
17831 continue;
17832 }
17833 Err(Error::PreconditionFailed) => {
17834 return Err(Error::other(format!(
17835 "failed to persist {stage} durable ILM decommission receipt `{receipt_path}` for {} after concurrent updates",
17836 locator.context()
17837 )));
17838 }
17839 Err(err) => {
17840 return Err(Error::other(format!(
17841 "failed to persist {stage} durable ILM decommission receipt `{receipt_path}` for {}: {err}",
17842 locator.context()
17843 )));
17844 }
17845 }
17846 }
17847 }
17848
17849 async fn advance_durable_ilm_decommission_receipts(
17850 &self,
17851 path: &str,
17852 data: &[u8],
17853 terminal: bool,
17854 ) -> Result<Option<Vec<usize>>> {
17855 let active_runs = {
17856 let pool_meta = self.pool_meta.read().await;
17857 pool_meta
17858 .pools
17859 .iter()
17860 .enumerate()
17861 .filter_map(|(pool_idx, pool)| {
17862 pool.decommission
17863 .as_ref()
17864 .filter(|info| info.has_decommission_state() && !info.complete)
17865 .and_then(|info| info.start_time)
17866 .map(|start_time| (pool_idx, decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time)))
17867 })
17868 .collect::<Vec<_>>()
17869 };
17870 if active_runs.is_empty() {
17871 return Ok(None);
17872 }
17873
17874 let stage = if terminal { "terminal" } else { "progress" };
17875 let record = validate_durable_ilm_record(path, data)
17876 .map_err(|err| Error::other(format!("{stage} durable ILM record is invalid at path `{path}`: {err}")))?;
17877 let fleet_topology_generation = durable_ilm_v6_topology_generation(path, data)?;
17878 let active_source_pool_indices = active_runs.iter().map(|(pool_idx, _)| *pool_idx).collect::<Vec<_>>();
17879 let mut terminal_target_pool_indices = Vec::new();
17880 for (source_pool_idx, run_token) in active_runs {
17881 let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, record.id_kind, &record.id);
17882 let mut receipt_found = false;
17883 for pool_idx in 0..self.pools.len() {
17884 if pool_idx != source_pool_idx {
17885 let found = self
17886 .advance_durable_ilm_decommission_receipt(
17887 pool_idx,
17888 &receipt_path,
17889 &record,
17890 fleet_topology_generation.as_deref(),
17891 terminal,
17892 )
17893 .await?;
17894 receipt_found |= found;
17895 if terminal
17896 && found
17897 && !active_source_pool_indices.contains(&pool_idx)
17898 && !terminal_target_pool_indices.contains(&pool_idx)
17899 {
17900 terminal_target_pool_indices.push(pool_idx);
17901 }
17902 }
17903 }
17904 if terminal && !receipt_found {
17905 return Err(Error::other(format!(
17906 "terminal durable ILM record at path `{path}` {} is retained until its decommission receipt is committed",
17907 record.context()
17908 )));
17909 }
17910 }
17911 Ok(Some(terminal_target_pool_indices))
17912 }
17913
17914 pub(crate) async fn decommission_durable_ilm_checkpoint_targets(
17922 &self,
17923 path: &str,
17924 next_data: &[u8],
17925 expected_etag: &str,
17926 ) -> Result<Option<Vec<DecommissionDurableIlmCheckpointTarget>>> {
17927 let active_runs = {
17928 let pool_meta = self.pool_meta.read().await;
17929 let mut active_runs = Vec::new();
17930 for (source_pool_index, pool) in pool_meta.pools.iter().enumerate() {
17931 let Some(info) = pool
17932 .decommission
17933 .as_ref()
17934 .filter(|info| info.has_decommission_state() && !info.complete)
17935 else {
17936 continue;
17937 };
17938 let Some(start_time) = info.start_time else {
17939 continue;
17940 };
17941 let reservation = info.capacity_reservation.clone().ok_or_else(|| {
17942 decommission_capacity_blocked_error(format!(
17943 "active decommission source pool {source_pool_index} has no reservation for durable ILM checkpoint"
17944 ))
17945 })?;
17946 if !reservation.active() {
17947 return Err(decommission_capacity_blocked_error(format!(
17948 "active decommission source pool {source_pool_index} has a released reservation for durable ILM checkpoint"
17949 )));
17950 }
17951 active_runs.push((
17952 source_pool_index,
17953 decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time),
17954 reservation,
17955 ));
17956 }
17957 active_runs
17958 };
17959 if active_runs.is_empty() {
17960 return Ok(None);
17961 }
17962
17963 let next_record = validate_durable_ilm_record(path, next_data)?;
17964 if next_record.namespace != TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.name {
17965 return Ok(None);
17966 }
17967 let next_fleet_topology_generation = durable_ilm_v6_topology_generation(path, next_data)?;
17968
17969 let mut targets = Vec::<DecommissionDurableIlmCheckpointTarget>::new();
17970 let mut missing_source_receipts = 0usize;
17971 let mut stale_target_etag_mismatch = false;
17972 for (source_pool_index, run_token, reservation) in active_runs {
17973 let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, next_record.id_kind, &next_record.id);
17974 let mut source_receipt_found = false;
17975 for allocation in &reservation.targets {
17976 let receipt_data = match read_config_limited_preserve_empty(
17977 self.pools[allocation.pool_index].clone(),
17978 &receipt_path,
17979 DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
17980 )
17981 .await
17982 {
17983 Ok(data) => data,
17984 Err(err)
17985 if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17986 || is_err_object_not_found(&err)
17987 || is_err_version_not_found(&err) =>
17988 {
17989 continue;
17990 }
17991 Err(err) => return Err(err),
17992 };
17993 let receipt = DecommissionDurableIlmReceipt::decode(&receipt_data)?;
17994 let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
17995 Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &receipt)?;
17996 if receipt.source_path != path
17997 || receipt.namespace != next_record.namespace
17998 || receipt.id_kind != next_record.id_kind
17999 || receipt.id != next_record.id
18000 {
18001 return Err(Error::other_with_context(
18002 "durable ILM checkpoint receipt identity does not authorize source",
18003 format!("path `{path}` {}", next_record.context()),
18004 ));
18005 }
18006 if receipt.fleet_topology_generation != next_fleet_topology_generation {
18007 return Err(Error::other_with_context(
18008 "durable ILM checkpoint receipt fleet topology does not authorize source",
18009 format!("path `{path}` {}", next_record.context()),
18010 ));
18011 }
18012 receipt
18013 .checkpoint
18014 .validate_successor(&next_record.checkpoint)
18015 .map_err(|err| {
18016 Error::other_with_context(
18017 "durable ILM checkpoint receipt is not a predecessor of the requested generation",
18018 format!(
18019 "target pool {}, path `{path}` {}; receipt checkpoint {:?}, requested checkpoint {:?}: {err}",
18020 allocation.pool_index,
18021 next_record.context(),
18022 receipt.checkpoint,
18023 next_record.checkpoint
18024 ),
18025 )
18026 })?;
18027
18028 let (target_data, metadata) = read_config_limited_preserve_empty_with_metadata(
18029 self.pools[allocation.pool_index].clone(),
18030 path,
18031 TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.max_record_size,
18032 )
18033 .await?;
18034 let target_record = validate_durable_ilm_record(path, &target_data)?;
18035 if target_record.namespace != next_record.namespace
18036 || target_record.id_kind != next_record.id_kind
18037 || target_record.id != next_record.id
18038 {
18039 return Err(Error::other_with_context(
18040 "durable ILM checkpoint target identity does not match source",
18041 format!("path `{path}` {}", next_record.context()),
18042 ));
18043 }
18044 let already_committed = target_data.as_slice() == next_data;
18045 let target_etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| {
18046 Error::other_with_context("durable ILM checkpoint target is missing an ETag", format!("path `{path}`"))
18047 })?;
18048 if already_committed {
18049 receipt
18050 .checkpoint
18051 .validate_successor(&target_record.checkpoint)
18052 .map_err(|err| {
18053 Error::other_with_context(
18054 "durable ILM checkpoint receipt is not a predecessor of the committed target generation",
18055 format!("path `{path}` {}: {err}", next_record.context()),
18056 )
18057 })?;
18058 } else {
18059 if target_record.checkpoint != receipt.checkpoint {
18060 return Err(Error::other_with_context(
18061 "durable ILM checkpoint target is not the receipt generation",
18062 format!("path `{path}` {}", next_record.context()),
18063 ));
18064 }
18065 target_record
18066 .checkpoint
18067 .validate_successor(&next_record.checkpoint)
18068 .map_err(|err| {
18069 Error::other_with_context(
18070 "durable ILM checkpoint target is not a predecessor of the requested generation",
18071 format!("path `{path}` {}: {err}", next_record.context()),
18072 )
18073 })?;
18074 if next_data.len() > target_data.len() {
18075 return Err(decommission_capacity_blocked_error(format!(
18076 "durable ILM checkpoint update at `{path}` grows from {} to {} bytes",
18077 target_data.len(),
18078 next_data.len()
18079 )));
18080 }
18081 stale_target_etag_mismatch |= target_etag != expected_etag;
18082 }
18083
18084 source_receipt_found = true;
18085 if let Some(existing) = targets
18086 .iter_mut()
18087 .find(|target| target.target_pool_index == allocation.pool_index)
18088 {
18089 if existing.already_committed != already_committed {
18090 return Err(Error::other_with_context(
18091 "durable ILM checkpoint target state changed during authorization",
18092 format!("path `{path}`"),
18093 ));
18094 }
18095 continue;
18096 }
18097 let base_owner = DecommissionCapacityOwner {
18098 source_pool_index,
18099 operation_id: reservation.operation_id,
18100 generation: reservation.generation,
18101 owner_nonce: reservation.owner_nonce,
18102 mutation_id: None,
18103 };
18104 let mutation_id = decommission_capacity_mutation_id(
18105 base_owner,
18106 RUSTFS_META_BUCKET,
18107 path,
18108 Some(next_record.checkpoint.content_sha256()),
18109 false,
18110 None,
18111 );
18112 targets.push(DecommissionDurableIlmCheckpointTarget {
18113 source_pool_index,
18114 target_pool_index: allocation.pool_index,
18115 capacity_owner: base_owner.with_mutation_id(mutation_id),
18116 already_committed,
18117 target_etag: Some(target_etag),
18118 });
18119 }
18120 if !source_receipt_found {
18121 missing_source_receipts = missing_source_receipts.saturating_add(1);
18122 }
18123 }
18124
18125 if missing_source_receipts > 0 {
18126 return Err(decommission_capacity_blocked_error(format!(
18127 "durable ILM checkpoint at `{path}` is missing receipt coverage for {missing_source_receipts} active source(s)"
18128 )));
18129 }
18130 if targets.is_empty() {
18131 return Err(decommission_capacity_blocked_error(format!(
18132 "durable ILM checkpoint at `{path}` has no receipt-bearing reservation target"
18133 )));
18134 }
18135 if stale_target_etag_mismatch && !targets.iter().any(|target| target.already_committed) {
18136 return Err(Error::PreconditionFailed);
18137 }
18138 targets.sort_unstable_by_key(|target| target.target_pool_index);
18145 Ok(Some(targets))
18146 }
18147
18148 pub(crate) async fn record_durable_ilm_decommission_progress(&self, path: &str, data: &[u8]) -> Result<()> {
18149 self.advance_durable_ilm_decommission_receipts(path, data, false)
18150 .await
18151 .map(|_| ())
18152 }
18153
18154 pub(crate) async fn record_durable_ilm_decommission_terminal(&self, path: &str, data: &[u8]) -> Result<()> {
18155 self.record_durable_ilm_decommission_terminal_target_pools(path, data)
18156 .await
18157 .map(|_| ())
18158 }
18159
18160 pub(crate) async fn record_durable_ilm_decommission_terminal_target_pools(
18162 &self,
18163 path: &str,
18164 data: &[u8],
18165 ) -> Result<Option<Vec<usize>>> {
18166 self.advance_durable_ilm_decommission_receipts(path, data, true).await
18167 }
18168
18169 pub(crate) async fn durable_ilm_terminal_receipt_covers_active_source(&self, path: &str, data: &[u8]) -> Result<bool> {
18174 let namespace = classify_durable_ilm_record(path)?
18175 .ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{path}`")))?;
18176 let source_record = validate_durable_ilm_record(path, data)?;
18177 let active_runs = {
18178 let pool_meta = self.pool_meta.read().await;
18179 pool_meta
18180 .pools
18181 .iter()
18182 .enumerate()
18183 .filter_map(|(source_pool_index, pool)| {
18184 pool.decommission
18185 .as_ref()
18186 .filter(|info| info.has_decommission_state() && !info.complete)
18187 .and_then(|info| info.start_time)
18188 .map(|start_time| {
18189 (source_pool_index, decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time))
18190 })
18191 })
18192 .collect::<Vec<_>>()
18193 };
18194 let mut covered_active_source = false;
18195 for (source_pool_index, run_token) in active_runs {
18196 let source_data =
18197 match read_config_limited_preserve_empty(self.pools[source_pool_index].clone(), path, namespace.max_record_size)
18198 .await
18199 {
18200 Ok(source_data) => source_data,
18201 Err(err)
18202 if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
18203 || is_err_object_not_found(&err)
18204 || is_err_version_not_found(&err) =>
18205 {
18206 continue;
18207 }
18208 Err(err) => return Err(err),
18209 };
18210 if source_data.as_slice() != data {
18211 continue;
18212 }
18213 if self
18214 .load_decommission_durable_ilm_terminal_receipt_for_run(source_pool_index, path, &source_record, &run_token)
18215 .await?
18216 .is_some()
18217 {
18218 covered_active_source = true;
18219 } else {
18220 return Ok(false);
18224 }
18225 }
18226 Ok(covered_active_source)
18227 }
18228
18229 async fn cleanup_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> {
18230 for (pool_idx, receipt_path) in self.list_decommission_durable_ilm_receipts(source_pool_idx).await? {
18231 match delete_config(self.pools[pool_idx].clone(), &receipt_path).await {
18232 Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {}
18233 Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
18234 Err(err) => {
18235 return Err(Error::other(format!(
18236 "failed to clean durable ILM decommission receipt `{receipt_path}` from pool {pool_idx}: {err}"
18237 )));
18238 }
18239 }
18240 }
18241 for receipt_path in self.list_decommission_durable_ilm_manifest_receipts(source_pool_idx).await? {
18242 match delete_config(self.pools[source_pool_idx].clone(), &receipt_path).await {
18243 Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {}
18244 Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
18245 Err(err) => {
18246 return Err(Error::other(format!(
18247 "failed to clean durable ILM expected manifest receipt `{receipt_path}` from source pool {source_pool_idx}: {err}"
18248 )));
18249 }
18250 }
18251 }
18252 let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
18253 let manifest_path = decommission_durable_ilm_manifest_path(&run_token);
18254 match delete_config(self.pools[source_pool_idx].clone(), &manifest_path).await {
18255 Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {}
18256 Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
18257 Err(err) => {
18258 return Err(Error::other(format!(
18259 "failed to clean durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}"
18260 )));
18261 }
18262 }
18263 Ok(())
18264 }
18265
18266 #[cfg(all(test, feature = "test-util"))]
18267 async fn verify_and_cleanup_decommissioned_durable_ilm_record(
18268 &self,
18269 source_pool_idx: usize,
18270 source_set: Arc<SetDisks>,
18271 path: &str,
18272 ) -> Result<()> {
18273 let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
18274 self.verify_and_cleanup_decommissioned_durable_ilm_record_for_run(source_pool_idx, source_set, path, &run_token)
18275 .await
18276 }
18277
18278 async fn verify_and_cleanup_decommissioned_durable_ilm_record_for_run(
18279 &self,
18280 source_pool_idx: usize,
18281 source_set: Arc<SetDisks>,
18282 path: &str,
18283 run_token: &str,
18284 ) -> Result<()> {
18285 let namespace = classify_durable_ilm_record(path)?
18286 .ok_or_else(|| Error::other(format!("path `{path}` is not a durable ILM record")))?;
18287 let source_versions = source_set
18288 .load_file_info_versions_exact(RUSTFS_META_BUCKET, path)
18289 .await
18290 .map_err(|err| Error::other(format!("failed to load source durable ILM versions at path `{path}`: {err}")))?
18291 .ok_or_else(|| Error::other(format!("source durable ILM record is missing at path `{path}`")))?;
18292 let source = read_config_limited_preserve_empty(source_set.clone(), path, namespace.max_record_size)
18293 .await
18294 .map_err(|err| Error::other(format!("failed to read source durable ILM record at path `{path}`: {err}")))?;
18295 let source_record = validate_durable_ilm_record(path, &source)
18296 .map_err(|err| Error::other(format!("source durable ILM record is invalid at path `{path}`: {err}")))?;
18297 let source_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &source)?;
18298 let target = self
18299 .load_decommissioned_durable_ilm_target(source_pool_idx, path, namespace.max_record_size, &source_record.context())
18300 .await?;
18301 let manifest_receipt = if let Some((target_pool_idx, target)) = target {
18302 let target_record = validate_decommission_durable_ilm_copy(path, &source_record, &target)?;
18303 let target_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &target)?;
18304 if target_fleet_topology_generation != source_fleet_topology_generation {
18305 return Err(Error::other_with_context(
18306 "target durable ILM fleet topology generation differs from source",
18307 format!("path `{path}` {}", source_record.context()),
18308 ));
18309 }
18310 let receipt = DecommissionDurableIlmReceipt::new(path, &target_record, target_fleet_topology_generation);
18311 self.persist_decommission_durable_ilm_receipt_for_run(target_pool_idx, &receipt, run_token)
18312 .await?;
18313 receipt
18314 } else {
18315 self.load_decommission_durable_ilm_terminal_receipt_for_run(source_pool_idx, path, &source_record, run_token)
18316 .await?
18317 .ok_or_else(|| {
18318 Error::other(format!(
18319 "target durable ILM record is missing at path `{path}` {} without a matching terminal receipt",
18320 source_record.context()
18321 ))
18322 })?
18323 };
18324 if manifest_receipt.fleet_topology_generation != source_fleet_topology_generation {
18325 return Err(Error::other_with_context(
18326 "terminal durable ILM receipt fleet topology generation does not cover source",
18327 format!("path `{path}` {}", source_record.context()),
18328 ));
18329 }
18330 let fleet_proof = if let Some(expected_generation) = source_fleet_topology_generation.as_deref() {
18331 let proof = acquire_tier_delete_journal_fleet_proof()
18332 .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable for source cleanup"))?;
18333 if tier_delete_journal_topology_generation(&proof) != expected_generation
18334 || !tier_delete_journal_fleet_proof_matches(&proof)
18335 {
18336 return Err(Error::other("tier delete journal v6 fleet generation changed before source cleanup"));
18337 }
18338 Some(proof)
18339 } else {
18340 None
18341 };
18342 self.persist_decommission_durable_ilm_receipt_for_run(source_pool_idx, &manifest_receipt, run_token)
18343 .await?;
18344
18345 if fleet_proof
18346 .as_ref()
18347 .is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof))
18348 {
18349 return Err(Error::other("tier delete journal v6 fleet proof changed before source cleanup"));
18350 }
18351
18352 let cleanup_result = data_movement::cleanup_source_entry_if_unchanged(
18353 source_set,
18354 RUSTFS_META_BUCKET,
18355 path,
18356 &source_versions,
18357 &[],
18358 data_movement::SourceCleanupBucketFence::default(),
18359 "decommission durable ILM final sweep",
18360 )
18361 .await
18362 .map_err(|err| {
18363 Error::other(format!(
18364 "source durable ILM cleanup failed at path `{path}` {}: {err}",
18365 source_record.context()
18366 ))
18367 });
18368 let cleanup_result = resolve_decommission_entry_cleanup_delete_result(cleanup_result, RUSTFS_META_BUCKET, path);
18369 if fleet_proof
18370 .as_ref()
18371 .is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof))
18372 {
18373 return Err(Error::other(
18374 "tier delete journal v6 fleet proof changed during source cleanup; exact source outcome requires verification",
18375 ));
18376 }
18377 cleanup_result
18378 }
18379
18380 #[cfg(all(test, feature = "test-util"))]
18381 pub(crate) async fn verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
18382 &self,
18383 source_pool_idx: usize,
18384 source_set: Arc<SetDisks>,
18385 path: &str,
18386 ) -> Result<()> {
18387 self.verify_and_cleanup_decommissioned_durable_ilm_record(source_pool_idx, source_set, path)
18388 .await
18389 }
18390
18391 #[cfg(all(test, feature = "test-util"))]
18392 pub(crate) async fn decommission_durable_ilm_receipt_count_for_test(&self, source_pool_idx: usize) -> Result<usize> {
18393 Ok(self.list_decommission_durable_ilm_receipts(source_pool_idx).await?.len())
18394 }
18395
18396 #[cfg(all(test, feature = "test-util"))]
18397 pub(crate) async fn decommission_durable_ilm_receipt_paths_for_test(
18398 &self,
18399 source_pool_idx: usize,
18400 ) -> Result<Vec<(usize, String)>> {
18401 self.list_decommission_durable_ilm_receipts(source_pool_idx).await
18402 }
18403
18404 #[cfg(all(test, feature = "test-util"))]
18405 pub(crate) async fn persist_decommission_durable_ilm_receipt_for_test(
18406 &self,
18407 source_pool_idx: usize,
18408 target_pool_idx: usize,
18409 source_path: &str,
18410 record: &ValidatedDurableIlmRecord,
18411 terminal: bool,
18412 ) -> Result<String> {
18413 let fleet_topology_generation = match read_config_limited_preserve_empty(
18414 self.pools[target_pool_idx].clone(),
18415 source_path,
18416 classify_durable_ilm_record(source_path)?
18417 .ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{source_path}`")))?
18418 .max_record_size,
18419 )
18420 .await
18421 {
18422 Ok(target_data) => durable_ilm_v6_topology_generation(source_path, &target_data)?,
18423 Err(err)
18424 if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
18425 || is_err_object_not_found(&err)
18426 || is_err_version_not_found(&err) =>
18427 {
18428 None
18429 }
18430 Err(err) => return Err(err),
18431 };
18432 let mut receipt = DecommissionDurableIlmReceipt::new(source_path, record, fleet_topology_generation);
18433 if terminal {
18434 receipt.terminal_checkpoint = Some(record.checkpoint.clone());
18435 }
18436 self.persist_decommission_durable_ilm_receipt(source_pool_idx, target_pool_idx, &receipt)
18437 .await?;
18438 self.decommission_durable_ilm_receipt_path_for_test(source_pool_idx, source_path, record)
18439 .await
18440 }
18441
18442 #[cfg(all(test, feature = "test-util"))]
18443 pub(crate) async fn decommission_durable_ilm_receipt_path_for_test(
18444 &self,
18445 source_pool_idx: usize,
18446 source_path: &str,
18447 record: &ValidatedDurableIlmRecord,
18448 ) -> Result<String> {
18449 let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
18450 Ok(decommission_durable_ilm_receipt_path(&run_token, source_path, record.id_kind, &record.id))
18451 }
18452
18453 #[cfg(all(test, feature = "test-util"))]
18454 pub(crate) async fn persist_decommission_durable_ilm_manifest_for_test(&self, source_pool_idx: usize) -> Result<()> {
18455 self.persist_decommission_durable_ilm_manifest(source_pool_idx).await
18456 }
18457
18458 #[cfg(all(test, feature = "test-util"))]
18459 pub(crate) async fn cleanup_decommission_durable_ilm_receipts_for_test(&self, source_pool_idx: usize) -> Result<()> {
18460 self.cleanup_decommission_durable_ilm_receipts(source_pool_idx).await
18461 }
18462
18463 async fn check_after_decommission(
18464 self: &Arc<Self>,
18465 idx: usize,
18466 rx: &CancellationToken,
18467 generation: OffsetDateTime,
18468 ) -> Result<Vec<DecommissionUnresolvedEntry>> {
18469 let run_token = self.durable_ilm_receipt_run_token_for_generation(idx, generation).await?;
18470 let operation_gate = self.ctx.data_movement_operation_gate();
18471 self.run_guarded_decommission_side_effect(rx, &operation_gate, || {
18472 self.check_after_decommission_unfenced(idx, generation, run_token)
18473 })
18474 .await
18475 }
18476
18477 async fn check_after_decommission_unfenced(
18478 self: &Arc<Self>,
18479 idx: usize,
18480 generation: OffsetDateTime,
18481 run_token: String,
18482 ) -> Result<Vec<DecommissionUnresolvedEntry>> {
18483 let unresolved_entries = {
18484 let pool_meta = self.pool_meta.read().await;
18485 ensure_decommission_generation(&pool_meta, idx, generation)?;
18486 let info = pool_meta.pools[idx]
18487 .decommission
18488 .as_ref()
18489 .ok_or_else(|| decommission_metadata_not_initialized_error("verify unresolved decommission entries"))?;
18490 if info
18491 .unresolved_entries
18492 .iter()
18493 .any(|entry| entry.pool_index != idx || entry.source_generation != generation)
18494 {
18495 return Err(Error::other(format!(
18496 "failed to verify decommission for pool {idx}: unresolved listing ledger contains a different pool or generation"
18497 )));
18498 }
18499 info.unresolved_entries.clone()
18500 };
18501 let unresolved_entries_by_identity = Arc::new(
18502 unresolved_entries
18503 .iter()
18504 .map(|entry| (decommission_unresolved_entry_identity(entry), entry.clone()))
18505 .collect::<HashMap<_, _>>(),
18506 );
18507 let resolved_unresolved_entries = Arc::new(tokio::sync::Mutex::new(Vec::new()));
18508 let buckets = self.get_buckets_to_decommission().await?;
18509 let pool = self.pools[idx].clone();
18510
18511 self.ensure_decommission_multipart_uploads_drained(idx, &pool, &buckets)
18512 .await?;
18513
18514 for (set_index, set) in pool.disk_set.iter().enumerate() {
18515 let require_all_disks = unresolved_entries.iter().any(|entry| entry.set_index == set_index);
18516 for bucket_info in &buckets {
18517 let mut lifecycle_config = None;
18518 let mut object_lock_config = None;
18519 let mut replication_configured = false;
18520 if bucket_info.name != RUSTFS_META_BUCKET {
18521 let expiry_configs = get_expiry_configs(self, &bucket_info.name).await?;
18522 lifecycle_config = expiry_configs.lifecycle.map(|config| (*config).clone());
18523 object_lock_config = expiry_configs.object_lock.map(|config| (*config).clone());
18524 replication_configured = resolve_decommission_optional_bucket_config_result(
18525 &bucket_info.name,
18526 "replication",
18527 metadata_sys::get_replication_config_in(&self.ctx, &bucket_info.name).await,
18528 )?
18529 .is_some();
18530 }
18531
18532 let versions_found = Arc::new(AtomicUsize::new(0));
18533 let entry_error = Arc::new(tokio::sync::Mutex::new(None::<Error>));
18534 let first_remaining_path = Arc::new(tokio::sync::Mutex::new(None::<String>));
18535 let callback_rx = CancellationToken::new();
18536 let versions_found_cb = versions_found.clone();
18537 let entry_error_cb = entry_error.clone();
18538 let first_remaining_path_cb = first_remaining_path.clone();
18539 let bucket_name = bucket_info.name.clone();
18540 let lifecycle_config_cb = lifecycle_config.clone();
18541 let object_lock_config_cb = object_lock_config.clone();
18542 let replication_configured_cb = replication_configured;
18543 let store = Arc::clone(self);
18544 let source_set = set.clone();
18545 let callback_rx_cb = callback_rx.clone();
18546 let unresolved_entries_by_identity_cb = unresolved_entries_by_identity.clone();
18547 let resolved_unresolved_entries_cb = resolved_unresolved_entries.clone();
18548 let run_token_cb = run_token.clone();
18549
18550 let callback: ListCallback = Arc::new(move |entry: MetaCacheEntry| {
18551 let versions_found = versions_found_cb.clone();
18552 let entry_error = entry_error_cb.clone();
18553 let first_remaining_path = first_remaining_path_cb.clone();
18554 let bucket_name = bucket_name.clone();
18555 let lifecycle_config = lifecycle_config_cb.clone();
18556 let object_lock_config = object_lock_config_cb.clone();
18557 let replication_configured = replication_configured_cb;
18558 let store = Arc::clone(&store);
18559 let source_set = source_set.clone();
18560 let callback_rx = callback_rx_cb.clone();
18561 let unresolved_entries_by_identity = unresolved_entries_by_identity_cb.clone();
18562 let resolved_unresolved_entries = resolved_unresolved_entries_cb.clone();
18563 let run_token = run_token_cb.clone();
18564 Box::pin(async move {
18565 if callback_rx.is_cancelled() {
18566 return;
18567 }
18568
18569 if !entry.is_object() {
18570 return;
18571 }
18572
18573 if is_decommission_set_local_usage_cache(&bucket_name, &entry.name) {
18574 return;
18575 }
18576
18577 let durable_ilm_record = if bucket_name == RUSTFS_META_BUCKET {
18578 match classify_durable_ilm_record(&entry.name) {
18579 Ok(record) => record,
18580 Err(err) => {
18581 let mut first_err = entry_error.lock().await;
18582 if first_err.is_none() {
18583 *first_err = Some(with_decommission_entry_context(
18584 "check_after_decommission.durable_ilm_namespace",
18585 &bucket_name,
18586 &entry.name,
18587 err,
18588 ));
18589 callback_rx.cancel();
18590 }
18591 return;
18592 }
18593 }
18594 } else {
18595 None
18596 };
18597
18598 if durable_ilm_record.is_some() {
18599 if let Err(err) = store
18600 .verify_and_cleanup_decommissioned_durable_ilm_record_for_run(
18601 idx,
18602 source_set,
18603 &entry.name,
18604 &run_token,
18605 )
18606 .await
18607 {
18608 let mut first_err = entry_error.lock().await;
18609 if first_err.is_none() {
18610 *first_err = Some(err);
18611 callback_rx.cancel();
18612 }
18613 }
18614 return;
18615 }
18616
18617 let mut fivs = match load_decommission_entry_exact_versions(
18618 &source_set,
18619 &entry,
18620 &bucket_name,
18621 "check_after_decommission.file_info_versions",
18622 )
18623 .await
18624 {
18625 Ok(fivs) => fivs,
18626 Err(err) => {
18627 let mut first_err = entry_error.lock().await;
18628 if first_err.is_none() {
18629 *first_err = Some(err);
18630 callback_rx.cancel();
18631 }
18632 return;
18633 }
18634 };
18635
18636 fivs.versions
18637 .sort_by_key(|version| (version.mod_time.is_none(), std::cmp::Reverse(version.mod_time)));
18638
18639 let mut remaining = 0;
18640 let mut expired = 0;
18641 for version in fivs.versions.iter().chain(fivs.free_versions.iter()) {
18642 if version.tier_free_version() {
18643 remaining += 1;
18644 debug!(
18645 event = EVENT_DECOMMISSION_ENTRY,
18646 component = LOG_COMPONENT_ECSTORE,
18647 subsystem = LOG_SUBSYSTEM_POOLS,
18648 pool_index = idx,
18649 bucket = %bucket_name,
18650 object = %entry.name,
18651 version_id = ?version.version_id,
18652 reason = DECOMMISSION_FREE_VERSION_SWEEP_REASON,
18653 state = "free_version_retained",
18654 "Decommission final sweep retained a free version"
18655 );
18656 continue;
18657 }
18658 let skip_lifecycle = match should_skip_lifecycle_for_data_movement(
18659 Arc::clone(&store),
18660 &bucket_name,
18661 version,
18662 lifecycle_config.as_ref(),
18663 object_lock_config.as_ref(),
18664 false,
18665 &LcEventSrc::Decom,
18666 None,
18667 )
18668 .await
18669 {
18670 Ok(skip_lifecycle) => skip_lifecycle,
18671 Err(err) => {
18672 let mut first_err = entry_error.lock().await;
18673 if first_err.is_none() {
18674 *first_err = Some(err);
18675 callback_rx.cancel();
18676 }
18677 return;
18678 }
18679 };
18680 if skip_lifecycle {
18681 expired += 1;
18682 continue;
18683 }
18684 let remaining_versions = decommission_remaining_version_count(&fivs.versions, expired);
18685 if should_skip_decommission_delete_marker(version, remaining_versions, replication_configured) {
18686 continue;
18687 }
18688 remaining += 1;
18689 }
18690
18691 if remaining > 0 {
18692 let mut first_path = first_remaining_path.lock().await;
18693 if first_path.is_none() {
18694 *first_path = Some(format!("{bucket_name}/{}", entry.name));
18695 }
18696 } else {
18697 let identity = (set_index, bucket_name.clone(), entry.name.clone());
18698 if let Some(unresolved_entry) = unresolved_entries_by_identity.get(&identity) {
18699 let mut resolved = resolved_unresolved_entries.lock().await;
18700 if !resolved.contains(unresolved_entry) {
18701 resolved.push(unresolved_entry.clone());
18702 }
18703 }
18704 }
18705
18706 versions_found.fetch_add(remaining, Ordering::Relaxed);
18707 })
18708 });
18709
18710 let list_result = set
18711 .list_objects_to_decommission(
18712 self.clone(),
18713 callback_rx,
18714 bucket_info.clone(),
18715 callback,
18716 entry_error.clone(),
18717 idx,
18718 set_index,
18719 generation,
18720 require_all_disks,
18721 )
18722 .await;
18723 let entry_error = entry_error.lock().await.clone();
18724 resolve_decommission_check_after_list_result(list_result, entry_error)?;
18725
18726 let versions_found = versions_found.load(Ordering::Relaxed);
18727 if versions_found > 0 {
18728 let first_remaining_path = first_remaining_path
18729 .lock()
18730 .await
18731 .clone()
18732 .unwrap_or_else(|| format!("{}/<unknown>", bucket_info.name));
18733 return Err(Error::other(format!(
18734 "at least {versions_found} object(s)/version(s) were found in bucket `{}` after decommissioning; first remaining path `{first_remaining_path}`",
18735 bucket_info.name,
18736 )));
18737 }
18738 }
18739 }
18740
18741 let mut verified_unresolved_entries = resolved_unresolved_entries.lock().await.clone();
18742 for entry in &unresolved_entries {
18743 let set = pool.disk_set.get(entry.set_index).ok_or_else(|| {
18744 Error::other(format!(
18745 "failed to verify decommission for pool {idx}: unresolved listing entry references missing set {}",
18746 entry.set_index
18747 ))
18748 })?;
18749 if set.decommission_unresolved_entry_absent_on_all_disks(idx, entry).await?
18750 && !verified_unresolved_entries.contains(entry)
18751 {
18752 verified_unresolved_entries.push(entry.clone());
18753 }
18754 if !verified_unresolved_entries.contains(entry) {
18755 return Err(Error::other(format!(
18756 "failed to verify decommission for pool {idx}: unresolved listing entry {}/{} in set {} was neither re-observed as resolved nor absent on every source disk",
18757 entry.bucket, entry.object, entry.set_index
18758 )));
18759 }
18760 }
18761
18762 self.persist_decommission_durable_ilm_manifest_for_run(idx, &run_token)
18763 .await?;
18764 self.verify_decommission_durable_ilm_receipts_for_run(idx, &run_token).await?;
18765
18766 Ok(verified_unresolved_entries)
18767 }
18768
18769 async fn ensure_decommission_multipart_uploads_drained(
18770 &self,
18771 idx: usize,
18772 pool: &Sets,
18773 buckets: &[DecomBucketInfo],
18774 ) -> Result<()> {
18775 let mut bucket_names = buckets
18776 .iter()
18777 .filter(|bucket| bucket.name != RUSTFS_META_BUCKET)
18778 .map(|bucket| bucket.name.as_str())
18779 .collect::<Vec<_>>();
18780 bucket_names.sort_unstable();
18781 bucket_names.dedup();
18782
18783 for bucket in bucket_names {
18786 let lifecycle_guard = self.acquire_bucket_lifecycle_write_lock(bucket).await?;
18787 if lifecycle_guard.is_lock_lost() {
18788 return Err(Error::other(format!(
18789 "decommission multipart drain lost the bucket lifecycle fence for `{bucket}`"
18790 )));
18791 }
18792 for set in &pool.disk_set {
18793 if let Some(upload_path) = set.first_multipart_upload_path_for_decommission(bucket).await? {
18794 return Err(Error::other(format!(
18795 "pool {idx} still contains multipart upload `{upload_path}` for bucket `{bucket}`; resolve it before retrying decommission"
18796 )));
18797 }
18798 }
18799 }
18800
18801 Ok(())
18802 }
18803
18804 #[cfg(test)]
18805 pub(crate) async fn ensure_decommission_multipart_uploads_drained_for_test(self: &Arc<Self>, idx: usize) -> Result<()> {
18806 let buckets = self.get_buckets_to_decommission().await?;
18807 let pool = self.pools[idx].clone();
18808 self.ensure_decommission_multipart_uploads_drained(idx, pool.as_ref(), &buckets)
18809 .await
18810 }
18811 #[cfg(all(test, feature = "test-util"))]
18812 pub(crate) async fn check_after_decommission_for_test(self: &Arc<Self>, idx: usize) -> Result<()> {
18813 let generation = self.active_decommission_generation(idx).await?;
18814 self.check_after_decommission(idx, &CancellationToken::new(), generation)
18815 .await
18816 .map(|_| ())
18817 }
18818
18819 #[tracing::instrument(skip(self, rd))]
18820 async fn decommission_object(
18821 self: Arc<Self>,
18822 pool_idx: usize,
18823 bucket: String,
18824 rd: GetObjectReader,
18825 expected_bucket_incarnation_id: Option<uuid::Uuid>,
18826 capacity_owner: Option<DecommissionCapacityOwner>,
18827 ) -> Result<()> {
18828 warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name);
18829 let object_name = rd.object_info.name.clone();
18830 let mut migration = tokio::task::JoinSet::new();
18831 migration.spawn(data_movement::migrate_decommission_object(
18832 self,
18833 pool_idx,
18834 bucket.clone(),
18835 rd,
18836 expected_bucket_incarnation_id,
18837 "decommission_object",
18838 capacity_owner,
18839 ));
18840 let result = migration
18841 .join_next()
18842 .await
18843 .ok_or_else(|| Error::other("decommission migration task was not started"))?
18844 .map_err(|err| Error::other(format!("decommission migration task join error: {err}")))?;
18845 if result.is_ok() {
18846 warn!("decommission_object: migrated {} {}", &bucket, &object_name);
18847 }
18848 result
18849 }
18850}
18851
18852#[cfg(test)]
18853async fn persist_v3_pool_meta_for_test(store: &Arc<ECStore>) {
18854 let mut meta = PoolMeta::default();
18855 meta.load_no_lock_from_replicas(store.pools.clone())
18856 .await
18857 .expect("the baseline pool metadata should be readable before V3 migration");
18858 meta.version = POOL_META_GENERATION_VERSION;
18859 let mut write_state = store.pool_meta_save_gate.lock().await.clone();
18860 temp_env::async_with_vars(
18861 [
18862 (rustfs_config::ENV_POOL_META_V3_WRITE, Some("true")),
18863 (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("true")),
18864 ],
18865 async {
18866 meta.save_no_lock_observing(store.pools.clone(), &mut write_state)
18867 .await
18868 .expect("the baseline metadata should be upgraded to a durable V3 replica");
18869 },
18870 )
18871 .await;
18872
18873 let mut loaded = PoolMeta::default();
18874 loaded
18875 .load_no_lock_from_replicas(store.pools.clone())
18876 .await
18877 .expect("the upgraded V3 pool metadata should be readable");
18878 assert_eq!(loaded.version, POOL_META_GENERATION_VERSION);
18879 store.pool_meta.write().await.version = POOL_META_GENERATION_VERSION;
18880}
18881
18882#[cfg(test)]
18883#[allow(clippy::items_after_test_module)]
18884mod tests {
18885 use super::*;
18886 use crate::bucket::replication::{ReplicationState, ReplicationStatusType};
18887 use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
18888 use crate::storage_api_contracts::multipart::MultipartOperations as _;
18889 use serde::Serialize;
18890
18891 #[test]
18892 fn decommission_set_local_usage_cache_classification_is_exact() {
18893 for object in [
18894 "buckets/.usage-cache.bin",
18895 "buckets/.usage-cache.bin.bkp",
18896 "buckets/photos/.usage-cache.bin",
18897 "buckets/photos/.usage-cache.bin.bkp",
18898 ] {
18899 assert!(is_decommission_set_local_usage_cache(RUSTFS_META_BUCKET, object), "{object}");
18900 assert!(!is_decommission_set_local_usage_cache("user-bucket", object), "{object}");
18901 }
18902 for object in [
18903 "buckets/.usage.v2.json",
18904 "buckets/.usage.v2.json.bkp",
18905 "buckets/.usage-cache.bin.extra",
18906 "buckets/.usage-cache.bin.bkp.extra",
18907 "buckets/prefix.usage-cache.bin",
18908 "buckets/photos/.usage-cache.bin.bkp.bkp",
18909 "buckets/photos/nested/.usage-cache.bin",
18910 "buckets//.usage-cache.bin",
18911 "buckets/../.usage-cache.bin",
18912 "buckets/./.usage-cache.bin",
18913 "buckets/.usage-cache.bin/child",
18914 "config/.usage-cache.bin",
18915 "buckets-other/.usage-cache.bin",
18916 ".usage-cache.bin",
18917 ] {
18918 assert!(!is_decommission_set_local_usage_cache(RUSTFS_META_BUCKET, object), "{object}");
18919 }
18920 }
18921
18922 #[tokio::test]
18923 #[serial_test::serial]
18924 async fn decommission_keeps_set_local_usage_caches_out_of_target_capacity() {
18925 use crate::object_api::PutObjReader;
18926 use tokio::io::AsyncReadExt as _;
18927
18928 let (_temp_dirs, store, _other_store) =
18931 Box::pin(crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None)).await;
18932 let user_bucket = "decommission-usage-cache-control";
18933 Box::pin(store.make_bucket(user_bucket, &MakeBucketOptions::default()))
18934 .await
18935 .expect("create the ordinary-object control bucket");
18936 let incarnation = Box::pin(store.bucket_incarnation_id(user_bucket))
18937 .await
18938 .expect("control bucket incarnation");
18939 let source_time = OffsetDateTime::now_utc();
18940 let cache_objects = [
18941 "buckets/.usage-cache.bin",
18942 "buckets/.usage-cache.bin.bkp",
18943 "buckets/photos/.usage-cache.bin",
18944 "buckets/photos/.usage-cache.bin.bkp",
18945 ];
18946 let conflict_object = "buckets/.usage-cache.bin.conflict";
18947 let source_read_failure_object = "buckets/.usage-cache.bin.read-error";
18948 let source_body = b"source set cache";
18949 let target_body = b"independent older target set cache";
18950 for object in cache_objects.into_iter().chain([conflict_object]) {
18951 for (pool_index, body, mod_time) in [
18952 (0, source_body.as_slice(), source_time),
18953 (1, target_body.as_slice(), source_time - Duration::seconds(1)),
18954 ] {
18955 store.pools[pool_index]
18957 .get_disks_by_key(object)
18958 .put_object(
18959 RUSTFS_META_BUCKET,
18960 object,
18961 &mut PutObjReader::from_vec(body.to_vec()),
18962 &ObjectOptions {
18963 mod_time: Some(mod_time),
18964 ..Default::default()
18965 },
18966 )
18967 .await
18968 .expect("seed distinct native set-local objects");
18969 }
18970 }
18971 let controls = [
18972 (RUSTFS_META_BUCKET, "buckets/.usage.v2.json"),
18973 (RUSTFS_META_BUCKET, "buckets/photos/.usage-cache.bin.extra"),
18974 (user_bucket, "ordinary-object"),
18975 (user_bucket, "buckets/.usage-cache.bin"),
18976 ];
18977 for (bucket, object) in controls {
18978 store.pools[0]
18979 .put_object(
18980 bucket,
18981 object,
18982 &mut PutObjReader::from_vec(b"ordinary object contents".to_vec()),
18983 &ObjectOptions {
18984 expected_bucket_incarnation_id: (bucket == user_bucket).then_some(incarnation),
18985 ..Default::default()
18986 },
18987 )
18988 .await
18989 .expect("seed a control that must migrate");
18990 }
18991 store.pools[0]
18992 .put_object(
18993 RUSTFS_META_BUCKET,
18994 source_read_failure_object,
18995 &mut PutObjReader::from_vec(source_body.to_vec()),
18996 &ObjectOptions::default(),
18997 )
18998 .await
18999 .expect("seed a similarly named object whose source read will fail");
19000
19001 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19002 set_decommission_capacity_info_overrides_for_test(
19003 store.id,
19004 vec![vec![
19005 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 16_384, 16_384),
19006 DecommissionPoolCapacityInfo::for_test(1, layout, 131_072, 131_072, 0),
19007 ]],
19008 );
19009 Box::pin(store.save_current_pool_meta_for_decommission_start(&[0], Vec::new()))
19010 .await
19011 .expect("activate the decommission capacity reservation");
19012
19013 for object in cache_objects {
19014 Box::pin(store.decommission_entry_for_test(
19015 0,
19016 MetaCacheEntry {
19017 name: object.to_string(),
19018 ..Default::default()
19019 },
19020 RUSTFS_META_BUCKET.to_string(),
19021 store.pools[0].get_disks_by_key(object),
19022 ))
19023 .await
19024 .expect("set-local cache must not enter cross-pool migration");
19025 for (pool_index, expected) in [(0, source_body.as_slice()), (1, target_body.as_slice())] {
19026 let mut reader = store.pools[pool_index]
19027 .get_disks_by_key(object)
19028 .get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
19029 .await
19030 .expect("each set must retain its own cache");
19031 let mut actual = Vec::new();
19032 reader.stream.read_to_end(&mut actual).await.expect("read retained cache");
19033 assert_eq!(actual, expected, "pool {pool_index}, {object}");
19034 }
19035 let meta = store.pool_meta.read().await;
19036 let info = meta.pools[0].decommission.as_ref().expect("decommission progress");
19037 assert_eq!((info.items_decommissioned, info.items_decommission_failed), (0, 0));
19038 assert_eq!((info.bytes_done, info.bytes_failed), (0, 0));
19039 let reservation = info.capacity_reservation.as_ref().expect("capacity reservation");
19040 assert_eq!(reservation.pending_target_physical_bytes, 0);
19041 assert_eq!(reservation.consumed_target_physical_bytes, 0);
19042 assert!(reservation.targets.iter().all(|target| target.pending_mutation_id.is_none()));
19043 }
19044 let mut persisted = PoolMeta::default();
19045 Box::pin(persisted.load_no_lock_from_replicas(store.pools.clone()))
19046 .await
19047 .expect("reload durable capacity intents after cache entries");
19048 let reservation = persisted.pools[0]
19049 .decommission
19050 .as_ref()
19051 .and_then(|info| info.capacity_reservation.as_ref())
19052 .expect("durable reservation");
19053 assert_eq!(reservation.pending_target_physical_bytes, 0);
19054 assert!(reservation.targets.iter().all(|target| target.pending_mutation_id.is_none()));
19055
19056 let injected_reads = Arc::new(AtomicUsize::new(0));
19057 let observed_reads = Arc::clone(&injected_reads);
19058 let read_fault = DecommissionTestFaultGuard::install(Arc::new(move |stage, bucket, object, _, success| {
19059 if stage == "object_read" && bucket == RUSTFS_META_BUCKET && object == source_read_failure_object && success {
19060 observed_reads.fetch_add(1, Ordering::SeqCst);
19061 return true;
19062 }
19063 false
19064 }));
19065 Box::pin(store.decommission_entry_for_test(
19066 0,
19067 MetaCacheEntry {
19068 name: source_read_failure_object.to_string(),
19069 ..Default::default()
19070 },
19071 RUSTFS_META_BUCKET.to_string(),
19072 store.pools[0].get_disks_by_key(source_read_failure_object),
19073 ))
19074 .await
19075 .expect("entry must record the non-NotFound source read failure");
19076 drop(read_fault);
19077 assert_eq!(injected_reads.load(Ordering::SeqCst), DECOMMISSION_VERSION_COPY_ATTEMPTS);
19078 {
19079 let meta = store.pool_meta.read().await;
19080 let info = meta.pools[0].decommission.as_ref().expect("source read failure progress");
19081 assert_eq!((info.items_decommissioned, info.items_decommission_failed), (0, 1));
19082 assert_eq!(info.bytes_failed, source_body.len());
19083 assert_eq!(
19084 info.capacity_reservation
19085 .as_ref()
19086 .expect("reservation")
19087 .pending_target_physical_bytes,
19088 0
19089 );
19090 }
19091 let mut retained = store.pools[0]
19092 .get_object_reader(
19093 RUSTFS_META_BUCKET,
19094 source_read_failure_object,
19095 None,
19096 HeaderMap::new(),
19097 &ObjectOptions::default(),
19098 )
19099 .await
19100 .expect("source read failure must retain the source");
19101 let mut retained_body = Vec::new();
19102 retained
19103 .stream
19104 .read_to_end(&mut retained_body)
19105 .await
19106 .expect("read retained source");
19107 assert_eq!(retained_body, source_body);
19108 drop(retained);
19109 let target_err = store.pools[1]
19110 .get_object_info(RUSTFS_META_BUCKET, source_read_failure_object, &ObjectOptions::default())
19111 .await
19112 .expect_err("failed source read must not create a target object");
19113 assert!(is_err_object_not_found(&target_err), "unexpected target state: {target_err:?}");
19114
19115 for (bucket, object) in controls {
19116 Box::pin(store.decommission_entry_for_test(
19117 0,
19118 MetaCacheEntry {
19119 name: object.to_string(),
19120 ..Default::default()
19121 },
19122 bucket.to_string(),
19123 store.pools[0].get_disks_by_key(object),
19124 ))
19125 .await
19126 .expect("ordinary and similarly named objects must migrate");
19127 let mut reader = store.pools[1]
19128 .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
19129 .await
19130 .expect("control must exist on the target");
19131 let mut actual = Vec::new();
19132 reader.stream.read_to_end(&mut actual).await.expect("read migrated control");
19133 assert_eq!(actual, b"ordinary object contents", "{bucket}/{object}");
19134 let err = store.pools[0]
19135 .get_object_info(bucket, object, &ObjectOptions::default())
19136 .await
19137 .expect_err("migrated control must be removed from the source");
19138 assert!(is_err_object_not_found(&err), "{bucket}/{object}: {err:?}");
19139 }
19140
19141 Box::pin(store.decommission_entry_for_test(
19142 0,
19143 MetaCacheEntry {
19144 name: conflict_object.to_string(),
19145 ..Default::default()
19146 },
19147 RUSTFS_META_BUCKET.to_string(),
19148 store.pools[0].get_disks_by_key(conflict_object),
19149 ))
19150 .await
19151 .expect("entry must record a real conditional-copy failure");
19152 let meta = store.pool_meta.read().await;
19153 let info = meta.pools[0].decommission.as_ref().expect("final progress");
19154 assert_eq!(info.items_decommissioned, controls.len());
19155 assert_eq!(
19156 info.items_decommission_failed, 2,
19157 "similar names must not hide read or migration failures"
19158 );
19159 assert_eq!(info.bytes_failed, source_body.len() * 2);
19160 drop(meta);
19161 for (pool_index, expected) in [(0, source_body.as_slice()), (1, target_body.as_slice())] {
19162 let mut reader = store.pools[pool_index]
19163 .get_object_reader(RUSTFS_META_BUCKET, conflict_object, None, HeaderMap::new(), &ObjectOptions::default())
19164 .await
19165 .expect("failed migration must preserve both objects");
19166 let mut actual = Vec::new();
19167 reader.stream.read_to_end(&mut actual).await.expect("read conflict object");
19168 assert_eq!(actual, expected);
19169 }
19170 }
19171
19172 #[tokio::test]
19173 #[serial_test::serial]
19174 async fn decommission_final_sweep_excludes_only_set_local_usage_caches() {
19175 use crate::object_api::PutObjReader;
19176
19177 let (_temp_dirs, store, _other_store) =
19178 crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
19179 for object in [
19180 "buckets/.usage-cache.bin",
19181 "buckets/.usage-cache.bin.bkp",
19182 "buckets/photos/.usage-cache.bin",
19183 "buckets/photos/.usage-cache.bin.bkp",
19184 ] {
19185 store.pools[0]
19186 .get_disks_by_key(object)
19187 .put_object(
19188 RUSTFS_META_BUCKET,
19189 object,
19190 &mut PutObjReader::from_vec(b"set-local cache".to_vec()),
19191 &ObjectOptions::default(),
19192 )
19193 .await
19194 .expect("seed each supported set-local cache path");
19195 }
19196 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19197 set_decommission_capacity_info_overrides_for_test(
19198 store.id,
19199 vec![vec![
19200 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 16_384, 16_384),
19201 DecommissionPoolCapacityInfo::for_test(1, layout, 131_072, 131_072, 0),
19202 ]],
19203 );
19204 store
19205 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19206 .await
19207 .expect("activate the final-sweep generation");
19208 let generation = store.active_decommission_generation(0).await.expect("active generation");
19209 store
19210 .check_after_decommission(0, &CancellationToken::new(), generation)
19211 .await
19212 .expect("the four set-local cache forms must not block the final sweep");
19213
19214 for object in [
19215 "buckets/.usage-cache.bin.extra",
19216 "buckets/photos/.usage-cache.bin.bkp.extra",
19217 "buckets/.usage.v2.json",
19218 ] {
19219 let source_set = store.pools[0].get_disks_by_key(object);
19220 source_set
19221 .put_object(
19222 RUSTFS_META_BUCKET,
19223 object,
19224 &mut PutObjReader::from_vec(b"unmigrated ordinary metadata".to_vec()),
19225 &ObjectOptions::default(),
19226 )
19227 .await
19228 .expect("seed ordinary metadata that must prevent completion");
19229 let err = store
19230 .check_after_decommission(0, &CancellationToken::new(), generation)
19231 .await
19232 .expect_err("a remaining similar name or global usage snapshot must block completion");
19233 assert!(err.to_string().contains("after decommissioning"), "unexpected final-sweep error: {err:?}");
19234 assert!(err.to_string().contains(object), "the final sweep must identify {object}: {err:?}");
19235 source_set
19236 .delete_object(RUSTFS_META_BUCKET, object, ObjectOptions::default())
19237 .await
19238 .expect("remove only the ordinary-metadata control before the next sweep");
19239 }
19240 store
19241 .check_after_decommission(0, &CancellationToken::new(), generation)
19242 .await
19243 .expect("only the four set-local caches remain after removing the controls");
19244 }
19245
19246 #[test]
19247 fn pool_activation_fleet_proof_error_classifier_matches_only_retryable_proof_failures() {
19248 assert!(is_pool_activation_fleet_proof_error(&Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED)));
19249 assert!(is_pool_activation_fleet_proof_error(&Error::other(POOL_ACTIVATION_FLEET_PROOF_EXPIRED)));
19250 let wrapped = format!("rebalance meta save failed during start_rebalance: {POOL_ACTIVATION_FLEET_PROOF_EXPIRED}");
19251 assert!(is_pool_activation_fleet_proof_error(&Error::other(wrapped)));
19252 assert!(!is_pool_activation_fleet_proof_error(&Error::ConfigNotFound));
19253 }
19254
19255 #[tokio::test]
19256 #[serial_test::serial]
19257 async fn decommission_v1_start_preflights_reject_before_metadata_writes() {
19258 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
19259 let mut v1_meta = store.pool_meta.read().await.clone();
19260 v1_meta.version = POOL_META_V1_VERSION;
19261 let v1_data = pool_meta_v1_replica_test_data(&v1_meta);
19262 for pool in &store.pools {
19263 save_config_with_opts(
19264 pool.clone(),
19265 POOL_META_NAME,
19266 v1_data.clone(),
19267 &ObjectOptions {
19268 max_parity: true,
19269 ..Default::default()
19270 },
19271 )
19272 .await
19273 .expect("the V1 baseline should be persisted to every pool");
19274 }
19275 let baseline = load_pool_meta_replicas(store.pools.clone(), true)
19276 .await
19277 .expect("baseline pool metadata should be readable");
19278 assert_eq!(baseline.meta.version, POOL_META_V1_VERSION);
19279 *store.pool_meta.write().await = baseline.meta.clone();
19280 let start_probe = PoolActivationStartProbe::install(PoolActivationStartKind::Decommission);
19281 let err = store
19282 .start_decommission(vec![0])
19283 .await
19284 .expect_err("the initial V1 start preflight must reject before side effects");
19285 assert!(matches!(err, Error::InvalidArgument(..)));
19286 assert!(
19287 !start_probe.preflight_side_effect_was_attempted(),
19288 "V1 rejection must precede bucket listing, healing, and metadata-bucket creation"
19289 );
19290 assert!(
19291 !start_probe.activation_was_attempted(),
19292 "V1 rejection must not enter the authoritative activation save"
19293 );
19294 let after_early_rejection = load_pool_meta_replicas(store.pools.clone(), true)
19295 .await
19296 .expect("early rejection must leave durable pool metadata readable");
19297 assert_eq!(after_early_rejection.canonical, baseline.canonical);
19298 assert!(
19299 store
19300 .pool_meta
19301 .read()
19302 .await
19303 .pools
19304 .iter()
19305 .all(|pool| pool.decommission.is_none())
19306 );
19307 assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
19308 store
19309 .ensure_pool_meta_side_effects_safe("V1 start preflight")
19310 .await
19311 .expect("a deterministic start rejection must not latch recovery");
19312 drop(start_probe);
19313
19314 let err = store
19315 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19316 .await
19317 .expect_err("the authoritative V1 start preflight must reject before saving");
19318 assert!(matches!(err, Error::InvalidArgument(..)));
19319
19320 let after_authoritative_rejection = load_pool_meta_replicas(store.pools.clone(), true)
19321 .await
19322 .expect("authoritative rejection must leave durable pool metadata readable");
19323 assert_eq!(after_authoritative_rejection.canonical, baseline.canonical);
19324 assert!(
19325 after_authoritative_rejection
19326 .meta
19327 .pools
19328 .iter()
19329 .all(|pool| pool.decommission.is_none())
19330 );
19331 assert!(
19332 store
19333 .pool_meta
19334 .read()
19335 .await
19336 .pools
19337 .iter()
19338 .all(|pool| pool.decommission.is_none())
19339 );
19340 assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
19341 store
19342 .ensure_pool_meta_side_effects_safe("authoritative V1 start preflight")
19343 .await
19344 .expect("an authoritative capability rejection must not latch recovery");
19345 }
19346
19347 #[tokio::test]
19348 #[serial_test::serial]
19349 async fn decommission_activation_fence_loss_after_durable_save_blocks_publication() {
19350 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
19351 crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
19352 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19353 set_decommission_capacity_info_overrides_for_test(
19354 store.id,
19355 vec![vec![
19356 DecommissionPoolCapacityInfo::for_test(0, layout, 60, 60, 0),
19357 DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100),
19358 ]],
19359 );
19360 let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
19361 let start_store = Arc::clone(&store);
19362 let mut start_task = tokio::spawn(async move {
19363 start_store
19364 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19365 .await
19366 });
19367
19368 tokio::select! {
19369 result = &mut start_task => panic!("activation finished before the durable-save barrier: {result:?}"),
19370 () = barrier.wait_until_paused() => {}
19371 }
19372 barrier.release_after_fence_loss();
19373 let err = tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
19374 .await
19375 .expect("decommission activation should stop after losing its fence")
19376 .expect("decommission activation task should not panic")
19377 .expect_err("post-durable-save fence loss must reject in-memory publication");
19378 assert!(err.to_string().contains("activation lock lost"));
19379
19380 let local = store.pool_meta.read().await;
19381 assert!(
19382 !pool_meta_has_active_decommission(&local),
19383 "the activation must not publish after its durable fence is lost"
19384 );
19385 drop(local);
19386 store
19387 .ensure_pool_meta_side_effects_safe("post-durable-save activation fence loss")
19388 .await
19389 .expect_err("the undisarmed transaction must latch the sticky pool metadata gate");
19390
19391 let mut persisted = PoolMeta::default();
19392 persisted
19393 .load_no_lock_from_replicas(vec![store.pools[0].clone()])
19394 .await
19395 .expect("the durable replica should remain readable for recovery");
19396 assert!(
19397 pool_meta_has_active_decommission(&persisted),
19398 "the test must lose the fence only after one durable replica commit"
19399 );
19400 }
19401
19402 #[tokio::test]
19403 #[serial_test::serial]
19404 async fn decommission_activation_adopts_canonical_commit_after_replica_failure() {
19405 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
19406 crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
19407 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19408 set_decommission_capacity_info_overrides_for_test(
19409 store.id,
19410 vec![vec![
19411 DecommissionPoolCapacityInfo::for_test(0, layout, 60, 60, 0),
19412 DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100),
19413 ]],
19414 );
19415 let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
19416 let start_store = Arc::clone(&store);
19417 let mut start_task = tokio::spawn(async move {
19418 start_store
19419 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19420 .await
19421 });
19422
19423 tokio::select! {
19424 result = &mut start_task => panic!("activation finished before the durable-save barrier: {result:?}"),
19425 () = barrier.wait_until_paused() => {}
19426 }
19427 let mut replica_disks = Vec::new();
19428 for set in &store.pools[1].disk_set {
19429 let mut disks = set.disks.write().await;
19430 let saved = std::mem::take(&mut *disks);
19431 *disks = vec![None; saved.len()];
19432 replica_disks.push(saved);
19433 }
19434 barrier.release_without_fence_loss();
19435
19436 tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
19437 .await
19438 .expect("decommission activation should finish after its canonical commit")
19439 .expect("decommission activation task should not panic")
19440 .expect("a replica save failure must not report the committed activation as failed");
19441
19442 for (set, disks) in store.pools[1].disk_set.iter().zip(replica_disks) {
19443 *set.disks.write().await = disks;
19444 }
19445
19446 let local = store.pool_meta.read().await;
19447 assert!(pool_meta_has_active_decommission(&local));
19448 drop(local);
19449
19450 let mut canonical = PoolMeta::default();
19451 canonical
19452 .load_no_lock_from_replicas(vec![store.pools[0].clone()])
19453 .await
19454 .expect("the canonical committed decommission metadata should remain readable");
19455 assert!(pool_meta_has_active_decommission(&canonical));
19456
19457 let mut replica = PoolMeta::default();
19458 replica
19459 .load_no_lock_from_replicas(vec![store.pools[1].clone()])
19460 .await
19461 .expect("the stale replica metadata should remain readable after disks recover");
19462 assert!(!pool_meta_has_active_decommission(&replica));
19463
19464 let mut reloaded = PoolMeta::default();
19465 let replica_state = reloaded
19466 .load_no_lock_from_replicas(store.pools.clone())
19467 .await
19468 .expect("the canonical commit should remain reloadable after a replica recovers stale");
19469 assert!(pool_meta_has_active_decommission(&reloaded));
19470 assert!(replica_state.needs_repair);
19471 assert!(replica_state.repair_write_safe);
19472
19473 let worker_cancel = CancellationToken::new();
19474 let index_cancelers = store
19475 .reserve_decommission_routines(&worker_cancel, &[0])
19476 .await
19477 .expect("the committed activation should admit its decommission worker");
19478 drop(spawn_decommission_index_cancelers(
19479 Arc::clone(&store),
19480 worker_cancel.clone(),
19481 index_cancelers,
19482 Arc::new(Semaphore::new(decommission_entry_concurrency_limit())),
19483 ));
19484 let admitted_cancel = store.decommission_cancelers.read().await[0]
19485 .clone()
19486 .expect("the admitted decommission worker should have a cancellation token");
19487 assert!(!admitted_cancel.is_cancelled());
19488 worker_cancel.cancel();
19489 assert!(admitted_cancel.is_cancelled());
19490 }
19491
19492 #[tokio::test]
19493 #[serial_test::serial(pool_meta_version_env)]
19494 async fn decommission_capacity_reservation_serializes_two_nodes_without_overselling() {
19495 let (_temp_dirs, first_node, second_node) =
19496 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19497 persist_v3_pool_meta_for_test(&first_node).await;
19498 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19499 let capacity_snapshot = vec![
19500 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
19501 DecommissionPoolCapacityInfo::for_test(1, layout, 0, 30, 30),
19502 DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0),
19503 ];
19504 set_decommission_capacity_info_overrides_for_test(first_node.id, vec![capacity_snapshot.clone()]);
19505 set_decommission_capacity_info_overrides_for_test(second_node.id, vec![capacity_snapshot]);
19506
19507 let barrier =
19508 PutObjectCommitBarrier::install(RUSTFS_META_BUCKET, POOL_META_NAME, PutObjectCommitPause::BeforeQuotaRename);
19509 let first_store = Arc::clone(&first_node);
19510 let first = tokio::spawn(async move {
19511 first_store
19512 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19513 .await
19514 });
19515 barrier.wait_until_paused().await;
19516
19517 let second_probe = PoolActivationStartProbe::install(PoolActivationStartKind::Decommission);
19518 let second_store = Arc::clone(&second_node);
19519 let second = tokio::spawn(async move {
19520 second_store
19521 .save_current_pool_meta_for_decommission_start(&[1], Vec::new())
19522 .await
19523 });
19524 second_probe.wait_until_attempted().await;
19525 assert!(
19526 !second.is_finished(),
19527 "the competing node must wait behind the distributed activation fence"
19528 );
19529
19530 drop(barrier);
19531 first
19532 .await
19533 .expect("first-node activation should not panic")
19534 .expect("first-node reservation should fit");
19535 let mut started_v3 = PoolMeta::default();
19536 started_v3
19537 .load_no_lock_from_replicas(first_node.pools.clone())
19538 .await
19539 .expect("the started capacity reservation should reload from V3 replicas");
19540 assert_eq!(started_v3.version, POOL_META_GENERATION_VERSION);
19541 let err = second
19542 .await
19543 .expect("second-node activation should not panic")
19544 .expect_err("the second reservation must observe and reject the committed first reservation");
19545 assert!(
19546 err.to_string().contains("requires 60 bytes, but 40 bytes are available"),
19547 "the second reservation must report the committed capacity rejection, got {err:?}"
19548 );
19549
19550 let mut persisted = PoolMeta::default();
19551 persisted
19552 .load_no_lock_from_replicas(first_node.pools.clone())
19553 .await
19554 .expect("the winning capacity reservation should remain readable");
19555 let reservation = persisted.pools[0]
19556 .decommission
19557 .as_ref()
19558 .and_then(|info| info.capacity_reservation.as_ref())
19559 .expect("the winning operation must retain its durable reservation");
19560 assert_eq!(
19561 reservation.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
19562 "an all-v4 proof must persist the per-target lock model through V3 replicas"
19563 );
19564 assert_eq!(reservation.peak_physical_bytes, 60);
19565 assert!(persisted.pools[1].decommission.is_none());
19566 }
19567
19568 #[tokio::test]
19569 #[serial_test::serial]
19570 async fn decommission_start_without_dedicated_v4_proof_persists_the_legacy_lock_model() {
19571 let (_temp_dirs, store, _other_store) =
19572 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19573 crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
19574 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19575 set_decommission_capacity_info_overrides_for_test(
19576 store.id,
19577 vec![vec![
19578 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
19579 DecommissionPoolCapacityInfo::for_test(1, layout, 0, 10, 10),
19580 DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
19581 ]],
19582 );
19583 let _proof_guard = crate::services::notification_sys::without_decommission_target_fence_fleet_proof_for_test();
19584
19585 store
19586 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19587 .await
19588 .expect("a mixed-version fleet should retain the compatible global lock model");
19589 let reservation = store.pool_meta.read().await.pools[0]
19590 .decommission
19591 .as_ref()
19592 .and_then(|info| info.capacity_reservation.as_ref())
19593 .expect("proofless start should persist a capacity reservation")
19594 .clone();
19595 assert_eq!(reservation.model_version, DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION);
19596 }
19597
19598 #[tokio::test]
19599 #[serial_test::serial]
19600 async fn decommission_cancel_replans_target_fences_after_a_concurrent_v2_start() {
19601 let (_temp_dirs, store, _other_store) =
19602 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19603 crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
19604 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19605 set_decommission_capacity_info_overrides_for_test(
19606 store.id,
19607 vec![vec![
19608 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
19609 DecommissionPoolCapacityInfo::for_test(1, layout, 0, 10, 10),
19610 DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
19611 ]],
19612 );
19613 let barrier = super::DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19614 barrier.pause_cancel_before_start();
19615 let cancel_store = Arc::clone(&store);
19616 let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
19617 barrier.wait_until_cancel_before_start().await;
19618
19619 store
19620 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19621 .await
19622 .expect("the concurrent start should persist a v2 reservation");
19623 let target_pool_index = store.pool_meta.read().await.pools[0]
19624 .decommission
19625 .as_ref()
19626 .and_then(|info| info.capacity_reservation.as_ref())
19627 .map(|reservation| {
19628 assert_eq!(reservation.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION);
19629 reservation.targets[0].pool_index
19630 })
19631 .expect("the concurrent start should publish its target cohort");
19632 let target_lock = store.pools[0]
19633 .new_ns_lock(
19634 RUSTFS_META_BUCKET,
19635 &format!("{}/{target_pool_index}", super::DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX),
19636 )
19637 .await
19638 .expect("create the cancel/start target-fence probe");
19639 let target_guard = target_lock
19640 .get_write_lock(std::time::Duration::from_secs(30))
19641 .await
19642 .expect("hold the newly started target fence");
19643
19644 barrier.release_cancel_before_start();
19645 tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel)
19646 .await
19647 .expect_err("cancel must wait for the target fence selected by the concurrent start");
19648 assert!(
19649 store.pool_meta.read().await.pools[0]
19650 .decommission
19651 .as_ref()
19652 .and_then(|info| info.capacity_reservation.as_ref())
19653 .is_some_and(DecommissionCapacityReservation::active),
19654 "cancel must not release the new reservation before fencing its target"
19655 );
19656
19657 drop(target_guard);
19658 tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
19659 .await
19660 .expect("cancel should finish after the target fence is released")
19661 .expect("cancel task should not panic")
19662 .expect("cancel should publish the terminal state");
19663 drop(barrier);
19664
19665 let pool_meta = store.pool_meta.read().await;
19666 let info = pool_meta.pools[0]
19667 .decommission
19668 .as_ref()
19669 .expect("canceled decommission metadata should remain present");
19670 assert!(info.canceled);
19671 let reservation = info
19672 .capacity_reservation
19673 .as_ref()
19674 .expect("canceled capacity accounting should remain inspectable");
19675 assert!(!reservation.active());
19676 assert_eq!(reservation.pending_target_physical_bytes, 0);
19677 assert_eq!(reservation.inflight_target_physical_bytes, 0);
19678 }
19679
19680 async fn start_target_fenced_cancel_test(
19681 store: &Arc<ECStore>,
19682 first_target_free: usize,
19683 ) -> (DecommissionCapacityTerminalFencePlan, DecommissionCanceler) {
19684 crate::services::rebalance::promote_test_pool_meta_to_v2(store).await;
19685 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19686 set_decommission_capacity_info_overrides_for_test(
19687 store.id,
19688 vec![vec![
19689 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
19690 DecommissionPoolCapacityInfo::for_test(1, layout, first_target_free, 10, 10 - first_target_free),
19691 DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
19692 ]],
19693 );
19694 store
19695 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19696 .await
19697 .expect("persist an active target-fenced decommission");
19698 let plan = decommission_capacity_terminal_fence_plan(&*store.pool_meta.read().await, 0)
19699 .expect("active decommission should have a valid fence plan")
19700 .expect("active decommission should retain its reservation");
19701 assert_eq!(plan.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION);
19702 let canceler = DecommissionCanceler::new(CancellationToken::new());
19703 *store.decommission_cancelers.write().await = vec![Some(canceler.clone()), None, None];
19704 (plan, canceler)
19705 }
19706
19707 #[tokio::test]
19708 #[serial_test::serial]
19709 async fn decommission_cancel_waits_for_target_contention_past_one_lock_timeout() {
19710 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("5"))], async {
19711 let (_temp_dirs, store, _other_store) =
19712 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19713 let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
19714 assert_eq!(plan.target_pool_indices, vec![2]);
19715 let target_lock = store.pools[0]
19716 .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
19717 .await
19718 .expect("create the active migration target gate");
19719 let target_guard = target_lock
19720 .get_write_lock(std::time::Duration::from_secs(30))
19721 .await
19722 .expect("hold the target gate across the first cancel acquisition timeout");
19723 let movement_gate = store.ctx.data_movement_operation_gate();
19724 let movement_guard = movement_gate.read().await;
19725 let cancel_store = Arc::clone(&store);
19726 let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
19727
19728 tokio::time::timeout(get_lock_acquire_timeout() + std::time::Duration::from_secs(1), &mut cancel)
19729 .await
19730 .expect_err("one target-lock timeout must not end a legitimate cancel while its retry budget remains");
19731 assert!(!canceler.is_cancelled(), "cancel must not signal its worker before durable fencing");
19732 let mut durable = PoolMeta::default();
19733 durable
19734 .load_no_lock_from_replicas(store.pools.clone())
19735 .await
19736 .expect("the active reservation must remain readable while cancel waits");
19737 assert_eq!(
19738 decommission_capacity_terminal_fence_plan(&durable, 0).expect("read the durable active plan"),
19739 Some(plan),
19740 "a failed target acquisition must not release or replace the durable reservation"
19741 );
19742 assert!(!durable.pools[0].decommission.as_ref().expect("active decommission").canceled);
19743
19744 drop(target_guard);
19745 tokio::time::timeout(std::time::Duration::from_secs(30), canceler.token().cancelled())
19746 .await
19747 .expect("cancel should persist and signal its worker after target contention is released");
19748 durable
19749 .load_no_lock_from_replicas(store.pools.clone())
19750 .await
19751 .expect("reload the committed cancellation from native replicas");
19752 let info = durable.pools[0]
19753 .decommission
19754 .as_ref()
19755 .expect("canceled state must remain durable");
19756 assert!(info.canceled);
19757 assert!(!info.complete && !info.failed);
19758 let reservation = info
19759 .capacity_reservation
19760 .as_ref()
19761 .expect("terminal capacity accounting must remain inspectable");
19762 assert!(!reservation.active());
19763 assert_eq!(reservation.pending_target_physical_bytes, 0);
19764 assert_eq!(reservation.inflight_target_physical_bytes, 0);
19765 tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel)
19766 .await
19767 .expect_err("the runtime-fenced cancel must wait for in-flight movement after signaling");
19768 drop(movement_guard);
19769 tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
19770 .await
19771 .expect("cancel should return after in-flight movement quiesces")
19772 .expect("cancel task should not panic")
19773 .expect("the same cancel request should finish its durable terminal transition");
19774 })
19775 .await;
19776 }
19777
19778 #[tokio::test]
19779 #[serial_test::serial]
19780 async fn decommission_cancel_target_contention_exhausts_bounded_attempts_without_committing() {
19781 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
19782 let (_temp_dirs, store, _other_store) =
19783 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19784 let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
19785 let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19786 let target_lock = store.pools[0]
19787 .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
19788 .await
19789 .expect("create the persistently contended target gate");
19790 let target_guard = target_lock
19791 .get_write_lock(std::time::Duration::from_secs(30))
19792 .await
19793 .expect("hold the target gate for every cancel attempt");
19794
19795 let err = tokio::time::timeout(std::time::Duration::from_secs(30), store.decommission_cancel(0))
19796 .await
19797 .expect("target contention must not retry indefinitely")
19798 .expect_err("exhausting the acquisition budget must not report cancellation success");
19799 match err {
19800 Error::Lock(rustfs_lock::LockError::Timeout { resource, timeout }) => {
19801 assert_eq!(resource, ".rustfs.sys/decommission/capacity-target/2@latest");
19802 assert_eq!(timeout, std::time::Duration::from_secs(1));
19803 }
19804 other => panic!("expected the final typed target-lock timeout, got {other:?}"),
19805 }
19806 assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 3);
19807 assert!(!canceler.is_cancelled());
19808 assert!(canceler.is_active());
19809 let mut durable = PoolMeta::default();
19810 durable
19811 .load_no_lock_from_replicas(store.pools.clone())
19812 .await
19813 .expect("reload the reservation after the canceled request exhausted its budget");
19814 assert_eq!(
19815 decommission_capacity_terminal_fence_plan(&durable, 0).expect("the active plan should remain valid"),
19816 Some(plan)
19817 );
19818 assert!(!durable.pools[0].decommission.as_ref().expect("active decommission").canceled);
19819 drop(target_guard);
19820 })
19821 .await;
19822 }
19823
19824 #[tokio::test]
19825 #[serial_test::serial]
19826 async fn decommission_cancel_target_retry_releases_partial_cohort_and_rejects_remote_replacement() {
19827 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
19828 let (_temp_dirs, store, other_store) =
19829 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19830 let (plan, _old_canceler) = start_target_fenced_cancel_test(&store, 8).await;
19831 assert_eq!(plan.target_pool_indices, vec![1, 2]);
19832 let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19833 barrier.pause_cancel_target_timeout();
19834 let target_lock = store.pools[0]
19835 .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
19836 .await
19837 .expect("create the second target gate");
19838 let target_guard = target_lock
19839 .get_write_lock(std::time::Duration::from_secs(30))
19840 .await
19841 .expect("block cancellation after it acquires the first target");
19842 let cancel_store = Arc::clone(&store);
19843 let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
19844 barrier.wait_until_cancel_target_timeout().await;
19845
19846 let first_target_lock = store.pools[0]
19847 .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/1"))
19848 .await
19849 .expect("create the partial-cohort release probe");
19850 let first_target_guard = first_target_lock
19851 .get_write_lock(std::time::Duration::from_secs(1))
19852 .await
19853 .expect("cancel must release its first target before retrying the blocked second target");
19854 drop(first_target_guard);
19855 drop(target_guard);
19856
19857 other_store
19858 .reload_pool_meta()
19859 .await
19860 .expect("load the active generation on the remote node");
19861 tokio::time::timeout(std::time::Duration::from_secs(30), other_store.decommission_cancel(0))
19862 .await
19863 .expect("remote cancel must persist pool metadata without re-entering its state locks")
19864 .expect("the remote node should cancel the old generation");
19865 other_store
19866 .clear_decommission(0)
19867 .await
19868 .expect("the remote node should clear the old generation");
19869 let (replacement_plan, replacement_canceler) = start_target_fenced_cancel_test(&other_store, 8).await;
19870 assert_ne!(replacement_plan.operation_id, plan.operation_id);
19871
19872 barrier.release_cancel_target_timeout();
19873 let err = tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
19874 .await
19875 .expect("the stale request must finish without retrying a replacement operation")
19876 .expect("the stale cancel task should not panic")
19877 .expect_err("the original request must not cancel a remotely replaced generation");
19878 assert!(err.to_string().contains("changed while retrying terminal fences"));
19879 assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
19880 assert!(!replacement_canceler.is_cancelled());
19881 let mut durable = PoolMeta::default();
19882 durable
19883 .load_no_lock_from_replicas(store.pools.clone())
19884 .await
19885 .expect("reload the replacement after the stale cancel returns");
19886 assert_eq!(
19887 decommission_capacity_terminal_fence_plan(&durable, 0).expect("the replacement must retain its valid plan"),
19888 Some(replacement_plan)
19889 );
19890 assert!(
19891 !durable.pools[0]
19892 .decommission
19893 .as_ref()
19894 .expect("replacement decommission")
19895 .canceled
19896 );
19897 })
19898 .await;
19899 }
19900
19901 #[tokio::test]
19902 #[serial_test::serial]
19903 async fn decommission_cancel_target_retry_survives_caller_abort_and_settles_inflight_mutation() {
19904 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
19905 let (_temp_dirs, store, _other_store) =
19906 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19907 let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
19908 let owner = DecommissionCapacityOwner {
19909 source_pool_index: 0,
19910 operation_id: plan.operation_id,
19911 generation: plan.generation,
19912 owner_nonce: plan.owner_nonce,
19913 mutation_id: Some(uuid::Uuid::new_v4()),
19914 };
19915 let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
19916 let (release_tx, release_rx) = tokio::sync::oneshot::channel();
19917 let mutation_store = Arc::clone(&store);
19918 let mutation = tokio::spawn(async move {
19919 mutation_store
19920 .run_decommission_capacity_admitted_mutation(2, Some(owner), Some(1), || async {
19921 entered_tx
19922 .send(())
19923 .expect("the test should observe the admitted target mutation");
19924 release_rx.await.expect("the test should release the in-flight mutation");
19925 Ok(())
19926 })
19927 .await
19928 });
19929 tokio::time::timeout(std::time::Duration::from_secs(30), entered_rx)
19930 .await
19931 .expect("target mutation should reach its controlled I/O phase")
19932 .expect("target admission should succeed before cancel starts");
19933 let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19934 let cancel_store = Arc::clone(&store);
19935 let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
19936 barrier.wait_until_cancel_target_timeout().await;
19937 assert!(!canceler.is_cancelled(), "in-flight capacity settlement must precede the cancel signal");
19938 cancel.abort();
19939 assert!(cancel.await.expect_err("the RPC waiter should be aborted").is_cancelled());
19940
19941 release_tx
19942 .send(())
19943 .expect("the mutation must still be alive after the caller disconnects");
19944 tokio::time::timeout(std::time::Duration::from_secs(30), mutation)
19945 .await
19946 .expect("the in-flight mutation must be able to settle without a metadata lock cycle")
19947 .expect("the mutation task should not panic")
19948 .expect("the admitted mutation must settle before the target gate is released");
19949 tokio::time::timeout(std::time::Duration::from_secs(30), async {
19950 loop {
19951 if store.pool_meta.read().await.pools[0]
19952 .decommission
19953 .as_ref()
19954 .is_some_and(|info| info.canceled)
19955 {
19956 break;
19957 }
19958 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
19959 }
19960 })
19961 .await
19962 .expect("the detached cancel transaction must finish after target contention clears");
19963 assert!(canceler.is_cancelled());
19964 let mut durable = PoolMeta::default();
19965 durable
19966 .load_no_lock_from_replicas(store.pools.clone())
19967 .await
19968 .expect("reload cancellation after the RPC waiter was dropped");
19969 let info = durable.pools[0].decommission.as_ref().expect("durable canceled decommission");
19970 assert!(info.canceled && !info.failed && !info.complete);
19971 let reservation = info
19972 .capacity_reservation
19973 .as_ref()
19974 .expect("inspect settled capacity accounting");
19975 assert!(!reservation.active());
19976 assert_eq!(reservation.pending_target_physical_bytes, 0);
19977 assert_eq!(reservation.inflight_target_physical_bytes, 0);
19978 })
19979 .await;
19980 }
19981
19982 #[tokio::test]
19983 #[serial_test::serial]
19984 async fn decommission_cancel_target_retry_does_not_replay_a_failed_durable_save() {
19985 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
19986 let (_temp_dirs, store, _other_store) =
19987 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19988 let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
19989 let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19990 barrier.pause_cancel_target_timeout();
19991 let target_lock = store.pools[0]
19992 .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
19993 .await
19994 .expect("create the target gate preceding the failed save");
19995 let target_guard = target_lock
19996 .get_write_lock(std::time::Duration::from_secs(30))
19997 .await
19998 .expect("force one target acquisition retry before persistence");
19999 let save_calls = Arc::new(AtomicUsize::new(0));
20000 let calls = Arc::clone(&save_calls);
20001 let cancel_store = Arc::clone(&store);
20002 let owner = canceler.clone();
20003 let cancel = tokio::spawn(async move {
20004 cancel_store
20005 .decommission_cancel_transaction(0, Some(owner), true, move |_, _| async move {
20006 calls.fetch_add(1, Ordering::AcqRel);
20007 Err(Error::Timeout)
20008 })
20009 .await
20010 });
20011 barrier.wait_until_cancel_target_timeout().await;
20012 assert_eq!(save_calls.load(Ordering::Acquire), 0, "persistence must wait for every target fence");
20013 drop(target_guard);
20014 barrier.release_cancel_target_timeout();
20015 let err = tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
20016 .await
20017 .expect("a persistence failure must end the cancellation transaction")
20018 .expect("the failed-save task should not panic")
20019 .expect_err("the injected durable-save failure must reach the caller");
20020 assert!(matches!(err, Error::Timeout));
20021 assert_eq!(save_calls.load(Ordering::Acquire), 1);
20022 assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
20023 assert!(!canceler.is_cancelled());
20024 assert!(canceler.is_active());
20025 store
20026 .ensure_pool_meta_side_effects_safe("verify ambiguous cancellation save blocks further writes")
20027 .await
20028 .expect_err("a failed durable save must retain the existing recovery gate");
20029 let mut durable = PoolMeta::default();
20030 durable
20031 .load_no_lock_from_replicas(store.pools.clone())
20032 .await
20033 .expect("the original reservation should remain readable after the injected save failure");
20034 assert_eq!(
20035 decommission_capacity_terminal_fence_plan(&durable, 0).expect("the old reservation should remain valid"),
20036 Some(plan)
20037 );
20038 })
20039 .await;
20040 }
20041
20042 #[tokio::test]
20043 #[serial_test::serial]
20044 async fn decommission_cancel_target_retry_converges_after_successive_contenders() {
20045 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
20046 let (_temp_dirs, store, _other_store) =
20047 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
20048 let (_plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
20049 let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
20050 barrier.pause_cancel_target_timeout();
20051 let target_lock = store.pools[0]
20052 .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
20053 .await
20054 .expect("create the repeatedly contended target gate");
20055 let first_guard = target_lock
20056 .get_write_lock(std::time::Duration::from_secs(30))
20057 .await
20058 .expect("the first contender should own the target gate");
20059 let cancel_store = Arc::clone(&store);
20060 let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
20061 barrier.wait_until_cancel_target_timeout().await;
20062 assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
20063 drop(first_guard);
20064 let second_guard = target_lock
20065 .get_write_lock(std::time::Duration::from_secs(30))
20066 .await
20067 .expect("a second contender should be able to acquire between cancel attempts");
20068 barrier.release_cancel_target_timeout();
20069 barrier.pause_cancel_target_timeout();
20070 barrier.wait_until_cancel_target_timeout().await;
20071 assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 2);
20072 assert!(!canceler.is_cancelled());
20073 drop(second_guard);
20074 barrier.release_cancel_target_timeout();
20075
20076 tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
20077 .await
20078 .expect("cancel should converge after the repeated contention ends")
20079 .expect("the cancel task should not panic")
20080 .expect("the third target acquisition should permit a durable cancellation");
20081 assert!(canceler.is_cancelled());
20082 let mut durable = PoolMeta::default();
20083 durable
20084 .load_no_lock_from_replicas(store.pools.clone())
20085 .await
20086 .expect("the successful retried cancellation must survive a native metadata reload");
20087 assert!(
20088 durable.pools[0]
20089 .decommission
20090 .as_ref()
20091 .expect("canceled decommission")
20092 .canceled
20093 );
20094 assert!(
20095 decommission_capacity_terminal_fence_plan(&durable, 0)
20096 .expect("valid terminal metadata")
20097 .is_none()
20098 );
20099 })
20100 .await;
20101 }
20102
20103 #[tokio::test]
20104 #[serial_test::serial]
20105 async fn stale_node_cancel_cannot_replace_a_new_durable_v2_operation() {
20106 let (_temp_dirs, first_node, stale_node) =
20107 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
20108 crate::services::rebalance::promote_test_pool_meta_to_v2(&first_node).await;
20109 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20110 set_decommission_capacity_info_overrides_for_test(
20111 first_node.id,
20112 vec![vec![
20113 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
20114 DecommissionPoolCapacityInfo::for_test(1, layout, 0, 10, 10),
20115 DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
20116 ]],
20117 );
20118
20119 first_node
20120 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20121 .await
20122 .expect("start the operation retained by the stale node");
20123 let stale_operation = first_node.pool_meta.read().await.clone();
20124 let stale_plan = decommission_capacity_terminal_fence_plan(&stale_operation, 0)
20125 .expect("the stale operation should have a valid terminal fence plan")
20126 .expect("the stale operation should have an active reservation");
20127
20128 tokio::time::timeout(std::time::Duration::from_secs(30), first_node.decommission_cancel(0))
20129 .await
20130 .expect("cancel must persist pool metadata without re-entering its state locks")
20131 .expect("cancel the first durable operation");
20132 first_node
20133 .clear_decommission(0)
20134 .await
20135 .expect("clear the first terminal operation");
20136 first_node
20137 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20138 .await
20139 .expect("start the replacement durable operation");
20140 let replacement_plan = {
20141 let pool_meta = first_node.pool_meta.read().await;
20142 decommission_capacity_terminal_fence_plan(&pool_meta, 0)
20143 .expect("the replacement operation should have a valid terminal fence plan")
20144 .expect("the replacement operation should have an active reservation")
20145 };
20146 assert_ne!(replacement_plan.operation_id, stale_plan.operation_id);
20147
20148 *stale_node.pool_meta.write().await = stale_operation;
20149 let err = stale_node
20150 .decommission_cancel(0)
20151 .await
20152 .expect_err("a stale local operation must not be merged over the durable replacement");
20153 assert!(err.to_string().contains("differs from the durable terminal fence plan"));
20154
20155 let mut durable = PoolMeta::default();
20156 durable
20157 .load_no_lock_from_replicas(first_node.pools.clone())
20158 .await
20159 .expect("the replacement operation should remain readable from durable replicas");
20160 assert_eq!(
20161 decommission_capacity_terminal_fence_plan(&durable, 0)
20162 .expect("the durable replacement should retain a valid terminal fence plan")
20163 .expect("the durable replacement reservation should remain active"),
20164 replacement_plan
20165 );
20166 let replacement = durable.pools[0]
20167 .decommission
20168 .as_ref()
20169 .expect("the durable replacement metadata should remain present");
20170 assert!(!replacement.canceled);
20171 assert!(
20172 replacement
20173 .capacity_reservation
20174 .as_ref()
20175 .is_some_and(DecommissionCapacityReservation::active)
20176 );
20177 }
20178
20179 #[tokio::test]
20180 #[serial_test::serial]
20181 async fn decommission_capacity_publication_allows_immediate_bucket_completion() {
20182 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
20183 crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
20184 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20185 let capacity = vec![
20186 DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80),
20187 DecommissionPoolCapacityInfo::for_test(1, layout, 200, 200, 0),
20188 ];
20189 set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity.clone(), capacity.clone(), capacity]);
20190 let bucket = DecomBucketInfo {
20191 name: "capacity-publication-bucket".to_string(),
20192 prefix: String::new(),
20193 };
20194 store
20195 .save_current_pool_meta_for_decommission_start(&[0], vec![bucket.clone()])
20196 .await
20197 .expect("activate a reservation and queue the regression bucket");
20198 let (owner, initial_update, generation) = {
20199 let mut meta = store.pool_meta.write().await;
20200 meta.count_item(0, 17, false);
20201 let reservation = meta.pools[0]
20202 .decommission
20203 .as_ref()
20204 .and_then(|info| info.capacity_reservation.as_ref())
20205 .expect("the source must own an active reservation");
20206 (
20207 DecommissionCapacityOwner {
20208 source_pool_index: 0,
20209 operation_id: reservation.operation_id,
20210 generation: reservation.generation,
20211 owner_nonce: reservation.owner_nonce,
20212 mutation_id: Some(uuid::Uuid::new_v4()),
20213 },
20214 meta.pools[0].last_update,
20215 meta.pools[0]
20216 .decommission
20217 .as_ref()
20218 .and_then(|info| info.start_time)
20219 .expect("active worker generation"),
20220 )
20221 };
20222 store
20223 .run_decommission_capacity_admitted_mutation(1, Some(owner), Some(17), || async {
20224 let meta = store.pool_meta.read().await;
20225 assert!(
20226 meta.pools[0].last_update > initial_update,
20227 "durable capacity intent must publish its revision before target I/O"
20228 );
20229 assert_eq!(
20230 meta.pools[0]
20231 .decommission
20232 .as_ref()
20233 .expect("active source")
20234 .items_decommissioned,
20235 1
20236 );
20237 Ok(())
20238 })
20239 .await
20240 .expect("capacity intent and consumption must commit without a progress checkpoint");
20241 let persisted = load_pool_meta_replicas(store.pools.clone(), true)
20242 .await
20243 .expect("reload capacity consumption before bucket completion");
20244 {
20245 let meta = store.pool_meta.read().await;
20246 assert_eq!(meta.pools[0].last_update, persisted.meta.pools[0].last_update);
20247 let info = meta.pools[0].decommission.as_ref().expect("active source");
20248 assert_eq!(info.items_decommissioned, 1, "capacity publication must retain uncheckpointed progress");
20249 assert_eq!(
20250 info.progress_save_item_baseline, 0,
20251 "capacity-only saves do not checkpoint object counters"
20252 );
20253 }
20254 assert!(
20255 store
20256 .mark_decommission_bucket_done_and_save(0, generation, &bucket)
20257 .await
20258 .expect("bucket completion immediately after capacity persistence must not reject its own revision")
20259 );
20260 let persisted = load_pool_meta_replicas(store.pools.clone(), true)
20261 .await
20262 .expect("bucket completion must survive a metadata reload");
20263 let info = persisted.meta.pools[0]
20264 .decommission
20265 .as_ref()
20266 .expect("durable source progress");
20267 assert!(info.queued_buckets.is_empty());
20268 assert_eq!(info.decommissioned_buckets, vec![bucket.to_string()]);
20269 assert_eq!((info.items_decommissioned, info.bytes_done), (1, 17));
20270 let reservation = info.capacity_reservation.as_ref().expect("durable capacity consumption");
20271 assert_eq!(reservation.pending_target_physical_bytes, 0);
20272 assert_eq!(reservation.consumed_target_physical_bytes, 17);
20273 }
20274
20275 #[tokio::test]
20276 #[serial_test::serial]
20277 async fn decommission_bucket_completion_publishes_only_committed_progress() {
20278 for fault_stage in [
20279 "bucket_completion_before_save",
20280 "bucket_completion_after_save",
20281 "count_during_save",
20282 ] {
20283 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
20284 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20285 set_decommission_capacity_info_overrides_for_test(
20286 store.id,
20287 vec![vec![
20288 DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80),
20289 DecommissionPoolCapacityInfo::for_test(1, layout, 200, 200, 0),
20290 ]],
20291 );
20292 let bucket = DecomBucketInfo {
20293 name: format!("bucket-completion-{fault_stage}"),
20294 prefix: String::new(),
20295 };
20296 store
20297 .save_current_pool_meta_for_decommission_start(&[0], vec![bucket.clone()])
20298 .await
20299 .expect("activate the bucket completion fixture");
20300 let generation = {
20301 let mut meta = store.pool_meta.write().await;
20302 meta.count_item(0, 17, false);
20303 meta.pools[0]
20304 .decommission
20305 .as_ref()
20306 .and_then(|info| info.start_time)
20307 .expect("active generation")
20308 };
20309 let hook_store = Arc::clone(&store);
20310 let observed = Arc::new(AtomicBool::new(false));
20311 let hook_observed = Arc::clone(&observed);
20312 let expected_bucket = bucket.name.clone();
20313 let fault = DecommissionTestFaultGuard::install(Arc::new(move |stage, name, _, _, success| {
20314 if name != expected_bucket || !success {
20315 return false;
20316 }
20317 if stage == fault_stage {
20318 hook_observed.store(true, Ordering::SeqCst);
20319 return true;
20320 }
20321 if fault_stage == "count_during_save" && stage == "bucket_completion_after_save" {
20322 hook_observed.store(true, Ordering::SeqCst);
20323 hook_store
20324 .pool_meta
20325 .try_write()
20326 .expect("publication must not hold the local metadata lock over I/O")
20327 .count_item(0, 23, false);
20328 }
20329 false
20330 }));
20331 let result = store.mark_decommission_bucket_done_and_save(0, generation, &bucket).await;
20332 assert!(
20333 observed.load(Ordering::SeqCst),
20334 "{fault_stage}: the production save boundary was not reached"
20335 );
20336 drop(fault);
20337 let persisted = load_pool_meta_replicas(store.pools.clone(), true)
20338 .await
20339 .expect("durable metadata remains readable");
20340 let durable = persisted.meta.pools[0].decommission.as_ref().expect("durable progress");
20341 let local = store.pool_meta.read().await;
20342 let live = local.pools[0].decommission.as_ref().expect("live progress");
20343 if fault_stage == "count_during_save" {
20344 assert!(result.expect("successful bucket completion"));
20345 assert_eq!((live.items_decommissioned, live.bytes_done, live.progress_save_item_baseline), (2, 40, 1));
20346 assert_eq!((durable.items_decommissioned, durable.bytes_done), (1, 17));
20347 assert!(live.queued_buckets.is_empty());
20348 assert_eq!(live.decommissioned_buckets, vec![bucket.to_string()]);
20349 } else {
20350 let err = result.expect_err("injected failure must reach the caller");
20351 assert!(err.to_string().contains(fault_stage));
20352 assert_eq!(live.queued_buckets, vec![bucket.to_string()]);
20353 assert!(live.decommissioned_buckets.is_empty());
20354 assert_eq!(live.progress_save_item_baseline, 0);
20355 if fault_stage == "bucket_completion_before_save" {
20356 assert_eq!(durable.queued_buckets, vec![bucket.to_string()]);
20357 assert!(durable.decommissioned_buckets.is_empty());
20358 } else {
20359 assert!(durable.queued_buckets.is_empty());
20360 assert_eq!(durable.decommissioned_buckets, vec![bucket.to_string()]);
20361 assert_eq!((durable.items_decommissioned, durable.bytes_done), (1, 17));
20362 }
20363 }
20364 drop(local);
20365 if fault_stage == "bucket_completion_after_save" {
20366 store
20367 .ensure_pool_meta_side_effects_safe("ambiguous bucket completion")
20368 .await
20369 .expect_err("commit without publication must require recovery before another write");
20370 } else {
20371 store
20372 .ensure_pool_meta_side_effects_safe("bucket completion")
20373 .await
20374 .expect("no ambiguous commit");
20375 }
20376 }
20377 }
20378
20379 #[tokio::test]
20380 #[serial_test::serial]
20381 async fn legacy_capacity_model_holds_the_global_pool_meta_fence_through_target_io() {
20382 let (_temp_dirs, store, _other_store) =
20383 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
20384 crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
20385 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20386 set_decommission_capacity_info_overrides_for_test(
20387 store.id,
20388 vec![vec![
20389 DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80),
20390 DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
20391 DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0),
20392 ]],
20393 );
20394 let _proof_guard = crate::services::notification_sys::without_decommission_target_fence_fleet_proof_for_test();
20395 store
20396 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20397 .await
20398 .expect("activate a legacy reservation spanning two targets");
20399 let base_owner = {
20400 let pool_meta = store.pool_meta.read().await;
20401 let reservation = pool_meta.pools[0]
20402 .decommission
20403 .as_ref()
20404 .and_then(|info| info.capacity_reservation.as_ref())
20405 .expect("legacy capacity reservation should exist");
20406 assert_eq!(reservation.model_version, DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION);
20407 assert_eq!(reservation.targets.iter().map(|target| target.pool_index).collect::<Vec<_>>(), vec![1, 2]);
20408 DecommissionCapacityOwner {
20409 source_pool_index: 0,
20410 operation_id: reservation.operation_id,
20411 generation: reservation.generation,
20412 owner_nonce: reservation.owner_nonce,
20413 mutation_id: None,
20414 }
20415 };
20416
20417 let (first_entered_tx, first_entered_rx) = tokio::sync::oneshot::channel();
20418 let (first_release_tx, first_release_rx) = tokio::sync::oneshot::channel();
20419 let first_store = Arc::clone(&store);
20420 let first_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
20421 let first = tokio::spawn(async move {
20422 first_store
20423 .run_decommission_capacity_admitted_mutation_with_capacity_lease(1, Some(first_owner), Some(1), |_| async {
20424 first_entered_tx.send(()).expect("first legacy mutation should be observed");
20425 first_release_rx.await.expect("first legacy mutation should be released");
20426 Ok(())
20427 })
20428 .await
20429 });
20430 tokio::time::timeout(std::time::Duration::from_secs(30), first_entered_rx)
20431 .await
20432 .expect("first legacy mutation should enter")
20433 .expect("first legacy mutation should report entry");
20434 assert!(
20435 store.pool_meta_save_gate.try_lock().is_err(),
20436 "legacy target I/O must retain the local pool metadata save gate"
20437 );
20438 let pool_meta_lock = store.pools[0]
20439 .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
20440 .await
20441 .expect("create the legacy pool metadata lock probe");
20442 assert!(
20443 pool_meta_lock
20444 .get_write_lock_quiet(std::time::Duration::from_millis(300))
20445 .await
20446 .is_err(),
20447 "legacy target I/O must retain the distributed pool metadata write fence"
20448 );
20449
20450 let (second_entered_tx, mut second_entered_rx) = tokio::sync::oneshot::channel();
20451 let (second_release_tx, second_release_rx) = tokio::sync::oneshot::channel();
20452 let second_store = Arc::clone(&store);
20453 let second_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
20454 let second = tokio::spawn(async move {
20455 second_store
20456 .run_decommission_capacity_admitted_mutation_with_capacity_lease(2, Some(second_owner), Some(1), |_| async {
20457 second_entered_tx.send(()).expect("second legacy mutation should be observed");
20458 second_release_rx.await.expect("second legacy mutation should be released");
20459 Ok(())
20460 })
20461 .await
20462 });
20463 tokio::time::timeout(std::time::Duration::from_millis(500), &mut second_entered_rx)
20464 .await
20465 .expect_err("a different target must still wait behind the legacy global fence");
20466
20467 first_release_tx.send(()).expect("release first legacy mutation");
20468 first
20469 .await
20470 .expect("first legacy mutation task should not panic")
20471 .expect("first legacy mutation should finalize");
20472 tokio::time::timeout(std::time::Duration::from_secs(30), &mut second_entered_rx)
20473 .await
20474 .expect("second legacy mutation should enter after the first finalizes")
20475 .expect("second legacy mutation should report entry");
20476 assert!(
20477 pool_meta_lock
20478 .get_write_lock_quiet(std::time::Duration::from_millis(300))
20479 .await
20480 .is_err(),
20481 "the second legacy target I/O must also retain the global fence"
20482 );
20483 second_release_tx.send(()).expect("release second legacy mutation");
20484 second
20485 .await
20486 .expect("second legacy mutation task should not panic")
20487 .expect("second legacy mutation should finalize");
20488
20489 let pool_meta_guard = pool_meta_lock
20490 .get_write_lock(std::time::Duration::from_secs(30))
20491 .await
20492 .expect("the global pool metadata fence should release after both target tails");
20493 drop(pool_meta_guard);
20494 let pool_meta = store.pool_meta.read().await;
20495 let reservation = pool_meta.pools[0]
20496 .decommission
20497 .as_ref()
20498 .and_then(|info| info.capacity_reservation.as_ref())
20499 .expect("legacy reservation should remain active");
20500 assert_eq!(reservation.pending_target_physical_bytes, 0);
20501 assert_eq!(reservation.consumed_target_physical_bytes, 2);
20502 }
20503
20504 #[tokio::test]
20505 #[serial_test::serial]
20506 async fn decommission_capacity_mutations_overlap_across_targets_but_serialize_per_target() {
20507 let (_temp_dirs, store, _other_store) =
20508 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
20509 crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
20510 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20511 let capacity_snapshot = vec![
20512 DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80),
20513 DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
20514 DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0),
20515 ];
20516 set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot]);
20517 store
20518 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20519 .await
20520 .expect("activate a reservation spanning two targets");
20521
20522 let base_owner = {
20523 let pool_meta = store.pool_meta.read().await;
20524 let reservation = pool_meta.pools[0]
20525 .decommission
20526 .as_ref()
20527 .and_then(|info| info.capacity_reservation.as_ref())
20528 .expect("active capacity reservation should exist");
20529 assert_eq!(
20530 reservation.targets.iter().map(|target| target.pool_index).collect::<Vec<_>>(),
20531 vec![1, 2],
20532 "the fixture must reserve both target pools"
20533 );
20534 DecommissionCapacityOwner {
20535 source_pool_index: 0,
20536 operation_id: reservation.operation_id,
20537 generation: reservation.generation,
20538 owner_nonce: reservation.owner_nonce,
20539 mutation_id: None,
20540 }
20541 };
20542
20543 let (first_entered_tx, first_entered_rx) = tokio::sync::oneshot::channel();
20544 let (first_release_tx, first_release_rx) = tokio::sync::oneshot::channel();
20545 let first_store = Arc::clone(&store);
20546 let first_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
20547 let first = tokio::spawn(async move {
20548 first_store
20549 .run_decommission_capacity_admitted_mutation_with_capacity_lease(1, Some(first_owner), Some(1), |_| async {
20550 first_entered_tx.send(()).expect("first target mutation should be observed");
20551 first_release_rx.await.expect("first target mutation should be released");
20552 Ok(())
20553 })
20554 .await
20555 });
20556 tokio::time::timeout(std::time::Duration::from_secs(30), first_entered_rx)
20557 .await
20558 .expect("first target mutation should enter without hanging")
20559 .expect("first target mutation should report entry");
20560
20561 let (second_entered_tx, second_entered_rx) = tokio::sync::oneshot::channel();
20562 let (second_release_tx, second_release_rx) = tokio::sync::oneshot::channel();
20563 let second_store = Arc::clone(&store);
20564 let second_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
20565 let second = tokio::spawn(async move {
20566 second_store
20567 .run_decommission_capacity_admitted_mutation_with_capacity_lease(2, Some(second_owner), Some(1), |_| async {
20568 second_entered_tx.send(()).expect("second target mutation should be observed");
20569 second_release_rx.await.expect("second target mutation should be released");
20570 Ok(())
20571 })
20572 .await
20573 });
20574 tokio::time::timeout(std::time::Duration::from_secs(30), second_entered_rx)
20575 .await
20576 .expect("a different target mutation should overlap instead of waiting for the first")
20577 .expect("second target mutation should report entry");
20578
20579 let save_guard = store
20580 .pool_meta_save_gate
20581 .try_lock()
20582 .expect("target I/O must not retain the local pool metadata save gate");
20583 drop(save_guard);
20584 let pool_meta_lock = store.pools[0]
20585 .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
20586 .await
20587 .expect("create a pool metadata lock probe");
20588 let pool_meta_guard = pool_meta_lock
20589 .get_write_lock_quiet(std::time::Duration::from_secs(1))
20590 .await
20591 .expect("target I/O must not retain the distributed pool metadata write lock");
20592 drop(pool_meta_guard);
20593
20594 let blocked_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
20595 let blocked = store
20596 .run_decommission_capacity_admitted_mutation(1, Some(blocked_owner), Some(1), || async { Ok(()) })
20597 .await
20598 .expect_err("a second mutation on the same target must wait behind its target gate");
20599 assert!(is_decommission_capacity_blocked_error(&blocked));
20600 assert!(
20601 is_decommission_capacity_target_gate_busy(&blocked),
20602 "same-target serialization must remain distinguishable from durable intent conflicts"
20603 );
20604 assert!(
20605 decommission_capacity_retry_kind(&blocked, 0).is_none(),
20606 "target contention must be retried inside the current version instead of replaying the entry"
20607 );
20608
20609 let cancel_store = Arc::clone(&store);
20610 let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
20611 tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel)
20612 .await
20613 .expect_err("terminal reservation release must wait for prepared target mutations");
20614 assert!(
20615 store.pool_meta.read().await.pools[0]
20616 .decommission
20617 .as_ref()
20618 .and_then(|info| info.capacity_reservation.as_ref())
20619 .is_some_and(DecommissionCapacityReservation::active),
20620 "cancellation must not release the reservation while target mutations are active"
20621 );
20622
20623 first_release_tx.send(()).expect("release first target mutation");
20624 second_release_tx.send(()).expect("release second target mutation");
20625 first
20626 .await
20627 .expect("first target mutation task should not panic")
20628 .expect("first target mutation should finalize");
20629 second
20630 .await
20631 .expect("second target mutation task should not panic")
20632 .expect("second target mutation should finalize");
20633 tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
20634 .await
20635 .expect("cancel must finish after target mutations release their gates")
20636 .expect("decommission cancellation task should not panic")
20637 .expect("decommission cancellation should finish after target mutations");
20638
20639 let pool_meta = store.pool_meta.read().await;
20640 let reservation = pool_meta.pools[0]
20641 .decommission
20642 .as_ref()
20643 .and_then(|info| info.capacity_reservation.as_ref())
20644 .expect("released capacity reservation should remain available for accounting");
20645 assert!(!reservation.active());
20646 assert_eq!(reservation.release_reason.as_deref(), Some(DECOMMISSION_CAPACITY_RELEASE_CANCELED));
20647 assert_eq!(reservation.pending_target_physical_bytes, 0);
20648 assert_eq!(reservation.consumed_target_physical_bytes, 2);
20649 assert_eq!(
20650 reservation
20651 .targets
20652 .iter()
20653 .map(|target| (target.pool_index, target.consumed_physical_bytes))
20654 .collect::<Vec<_>>(),
20655 vec![(1, 1), (2, 1)]
20656 );
20657 }
20658
20659 #[tokio::test]
20660 #[serial_test::serial]
20661 async fn ordinary_write_capacity_fence_serializes_with_decommission_activation() {
20662 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
20663 let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
20664 let (release_tx, release_rx) = tokio::sync::oneshot::channel();
20665 let write_store = Arc::clone(&store);
20666 let ordinary_write = tokio::spawn(async move {
20667 write_store
20668 .run_decommission_capacity_admitted_mutation(1, None, None, || async move {
20669 entered_tx.send(()).expect("ordinary write admission should be observed");
20670 release_rx.await.expect("ordinary write should be released");
20671 Ok(())
20672 })
20673 .await
20674 });
20675 entered_rx
20676 .await
20677 .expect("ordinary write should hold the shared capacity fence");
20678
20679 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20680 set_decommission_capacity_info_overrides_for_test(
20681 store.id,
20682 vec![vec![
20683 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
20684 DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1),
20685 ]],
20686 );
20687 let activation_store = Arc::clone(&store);
20688 let activation = tokio::spawn(async move {
20689 activation_store
20690 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20691 .await
20692 });
20693 tokio::task::yield_now().await;
20694 assert!(
20695 !activation.is_finished(),
20696 "activation must wait until the already-admitted ordinary write leaves the distributed capacity boundary"
20697 );
20698
20699 release_tx.send(()).expect("ordinary write should be released");
20700 ordinary_write
20701 .await
20702 .expect("ordinary write task should join")
20703 .expect("the pre-activation ordinary write should complete");
20704 let err = activation
20705 .await
20706 .expect("activation task should join")
20707 .expect_err("activation must recheck and reject capacity consumed by the ordinary write");
20708 assert!(err.to_string().contains("requires 60 bytes, but 59 bytes are available"));
20709 }
20710
20711 async fn single_pool_capacity_admission_test_store() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
20712 let (temp_dirs, store) =
20713 crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta::default()).await;
20714 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
20715 (temp_dirs, store)
20716 }
20717
20718 #[tokio::test]
20719 #[serial_test::serial]
20720 async fn single_pool_public_writes_skip_decommission_capacity_admission() {
20721 let (_temp_dirs, store) = single_pool_capacity_admission_test_store().await;
20722 let bucket = format!("single-pool-capacity-skip-{}", uuid::Uuid::new_v4());
20723 store
20724 .make_bucket(&bucket, &MakeBucketOptions::default())
20725 .await
20726 .expect("create single-pool bucket before blocking pool metadata writes");
20727 let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
20728 store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
20729 let write_gate = store.pool_meta_write_gate_status().await;
20730 assert!(!write_gate.writes_ready);
20731 assert!(write_gate.write_blocked);
20732 assert!(!write_gate.transaction_aborted);
20733
20734 let object = "ordinary-put.bin";
20735 let mut put_data = crate::object_api::PutObjReader::from_vec(b"ordinary single-pool body".to_vec());
20736 store
20737 .put_object(&bucket, object, &mut put_data, &ObjectOptions::default())
20738 .await
20739 .expect("single-pool ordinary PUT must not enter decommission capacity admission");
20740 store
20741 .get_object_info(&bucket, object, &ObjectOptions::default())
20742 .await
20743 .expect("single-pool ordinary PUT must remain readable");
20744
20745 let multipart_object = "ordinary-multipart.bin";
20746 let upload = store
20747 .new_multipart_upload(
20748 &bucket,
20749 multipart_object,
20750 &ObjectOptions {
20751 expected_bucket_incarnation_id: Some(incarnation),
20752 ..Default::default()
20753 },
20754 )
20755 .await
20756 .expect("single-pool MPU creation must not enter decommission capacity admission");
20757 let mut part_data = crate::object_api::PutObjReader::from_vec(b"single-pool multipart body".to_vec());
20758 let part = store
20759 .put_object_part(
20760 &bucket,
20761 multipart_object,
20762 &upload.upload_id,
20763 1,
20764 &mut part_data,
20765 &ObjectOptions {
20766 expected_bucket_incarnation_id: Some(incarnation),
20767 ..Default::default()
20768 },
20769 )
20770 .await
20771 .expect("single-pool UploadPart must not enter decommission capacity admission");
20772 store
20773 .clone()
20774 .complete_multipart_upload(
20775 &bucket,
20776 multipart_object,
20777 &upload.upload_id,
20778 vec![crate::storage_api_contracts::multipart::CompletePart {
20779 part_num: part.part_num,
20780 etag: part.etag,
20781 ..Default::default()
20782 }],
20783 &ObjectOptions {
20784 expected_bucket_incarnation_id: Some(incarnation),
20785 ..Default::default()
20786 },
20787 )
20788 .await
20789 .expect("single-pool CompleteMultipartUpload must not enter decommission capacity admission");
20790 store
20791 .get_object_info(&bucket, multipart_object, &ObjectOptions::default())
20792 .await
20793 .expect("single-pool completed MPU must remain readable");
20794 }
20795
20796 #[tokio::test]
20797 #[serial_test::serial]
20798 async fn multipart_mutations_locate_later_upload_before_reserved_pool_admission() {
20799 let (_temp_dirs, store, _other_store) =
20800 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
20801 let bucket = format!("multipart-capacity-routing-{}", uuid::Uuid::new_v4());
20802 let object = "later-pool.bin";
20803 store
20804 .make_bucket(&bucket, &MakeBucketOptions::default())
20805 .await
20806 .expect("create multipart routing bucket");
20807 let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
20808 let bucket_guard = store
20809 .acquire_bucket_lifecycle_read_lock(&bucket)
20810 .await
20811 .expect("acquire multipart routing bucket lifecycle guard");
20812 let mut upload_opts = ObjectOptions {
20813 expected_bucket_incarnation_id: Some(incarnation),
20814 ..Default::default()
20815 };
20816 upload_opts.add_bucket_lifecycle_lock_guard(&bucket_guard);
20817 let upload = store.pools[1]
20818 .new_multipart_upload(&bucket, object, &upload_opts)
20819 .await
20820 .expect("seed an upload in the later pool");
20821
20822 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20823 set_decommission_capacity_info_overrides_for_test(
20824 store.id,
20825 vec![vec![
20826 DecommissionPoolCapacityInfo::for_test(0, layout, 20, 20, 0),
20827 DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
20828 DecommissionPoolCapacityInfo::for_test(2, layout, 0, 10, 10),
20829 ]],
20830 );
20831 store
20832 .save_current_pool_meta_for_decommission_start(&[2], Vec::new())
20833 .await
20834 .expect("reserve the first target pool");
20835 {
20836 let pool_meta = store.pool_meta.read().await;
20837 let targets = &pool_meta.pools[2]
20838 .decommission
20839 .as_ref()
20840 .and_then(|info| info.capacity_reservation.as_ref())
20841 .expect("source reservation should exist")
20842 .targets;
20843 assert_eq!(targets.len(), 1);
20844 assert_eq!(targets[0].pool_index, 0, "the first probed pool must be reserved");
20845 }
20846
20847 let mut data = crate::object_api::PutObjReader::from_vec(b"multipart body".to_vec());
20848 let part = store
20849 .put_object_part(&bucket, object, &upload.upload_id, 1, &mut data, &ObjectOptions::default())
20850 .await
20851 .expect("put-part must locate the later upload before capacity admission");
20852 store
20853 .clone()
20854 .complete_multipart_upload(
20855 &bucket,
20856 object,
20857 &upload.upload_id,
20858 vec![crate::storage_api_contracts::multipart::CompletePart {
20859 part_num: part.part_num,
20860 etag: part.etag,
20861 ..Default::default()
20862 }],
20863 &ObjectOptions::default(),
20864 )
20865 .await
20866 .expect("complete must locate the later upload before capacity admission");
20867
20868 store.pools[1]
20869 .get_object_info(&bucket, object, &ObjectOptions::default())
20870 .await
20871 .expect("the later-pool multipart upload should commit in place");
20872 let first_pool_result = store.pools[0]
20873 .get_object_info(&bucket, object, &ObjectOptions::default())
20874 .await;
20875 assert!(
20876 first_pool_result
20877 .as_ref()
20878 .err()
20879 .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)),
20880 "the reserved first pool must not receive the multipart mutation"
20881 );
20882 }
20883
20884 #[tokio::test]
20885 #[serial_test::serial]
20886 async fn expired_decommission_capacity_lease_requires_recovery_before_entry() {
20887 let (_temp_dirs, store, _other_store) =
20888 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
20889 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20890 set_decommission_capacity_info_overrides_for_test(
20891 store.id,
20892 vec![vec![
20893 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1, 1),
20894 DecommissionPoolCapacityInfo::for_test(1, layout, 2, 2, 0),
20895 DecommissionPoolCapacityInfo::for_test(2, layout, 2, 2, 0),
20896 ]],
20897 );
20898 store
20899 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20900 .await
20901 .expect("activate the expired lease reservation");
20902 let generation = store
20903 .active_decommission_generation(0)
20904 .await
20905 .expect("read the active decommission generation");
20906 {
20907 let mut pool_meta = store.pool_meta.write().await;
20908 let reservation = pool_meta.pools[0]
20909 .decommission
20910 .as_mut()
20911 .and_then(|info| info.capacity_reservation.as_mut())
20912 .expect("the expired lease reservation should exist");
20913 reservation.expires_at = OffsetDateTime::now_utc() - Duration::seconds(1);
20914 }
20915
20916 let err = store
20917 .decommission_capacity_owner_for_worker(0, generation)
20918 .await
20919 .expect_err("an expired capacity lease must not fall back to ownerless migration");
20920 assert!(is_decommission_capacity_blocked_error(&err));
20921 }
20922
20923 #[tokio::test]
20924 #[serial_test::serial]
20925 async fn expired_capacity_owner_nonce_rejects_stale_put_and_multipart_without_consuming() {
20926 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
20927 let bucket = format!("stale-capacity-owner-{}", uuid::Uuid::new_v4());
20928 let multipart_object = "stale-multipart.bin";
20929 store
20930 .make_bucket(&bucket, &MakeBucketOptions::default())
20931 .await
20932 .expect("create stale owner bucket");
20933 let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
20934 let bucket_guard = store
20935 .acquire_bucket_lifecycle_read_lock(&bucket)
20936 .await
20937 .expect("acquire stale owner bucket lifecycle guard");
20938 let mut upload_opts = ObjectOptions {
20939 expected_bucket_incarnation_id: Some(incarnation),
20940 ..Default::default()
20941 };
20942 upload_opts.add_bucket_lifecycle_lock_guard(&bucket_guard);
20943 let upload = store.pools[1]
20944 .new_multipart_upload(&bucket, multipart_object, &upload_opts)
20945 .await
20946 .expect("seed multipart upload before reserving the target");
20947
20948 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20949 set_decommission_capacity_info_overrides_for_test(
20950 store.id,
20951 vec![vec![
20952 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
20953 DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
20954 ]],
20955 );
20956 store
20957 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20958 .await
20959 .expect("activate target reservation");
20960
20961 let stale_owner = {
20962 let pool_meta = store.pool_meta.read().await;
20963 let reservation = pool_meta.pools[0]
20964 .decommission
20965 .as_ref()
20966 .and_then(|info| info.capacity_reservation.as_ref())
20967 .expect("active reservation should exist");
20968 DecommissionCapacityOwner {
20969 source_pool_index: 0,
20970 operation_id: reservation.operation_id,
20971 generation: reservation.generation,
20972 owner_nonce: reservation.owner_nonce,
20973 mutation_id: None,
20974 }
20975 };
20976 {
20977 let mut pool_meta = store.pool_meta.write().await;
20978 let reservation = pool_meta.pools[0]
20979 .decommission
20980 .as_mut()
20981 .and_then(|info| info.capacity_reservation.as_mut())
20982 .expect("active reservation should remain mutable");
20983 reservation.expires_at = OffsetDateTime::now_utc() - Duration::seconds(1);
20984 }
20985 store
20986 .save_current_pool_meta(&[0])
20987 .await
20988 .expect("persist the expired lease before recovery");
20989 store
20990 .pause_decommission_for_capacity(0, &decommission_capacity_blocked_error("test lease recovery"))
20991 .await
20992 .expect("pause recovery should renew the lease with a new owner nonce");
20993
20994 let (current_owner, before_progress) = {
20995 let pool_meta = store.pool_meta.read().await;
20996 let reservation = pool_meta.pools[0]
20997 .decommission
20998 .as_ref()
20999 .and_then(|info| info.capacity_reservation.as_ref())
21000 .expect("renewed reservation should exist");
21001 (
21002 DecommissionCapacityOwner {
21003 source_pool_index: 0,
21004 operation_id: reservation.operation_id,
21005 generation: reservation.generation,
21006 owner_nonce: reservation.owner_nonce,
21007 mutation_id: None,
21008 },
21009 (
21010 reservation.consumed_target_physical_bytes,
21011 reservation.inflight_target_physical_bytes,
21012 reservation.pending_target_physical_bytes,
21013 reservation.observed_target_physical_bytes,
21014 ),
21015 )
21016 };
21017 assert_ne!(stale_owner.owner_nonce, current_owner.owner_nonce);
21018 {
21019 let pool_meta = store.pool_meta.read().await;
21020 ensure_decommission_target_owner_admission(&pool_meta, current_owner, 1, 1, OffsetDateTime::now_utc())
21021 .expect("the renewed owner should retain its reservation");
21022 assert!(
21023 ensure_decommission_target_owner_admission(&pool_meta, stale_owner, 1, 1, OffsetDateTime::now_utc(),).is_err(),
21024 "the expired owner token must be rejected after nonce rotation"
21025 );
21026 }
21027
21028 let stale_put_object = "stale-put.bin";
21029 let mut stale_put_opts = ObjectOptions {
21030 data_movement: true,
21031 version_id: Some(uuid::Uuid::new_v4().to_string()),
21032 ..ObjectOptions::with_capacity_expected_data_bytes(Some(9))
21033 };
21034 stale_owner.apply_to(&mut stale_put_opts);
21035 let mut stale_put_data = crate::object_api::PutObjReader::from_vec(b"stale put".to_vec());
21036 let put_err = store
21037 .put_object_for_data_movement(&bucket, stale_put_object, &mut stale_put_data, &stale_put_opts, None)
21038 .await
21039 .expect_err("stale owner PUT must fail before selecting a target");
21040 assert!(is_decommission_capacity_blocked_error(&put_err));
21041
21042 let mut stale_part_opts = ObjectOptions {
21043 data_movement: true,
21044 part_number: Some(1),
21045 expected_bucket_incarnation_id: Some(incarnation),
21046 ..Default::default()
21047 };
21048 stale_owner.apply_to(&mut stale_part_opts);
21049 let mut stale_part_data = crate::object_api::PutObjReader::from_vec(b"stale part".to_vec());
21050 let part_err = store
21051 .put_object_part_for_data_movement(
21052 1,
21053 &bucket,
21054 multipart_object,
21055 &upload.upload_id,
21056 &mut stale_part_data,
21057 &stale_part_opts,
21058 )
21059 .await
21060 .expect_err("stale owner multipart PUT must fail before mutation");
21061 assert!(is_decommission_capacity_blocked_error(&part_err));
21062
21063 let after_progress = {
21064 let pool_meta = store.pool_meta.read().await;
21065 let reservation = pool_meta.pools[0]
21066 .decommission
21067 .as_ref()
21068 .and_then(|info| info.capacity_reservation.as_ref())
21069 .expect("renewed reservation should remain active");
21070 (
21071 reservation.consumed_target_physical_bytes,
21072 reservation.inflight_target_physical_bytes,
21073 reservation.pending_target_physical_bytes,
21074 reservation.observed_target_physical_bytes,
21075 )
21076 };
21077 assert_eq!(after_progress, before_progress, "stale mutations must not consume the capacity ledger");
21078 let parts = store.pools[1]
21079 .list_object_parts(
21080 &bucket,
21081 multipart_object,
21082 &upload.upload_id,
21083 None,
21084 1_000,
21085 &ObjectOptions {
21086 expected_bucket_incarnation_id: Some(incarnation),
21087 ..Default::default()
21088 },
21089 )
21090 .await
21091 .expect("the seeded upload should remain readable");
21092 assert!(parts.parts.is_empty(), "the stale multipart write must not stage a part");
21093 assert!(
21094 store.pools[1]
21095 .get_object_info(&bucket, stale_put_object, &ObjectOptions::default())
21096 .await
21097 .is_err(),
21098 "the stale PUT must not create a target object"
21099 );
21100 }
21101
21102 #[tokio::test]
21103 #[serial_test::serial]
21104 async fn decommission_activation_lock_recheck_rejects_sudden_space_drop_without_persisting() {
21105 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
21106 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
21107 let source = DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30);
21108 let preflight = vec![source, DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0)];
21109 {
21110 let pool_meta = store.pool_meta.read().await;
21111 ensure_decommission_start_target_capacity(&pool_meta, &[0], &preflight, true)
21112 .expect("the pre-lock capacity snapshot should fit exactly");
21113 }
21114
21115 set_decommission_capacity_info_overrides_for_test(
21116 store.id,
21117 vec![vec![source, DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)]],
21118 );
21119 let err = store
21120 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
21121 .await
21122 .expect_err("the capacity snapshot inside the activation lock must be authoritative");
21123 assert!(err.to_string().contains("requires 60 bytes, but 59 bytes are available"));
21124
21125 let local = store.pool_meta.read().await;
21126 assert!(local.pools[0].decommission.is_none());
21127 drop(local);
21128 let mut persisted = PoolMeta::default();
21129 persisted
21130 .load_no_lock_from_replicas(store.pools.clone())
21131 .await
21132 .expect("the rejected activation must leave baseline metadata readable");
21133 assert!(persisted.pools[0].decommission.is_none());
21134 }
21135
21136 #[tokio::test]
21137 #[serial_test::serial]
21138 async fn decommission_worker_pauses_on_runtime_capacity_shortage_without_source_side_effect() {
21139 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
21140 let bucket = format!("capacity-blocked-{}", uuid::Uuid::new_v4());
21141 let object = "object.bin";
21142 store
21143 .make_bucket(&bucket, &MakeBucketOptions::default())
21144 .await
21145 .expect("the production-path bucket should be created");
21146 let mut source = crate::object_api::PutObjReader::from_vec(b"source remains authoritative".to_vec());
21147 store.pools[0]
21148 .put_object(&bucket, object, &mut source, &ObjectOptions::default())
21149 .await
21150 .expect("the source object should be written before decommission starts");
21151
21152 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
21153 let enough = vec![
21154 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
21155 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
21156 ];
21157 set_decommission_capacity_info_overrides_for_test(store.id, vec![enough.clone()]);
21158 store
21159 .save_current_pool_meta_for_decommission_start(
21160 &[0],
21161 vec![DecomBucketInfo {
21162 name: bucket.clone(),
21163 prefix: String::new(),
21164 }],
21165 )
21166 .await
21167 .expect("the initial reservation should be activated");
21168
21169 let shortage = vec![enough[0], DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)];
21170 set_decommission_capacity_info_overrides_for_test(store.id, vec![shortage]);
21171 let canceler = DecommissionCanceler::new(CancellationToken::new());
21172 store.decommission_cancelers.write().await[0] = Some(canceler.clone());
21173 store
21174 .do_decommission_in_routine(canceler, 0, Arc::new(Semaphore::new(1)))
21175 .await
21176 .expect("runtime capacity shortage should pause the worker, not fail it");
21177
21178 store.pools[0]
21179 .get_object_info(&bucket, object, &ObjectOptions::default())
21180 .await
21181 .expect("capacity pause must not delete the source object");
21182 let target_result = store.pools[1]
21183 .get_object_info(&bucket, object, &ObjectOptions::default())
21184 .await;
21185 assert!(
21186 target_result
21187 .as_ref()
21188 .err()
21189 .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)),
21190 "capacity pause must happen before a target mutation"
21191 );
21192
21193 let local = store.pool_meta.read().await;
21194 let info = local.pools[0]
21195 .decommission
21196 .as_ref()
21197 .expect("the blocked decommission state should remain present");
21198 assert!(
21199 !info.complete && !info.failed && !info.canceled,
21200 "capacity-blocked state unexpectedly became terminal: complete={}, failed={}, canceled={}, blocked_reason={:?}",
21201 info.complete,
21202 info.failed,
21203 info.canceled,
21204 info.capacity_blocked_reason
21205 );
21206 assert_eq!(info.items_decommission_failed, 0);
21207 assert_eq!(info.bytes_failed, 0);
21208 assert!(info.capacity_blocked_reason.is_some());
21209 assert!(
21210 info.capacity_reservation
21211 .as_ref()
21212 .is_some_and(DecommissionCapacityReservation::active),
21213 "blocked work must retain and renew its reservation"
21214 );
21215 drop(local);
21216
21217 let mut persisted = PoolMeta::default();
21218 persisted
21219 .load_no_lock_from_replicas(store.pools.clone())
21220 .await
21221 .expect("the blocked state should be durable");
21222 let persisted_info = persisted.pools[0]
21223 .decommission
21224 .as_ref()
21225 .expect("the durable blocked state should remain nonterminal");
21226 assert!(!persisted_info.complete && !persisted_info.failed && !persisted_info.canceled);
21227 assert!(persisted_info.capacity_blocked_reason.is_some());
21228 assert!(
21229 persisted_info
21230 .capacity_reservation
21231 .as_ref()
21232 .is_some_and(DecommissionCapacityReservation::active)
21233 );
21234 }
21235
21236 #[tokio::test]
21237 #[serial_test::serial]
21238 async fn decommission_worker_rechecks_runtime_capacity_before_empty_background_completion() {
21239 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
21240 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
21241 let enough = vec![
21242 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
21243 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
21244 ];
21245 set_decommission_capacity_info_overrides_for_test(store.id, vec![enough.clone()]);
21246 store
21247 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
21248 .await
21249 .expect("the initial reservation should be activated");
21250
21251 let shortage = vec![enough[0], DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)];
21252 set_decommission_capacity_info_overrides_for_test(store.id, vec![shortage]);
21253 let canceler = DecommissionCanceler::new(CancellationToken::new());
21254 store.decommission_cancelers.write().await[0] = Some(canceler.clone());
21255 store
21256 .do_decommission_in_routine(canceler, 0, Arc::new(Semaphore::new(1)))
21257 .await
21258 .expect("runtime capacity shortage should pause the worker before background completion");
21259
21260 let local = store.pool_meta.read().await;
21261 let info = local.pools[0]
21262 .decommission
21263 .as_ref()
21264 .expect("the blocked decommission state should remain present");
21265 assert!(!info.complete && !info.failed && !info.canceled);
21266 assert!(info.capacity_blocked_reason.is_some());
21267 assert!(
21268 info.capacity_reservation
21269 .as_ref()
21270 .is_some_and(DecommissionCapacityReservation::active)
21271 );
21272 }
21273
21274 fn pool_meta_replica_test_meta(cmd_line: &str) -> PoolMeta {
21275 PoolMeta {
21276 version: POOL_META_VERSION,
21277 pools: vec![PoolStatus {
21278 id: 0,
21279 cmd_line: cmd_line.to_string(),
21280 last_update: OffsetDateTime::UNIX_EPOCH,
21281 decommission: None,
21282 }],
21283 dont_save: false,
21284 }
21285 }
21286
21287 fn pool_meta_replica_test_data(cmd_line: &str) -> Vec<u8> {
21288 pool_meta_replica_test_meta(cmd_line)
21289 .encode_config_data()
21290 .expect("pool metadata should encode")
21291 }
21292
21293 fn pool_meta_v1_replica_test_data(meta: &PoolMeta) -> Vec<u8> {
21294 let mut data = Vec::new();
21295 data.write_u16::<LittleEndian>(POOL_META_FORMAT)
21296 .expect("pool metadata format should encode");
21297 data.write_u16::<LittleEndian>(POOL_META_V1_VERSION)
21298 .expect("pool metadata version should encode");
21299 PersistedPoolMetaV1::from(meta)
21300 .serialize(&mut Serializer::new(&mut data))
21301 .expect("legacy pool metadata should encode");
21302 data
21303 }
21304
21305 fn pool_meta_persisted_v1_replica_test_data(cmd_line: &str) -> Vec<u8> {
21306 pool_meta_v1_replica_test_data(&pool_meta_replica_test_meta(cmd_line))
21307 }
21308
21309 fn pool_meta_legacy_v1_replica_test_data(cmd_line: &str) -> Vec<u8> {
21310 let mut data = Vec::new();
21311 data.write_u16::<LittleEndian>(POOL_META_FORMAT)
21312 .expect("pool metadata format should encode");
21313 data.write_u16::<LittleEndian>(POOL_META_V1_VERSION)
21314 .expect("pool metadata version should encode");
21315 LegacyPoolMeta {
21316 version: POOL_META_V1_VERSION,
21317 pools: vec![LegacyPoolStatus {
21318 id: 0,
21319 cmd_line: cmd_line.to_string(),
21320 last_update: OffsetDateTime::UNIX_EPOCH,
21321 decommission: None,
21322 }],
21323 dont_save: false,
21324 }
21325 .serialize(&mut Serializer::new(&mut data))
21326 .expect("legacy v1 pool metadata should encode");
21327 data
21328 }
21329
21330 #[test]
21331 fn pool_meta_replica_selection_falls_back_from_corrupt_first_copy() {
21332 let selection = select_pool_meta_replica(vec![
21333 PoolMetaReplica::Corrupt("truncated".to_string()),
21334 decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
21335 ])
21336 .expect("a validated backup replica should be selected");
21337
21338 assert!(selection.replica_state.needs_repair);
21339 assert!(selection.replica_state.repair_write_safe);
21340 assert_eq!(selection.meta.pools.len(), 1);
21341 assert_eq!(selection.meta.pools[0].cmd_line, "pool-0");
21342 }
21343
21344 #[test]
21345 fn pool_meta_replica_selection_rejects_incompatible_copy() {
21346 let valid = pool_meta_replica_test_data("pool-0");
21347 let mut incompatible = valid.clone();
21348 LittleEndian::write_u16(&mut incompatible[2..4], POOL_META_VERSION + 1);
21349
21350 let err = select_pool_meta_replica(vec![decode_pool_meta_replica(valid), decode_pool_meta_replica(incompatible)])
21351 .expect_err("an incompatible replica must block fallback and repair writes");
21352
21353 assert!(err.to_string().contains("pool 1 is incompatible"));
21354 assert!(err.to_string().contains("without overwriting it"));
21355 }
21356
21357 #[test]
21358 fn pool_meta_replica_selection_prefers_valid_canonical_first_copy() {
21359 let selection = select_pool_meta_replica(vec![
21360 decode_pool_meta_replica(pool_meta_replica_test_data("pool-canonical")),
21361 decode_pool_meta_replica(pool_meta_replica_test_data("pool-stale")),
21362 ])
21363 .expect("pool zero is the durable commit record when a later replica is stale");
21364
21365 assert_eq!(selection.meta.pools[0].cmd_line, "pool-canonical");
21366 assert!(selection.replica_state.needs_repair);
21367 assert!(selection.replica_state.repair_write_safe);
21368 }
21369
21370 #[test]
21371 fn pool_meta_replica_selection_rejects_divergent_backups_without_canonical() {
21372 let err = select_pool_meta_replica(vec![
21373 PoolMetaReplica::Corrupt("canonical unavailable".to_string()),
21374 decode_pool_meta_replica(pool_meta_replica_test_data("pool-old")),
21375 decode_pool_meta_replica(pool_meta_replica_test_data("pool-new")),
21376 ])
21377 .expect_err("divergent backups have no safe ordering when the canonical copy is unavailable");
21378
21379 assert!(err.to_string().contains("valid replicas in pools 1 and 2 diverge"));
21380 }
21381
21382 #[test]
21383 fn pool_meta_replica_selection_normalizes_equivalent_legacy_copy() {
21384 let selection = select_pool_meta_replica(vec![
21385 decode_pool_meta_replica(pool_meta_legacy_v1_replica_test_data("pool-0")),
21386 decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
21387 ])
21388 .expect("equivalent legacy and current encodings should be compatible");
21389
21390 assert!(selection.replica_state.needs_repair);
21391 assert!(selection.replica_state.repair_write_safe);
21392 assert_eq!(selection.meta.pools[0].cmd_line, "pool-0");
21393 }
21394
21395 #[test]
21396 fn pool_meta_v1_replica_migrates_to_v2_canonical_form() {
21397 let mut source = pool_meta_replica_test_meta("pool-0");
21398 source.pools[0].decommission = Some(PoolDecommissionInfo {
21399 start_time: Some(OffsetDateTime::UNIX_EPOCH),
21400 queued_buckets: vec!["bucket-a".to_string()],
21401 ..Default::default()
21402 });
21403 let data = pool_meta_v1_replica_test_data(&source);
21404 let PoolMetaReplica::Valid {
21405 raw, canonical, meta, ..
21406 } = decode_pool_meta_replica(data)
21407 else {
21408 panic!("v1 pool metadata should remain readable");
21409 };
21410
21411 assert_eq!(LittleEndian::read_u16(&raw[2..4]), POOL_META_V1_VERSION);
21412 assert_eq!(LittleEndian::read_u16(&canonical[2..4]), POOL_META_VERSION);
21413 assert_eq!(meta.version, POOL_META_V1_VERSION);
21414 let info = meta.pools[0]
21415 .decommission
21416 .as_ref()
21417 .expect("v1 decommission state should migrate");
21418 assert_eq!(info.queued_buckets, vec!["bucket-a".to_string()]);
21419 assert!(info.unresolved_entries.is_empty());
21420 }
21421
21422 #[test]
21423 fn pool_meta_legacy_v1_replica_migrates_to_v2_canonical_form() {
21424 let data = pool_meta_legacy_v1_replica_test_data("pool-0");
21425 let PoolMetaReplica::Valid {
21426 raw, canonical, meta, ..
21427 } = decode_pool_meta_replica(data)
21428 else {
21429 panic!("legacy v1 pool metadata should remain readable");
21430 };
21431
21432 assert_eq!(LittleEndian::read_u16(&raw[2..4]), POOL_META_V1_VERSION);
21433 assert_eq!(LittleEndian::read_u16(&canonical[2..4]), POOL_META_VERSION);
21434 assert_eq!(meta.version, POOL_META_V1_VERSION);
21435 assert_eq!(meta.pools[0].cmd_line, "pool-0");
21436 }
21437
21438 #[test]
21439 fn pool_meta_replica_selection_rejects_v1_v2_divergence() {
21440 let err = select_pool_meta_replica(vec![
21441 decode_pool_meta_replica(pool_meta_persisted_v1_replica_test_data("pool-old")),
21442 decode_pool_meta_replica(pool_meta_replica_test_data("pool-new")),
21443 ])
21444 .expect_err("different v1 and v2 snapshots must remain fail-closed");
21445
21446 assert!(err.to_string().contains("valid replicas in pools 0 and 1 diverge"));
21447 }
21448
21449 #[test]
21450 fn pool_meta_replica_rejects_header_payload_version_mismatch() {
21451 let mut v1_with_v2_header = pool_meta_persisted_v1_replica_test_data("pool-0");
21452 LittleEndian::write_u16(&mut v1_with_v2_header[2..4], POOL_META_VERSION);
21453 assert!(matches!(decode_pool_meta_replica(v1_with_v2_header), PoolMetaReplica::Corrupt(_)));
21454
21455 let mut v2_with_v1_header = pool_meta_replica_test_data("pool-0");
21456 LittleEndian::write_u16(&mut v2_with_v1_header[2..4], POOL_META_V1_VERSION);
21457 assert!(matches!(
21458 decode_pool_meta_replica(v2_with_v1_header),
21459 PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
21460 ));
21461 }
21462
21463 #[test]
21464 fn pool_meta_v2_header_guards_v1_readers_from_tuple_extension() {
21465 let mut meta = pool_meta_replica_test_meta("pool-0");
21466 meta.pools[0].decommission = Some(PoolDecommissionInfo::default());
21467 let data = meta.encode_config_data().expect("v2 pool metadata should encode");
21468
21469 assert_eq!(LittleEndian::read_u16(&data[2..4]), POOL_META_VERSION);
21470 assert_ne!(POOL_META_VERSION, POOL_META_V1_VERSION);
21471 assert!(rmp_serde::from_slice::<PersistedPoolMetaV1>(&data[4..]).is_err());
21472 }
21473
21474 #[test]
21475 fn pool_meta_writer_stays_v1_until_v2_is_fleet_confirmed() {
21476 let mut meta = pool_meta_replica_test_meta("pool-0");
21477 meta.version = POOL_META_V1_VERSION;
21478
21479 let v1 = meta
21480 .encode_config_data_for_v2_gate(false)
21481 .expect("the default writer should preserve v1");
21482 let v2 = meta
21483 .encode_config_data_for_v2_gate(true)
21484 .expect("the confirmed writer should emit v2");
21485
21486 assert_eq!(LittleEndian::read_u16(&v1[2..4]), POOL_META_V1_VERSION);
21487 assert_eq!(LittleEndian::read_u16(&v2[2..4]), POOL_META_VERSION);
21488 assert!(rmp_serde::from_slice::<PersistedPoolMetaV1>(&v1[4..]).is_ok());
21489 }
21490
21491 #[test]
21492 fn pool_meta_v2_writer_requires_both_gates() {
21493 assert!(!pool_meta_v2_writer_enabled_for(false, false));
21494 assert!(!pool_meta_v2_writer_enabled_for(true, false));
21495 assert!(!pool_meta_v2_writer_enabled_for(false, true));
21496 assert!(pool_meta_v2_writer_enabled_for(true, true));
21497 }
21498
21499 #[test]
21500 fn pool_meta_v3_writer_requires_both_gates() {
21501 assert!(!pool_meta_v3_writer_enabled_for(false, false));
21502 assert!(!pool_meta_v3_writer_enabled_for(true, false));
21503 assert!(!pool_meta_v3_writer_enabled_for(false, true));
21504 assert!(pool_meta_v3_writer_enabled_for(true, true));
21505 }
21506
21507 #[test]
21508 fn decommission_ledger_persistence_requires_an_observed_or_confirmed_format() {
21509 for (version, v2_enabled, v3_enabled, expected) in [
21510 (POOL_META_V1_VERSION, false, false, false),
21511 (POOL_META_V1_VERSION, true, false, true),
21512 (POOL_META_V1_VERSION, false, true, true),
21513 (POOL_META_VERSION, false, false, true),
21514 (super::POOL_META_GENERATION_VERSION, false, false, true),
21515 ] {
21516 let result = super::ensure_decommission_ledger_persistence_supported_for(version, v2_enabled, v3_enabled);
21517 assert_eq!(
21518 result.is_ok(),
21519 expected,
21520 "unexpected capability result for pool metadata version {version}"
21521 );
21522 }
21523
21524 let half_confirmed_v2 = pool_meta_v2_writer_enabled_for(true, false);
21525 let half_confirmed_v3 = pool_meta_v3_writer_enabled_for(false, true);
21526 let err = super::ensure_decommission_ledger_persistence_supported_for(
21527 POOL_META_V1_VERSION,
21528 half_confirmed_v2,
21529 half_confirmed_v3,
21530 )
21531 .expect_err("half-enabled rollout gates must not admit decommission");
21532 assert!(matches!(err, Error::InvalidArgument(..)));
21533 assert!(err.to_string().contains("durable unresolved-entry recovery"));
21534 assert!(err.to_string().contains(rustfs_config::ENV_POOL_META_V3_WRITE));
21535 assert!(err.to_string().contains(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED));
21536 }
21537
21538 #[test]
21539 fn decommission_capacity_accepts_v3_only_writer_rollout() {
21540 assert!(decommission_capacity_writer_supported_for(
21541 POOL_META_V1_VERSION,
21542 false,
21543 pool_meta_v3_writer_enabled_for(true, true),
21544 ));
21545 assert!(!decommission_capacity_writer_supported_for(
21546 POOL_META_V1_VERSION,
21547 false,
21548 pool_meta_v3_writer_enabled_for(true, false),
21549 ));
21550 }
21551
21552 #[test]
21553 fn pool_meta_stale_write_rejection_metric_is_countable() {
21554 let recorder = metrics_util::debugging::DebuggingRecorder::new();
21555 let snapshotter = recorder.snapshotter();
21556 metrics::with_local_recorder(&recorder, || {
21557 record_pool_meta_stale_write_rejection("prepare_cas");
21558 record_pool_meta_stale_write_rejection("prepare_cas");
21559 });
21560
21561 let total = snapshotter
21562 .snapshot()
21563 .into_vec()
21564 .into_iter()
21565 .filter(|(composite, _, _, _)| composite.key().name() == METRIC_POOL_META_STALE_WRITE_REJECTIONS_TOTAL)
21566 .filter_map(|(_, _, _, value)| match value {
21567 metrics_util::debugging::DebugValue::Counter(value) => Some(value),
21568 _ => None,
21569 })
21570 .sum::<u64>();
21571 assert_eq!(total, 2);
21572 }
21573
21574 #[test]
21575 fn pool_meta_block_transition_is_counted_once_across_both_latches() {
21576 let recorder = metrics_util::debugging::DebuggingRecorder::new();
21577 let snapshotter = recorder.snapshotter();
21578 metrics::with_local_recorder(&recorder, || {
21579 let mut state = PoolMetaWriteState::default();
21580 let mut arm = state.arm_transaction();
21581 arm.phase = Some("commit_cas");
21582 drop(arm);
21583 let first = state
21584 .ensure_write_safe("first block")
21585 .unwrap_err()
21586 .pool_metadata_failure()
21587 .unwrap()
21588 .since;
21589 state.block_writes_after_fence_loss();
21590 state.block_writes();
21591 assert_eq!(
21592 state
21593 .ensure_write_safe("later block")
21594 .unwrap_err()
21595 .pool_metadata_failure()
21596 .unwrap()
21597 .since,
21598 first
21599 );
21600 });
21601 let total = snapshotter
21602 .snapshot()
21603 .into_vec()
21604 .into_iter()
21605 .filter(|(key, _, _, _)| key.key().name() == "rustfs_pool_metadata_blocks_total")
21606 .filter_map(|(_, _, _, value)| match value {
21607 metrics_util::debugging::DebugValue::Counter(count) => Some(count),
21608 _ => None,
21609 })
21610 .sum::<u64>();
21611 assert_eq!(total, 1);
21612 }
21613
21614 fn pool_meta_v3_test_revision(cluster_id: uuid::Uuid, generation: u64, transaction_id: uuid::Uuid) -> PoolMetaRevision {
21615 PoolMetaRevision {
21616 version: POOL_META_GENERATION_VERSION,
21617 cluster_id: Some(cluster_id),
21618 epoch: POOL_META_INITIAL_EPOCH,
21619 generation,
21620 transaction_id: Some(transaction_id),
21621 }
21622 }
21623
21624 fn pool_meta_legacy_candidate(meta: PoolMeta) -> PoolMetaCommittedCandidate {
21625 PoolMetaCommittedCandidate {
21626 canonical: meta
21627 .encode_config_data_for_v2_gate(true)
21628 .expect("legacy pool metadata should encode"),
21629 revision: PoolMetaRevision::legacy(meta.version),
21630 meta,
21631 }
21632 }
21633
21634 #[test]
21635 fn pool_meta_v3_pending_prepare_recovers_previous_snapshot_after_restart() {
21636 let previous = pool_meta_replica_test_meta("pool-before");
21637 let mut next = pool_meta_replica_test_meta("pool-after");
21638 next.version = POOL_META_GENERATION_VERSION;
21639 let revision = pool_meta_v3_test_revision(uuid::Uuid::new_v4(), 1, uuid::Uuid::new_v4());
21640 let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&pool_meta_legacy_candidate(previous.clone())))
21641 .expect("pending generation should encode");
21642
21643 let selected = select_pool_meta_replica(vec![
21644 decode_pool_meta_replica(pending),
21645 decode_pool_meta_replica(
21646 previous
21647 .encode_config_data_for_v2_gate(true)
21648 .expect("previous snapshot should encode"),
21649 ),
21650 ])
21651 .expect("a prepare-only transaction should expose its committed predecessor");
21652
21653 assert_eq!(selected.meta.pools[0].cmd_line, "pool-before");
21654 assert_eq!(selected.revision.version, POOL_META_VERSION);
21655 assert!(selected.replica_state.needs_repair);
21656 }
21657
21658 #[test]
21659 fn pool_meta_v3_partial_commit_selects_new_generation_after_restart() {
21660 let previous = pool_meta_replica_test_meta("pool-before");
21661 let mut next = pool_meta_replica_test_meta("pool-after");
21662 next.version = POOL_META_GENERATION_VERSION;
21663 let revision = pool_meta_v3_test_revision(uuid::Uuid::new_v4(), 1, uuid::Uuid::new_v4());
21664 let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&pool_meta_legacy_candidate(previous)))
21665 .expect("pending generation should encode");
21666 let committed = encode_pool_meta_v3_envelope(&next, revision, true, None).expect("committed generation should encode");
21667
21668 let selected = select_pool_meta_replica(vec![decode_pool_meta_replica(committed), decode_pool_meta_replica(pending)])
21669 .expect("one committed replica should make the cross-pool transaction durable");
21670
21671 assert_eq!(selected.meta.pools[0].cmd_line, "pool-after");
21672 assert_eq!(selected.revision, revision);
21673 assert!(selected.replica_state.needs_repair);
21674 }
21675
21676 #[test]
21677 fn pool_meta_v3_uses_valid_committed_backup_but_rejects_same_generation_fork() {
21678 let cluster_id = uuid::Uuid::new_v4();
21679 let mut next = pool_meta_replica_test_meta("pool-after");
21680 next.version = POOL_META_GENERATION_VERSION;
21681 let revision = pool_meta_v3_test_revision(cluster_id, 7, uuid::Uuid::new_v4());
21682 let committed = encode_pool_meta_v3_envelope(&next, revision, true, None).expect("committed generation should encode");
21683 let selected = select_pool_meta_replica(vec![
21684 PoolMetaReplica::Corrupt("truncated".to_string()),
21685 decode_pool_meta_replica(committed.clone()),
21686 ])
21687 .expect("a corrupt first copy should fall back to a valid committed generation");
21688 assert_eq!(selected.revision, revision);
21689 assert!(selected.replica_state.needs_repair);
21690
21691 let fork_revision = pool_meta_v3_test_revision(cluster_id, 7, uuid::Uuid::new_v4());
21692 let fork = encode_pool_meta_v3_envelope(&next, fork_revision, true, None).expect("fork generation should encode");
21693 let err = select_pool_meta_replica(vec![decode_pool_meta_replica(committed), decode_pool_meta_replica(fork)])
21694 .expect_err("same-generation transactions must never be selected by pool order");
21695 assert!(err.to_string().contains("committed generation 7 diverges"));
21696 }
21697
21698 #[test]
21699 fn pool_meta_v3_migration_keeps_legacy_committed_until_commit_record_exists() {
21700 let previous = pool_meta_replica_test_meta("pool-before");
21701 let mut next = pool_meta_replica_test_meta("pool-after");
21702 next.version = POOL_META_GENERATION_VERSION;
21703 let revision = pool_meta_v3_test_revision(uuid::Uuid::new_v4(), 1, uuid::Uuid::new_v4());
21704 let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&pool_meta_legacy_candidate(previous.clone())))
21705 .expect("pending migration should encode");
21706 let legacy = previous
21707 .encode_config_data_for_v2_gate(true)
21708 .expect("legacy snapshot should encode");
21709
21710 let prepared = select_pool_meta_replica(vec![decode_pool_meta_replica(pending), decode_pool_meta_replica(legacy)])
21711 .expect("prepared migration should retain the legacy commit");
21712 assert_eq!(prepared.meta.version, POOL_META_VERSION);
21713 assert_eq!(prepared.meta.pools[0].cmd_line, "pool-before");
21714
21715 let committed = encode_pool_meta_v3_envelope(&next, revision, true, None).expect("committed migration should encode");
21716 let migrated = select_pool_meta_replica(vec![decode_pool_meta_replica(committed)])
21717 .expect("committed migration should establish the V3 floor");
21718 assert_eq!(migrated.meta.version, POOL_META_GENERATION_VERSION);
21719 assert_eq!(migrated.meta.pools[0].cmd_line, "pool-after");
21720 }
21721
21722 #[test]
21723 fn pool_meta_v3_unknown_fields_remain_incompatible_not_silently_ignored() {
21724 #[derive(Serialize)]
21725 struct FuturePoolMetaV3 {
21726 version: u16,
21727 cluster_id: String,
21728 epoch: u64,
21729 generation: u64,
21730 transaction_id: String,
21731 committed: bool,
21732 pools: Vec<PersistedPoolStatus>,
21733 previous: Option<PersistedPoolMetaV3Previous>,
21734 future_guard: u64,
21735 }
21736
21737 let mut data = Vec::new();
21738 data.write_u16::<LittleEndian>(POOL_META_FORMAT)
21739 .expect("pool metadata format should encode");
21740 data.write_u16::<LittleEndian>(POOL_META_GENERATION_VERSION)
21741 .expect("pool metadata version should encode");
21742 FuturePoolMetaV3 {
21743 version: POOL_META_GENERATION_VERSION,
21744 cluster_id: uuid::Uuid::new_v4().to_string(),
21745 epoch: POOL_META_INITIAL_EPOCH,
21746 generation: 1,
21747 transaction_id: uuid::Uuid::new_v4().to_string(),
21748 committed: true,
21749 pools: Vec::new(),
21750 previous: None,
21751 future_guard: 1,
21752 }
21753 .serialize(&mut Serializer::new(&mut data))
21754 .expect("future V3 payload should encode");
21755
21756 assert!(matches!(decode_pool_meta_replica(data), PoolMetaReplica::Incompatible(_)));
21757 }
21758
21759 #[test]
21760 fn pool_meta_identity_classifies_corrupt_incompatible_and_divergent_replicas() {
21761 let cluster_id = uuid::Uuid::new_v4();
21762 let identity = PersistedPoolMetaIdentity {
21763 version: POOL_META_IDENTITY_VERSION,
21764 cluster_id,
21765 epoch: POOL_META_INITIAL_EPOCH,
21766 initialized: true,
21767 fresh_bootstrap_nonce: None,
21768 };
21769 let selected = select_pool_meta_identity(
21770 vec![
21771 PoolMetaIdentityRead {
21772 replica: PoolMetaIdentityReplica::Corrupt("truncated".to_string()),
21773 cas: PoolMetaCasToken::Existing("corrupt".to_string()),
21774 },
21775 PoolMetaIdentityRead {
21776 replica: PoolMetaIdentityReplica::Valid(identity),
21777 cas: PoolMetaCasToken::Existing("valid".to_string()),
21778 },
21779 ],
21780 cluster_id,
21781 )
21782 .expect("a verified identity backup should survive a corrupt first replica");
21783 assert_eq!(selected.identity, Some(identity));
21784 assert!(selected.needs_repair);
21785
21786 let incompatible = select_pool_meta_identity(
21787 vec![PoolMetaIdentityRead {
21788 replica: PoolMetaIdentityReplica::Incompatible("future version".to_string()),
21789 cas: PoolMetaCasToken::Existing("future".to_string()),
21790 }],
21791 cluster_id,
21792 )
21793 .expect_err("an incompatible identity must fail closed");
21794 assert!(incompatible.to_string().contains("incompatible"));
21795
21796 let divergent = select_pool_meta_identity(
21797 vec![
21798 PoolMetaIdentityRead {
21799 replica: PoolMetaIdentityReplica::Valid(identity),
21800 cas: PoolMetaCasToken::Existing("epoch-1".to_string()),
21801 },
21802 PoolMetaIdentityRead {
21803 replica: PoolMetaIdentityReplica::Valid(PersistedPoolMetaIdentity {
21804 epoch: POOL_META_INITIAL_EPOCH + 1,
21805 ..identity
21806 }),
21807 cas: PoolMetaCasToken::Existing("epoch-2".to_string()),
21808 },
21809 ],
21810 cluster_id,
21811 )
21812 .expect_err("identity epoch divergence must require recovery");
21813 assert!(divergent.to_string().contains("recovery required"));
21814 }
21815
21816 #[test]
21817 fn pool_meta_v2_floor_is_sticky_after_observation() {
21818 let meta = pool_meta_replica_test_meta("pool-0");
21819 let data = meta
21820 .encode_config_data_for_v2_gate(false)
21821 .expect("an observed v2 snapshot must not be downgraded");
21822
21823 assert_eq!(LittleEndian::read_u16(&data[2..4]), POOL_META_VERSION);
21824 }
21825
21826 #[test]
21827 fn pool_meta_v1_writer_rejects_unresolved_ledger() {
21828 let mut meta = pool_meta_replica_test_meta("pool-0");
21829 meta.version = POOL_META_V1_VERSION;
21830 meta.pools[0].decommission = Some(PoolDecommissionInfo {
21831 start_time: Some(OffsetDateTime::UNIX_EPOCH),
21832 unresolved_entries: vec![DecommissionUnresolvedEntry {
21833 bucket: "bucket-a".to_string(),
21834 object: "object-a".to_string(),
21835 pool_index: 0,
21836 set_index: 0,
21837 source_generation: OffsetDateTime::UNIX_EPOCH,
21838 candidate_count: 1,
21839 disk_error_count: 1,
21840 observed_at: OffsetDateTime::UNIX_EPOCH,
21841 reason: "metadata_resolution_failed".to_string(),
21842 }],
21843 ..Default::default()
21844 });
21845
21846 let err = meta
21847 .encode_config_data_for_v2_gate(false)
21848 .expect_err("v1 must not drop the unresolved ledger");
21849 assert!(err.to_string().contains("pool metadata V2 is required"));
21850 }
21851
21852 #[test]
21853 fn pool_meta_v1_writer_rejects_capacity_reservations() {
21854 let mut meta = pool_meta_replica_test_meta("pool-0");
21855 meta.version = POOL_META_V1_VERSION;
21856 let reservation = build_decommission_capacity_reservation(
21857 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 1, parity: 0 }, 0, 10, 10),
21858 DecommissionErasureLayout { data: 1, parity: 0 },
21859 uuid::Uuid::new_v4(),
21860 1,
21861 OffsetDateTime::UNIX_EPOCH,
21862 )
21863 .expect("test reservation should be valid");
21864 meta.pools[0].decommission = Some(PoolDecommissionInfo {
21865 start_time: Some(OffsetDateTime::UNIX_EPOCH),
21866 capacity_reservation: Some(reservation),
21867 ..Default::default()
21868 });
21869
21870 let err = meta
21871 .encode_config_data_for_v2_gate(false)
21872 .expect_err("v1 must not drop a distributed capacity reservation");
21873 assert!(
21874 err.to_string()
21875 .contains("pool metadata V2 is required to persist decommission capacity reservations")
21876 );
21877 }
21878
21879 #[test]
21880 fn pool_meta_replica_selection_preserves_observed_v2_floor() {
21881 let selection = select_pool_meta_replica(vec![
21882 decode_pool_meta_replica(pool_meta_persisted_v1_replica_test_data("pool-0")),
21883 decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
21884 ])
21885 .expect("equivalent v1 and v2 replicas should converge");
21886
21887 assert_eq!(selection.meta.version, POOL_META_VERSION);
21888 let data = selection
21889 .meta
21890 .encode_config_data_for_v2_gate(false)
21891 .expect("the selected v2 floor must remain writable");
21892 assert_eq!(LittleEndian::read_u16(&data[2..4]), POOL_META_VERSION);
21893 }
21894
21895 #[test]
21896 fn pool_meta_replica_selection_distinguishes_absent_from_unrecoverable() {
21897 assert!(matches!(decode_pool_meta_replica(Vec::new()), PoolMetaReplica::Corrupt(_)));
21898
21899 let empty = select_pool_meta_replica(vec![PoolMetaReplica::Missing, PoolMetaReplica::Missing])
21900 .expect("all missing replicas should preserve new-deployment behavior");
21901 assert!(empty.meta.pools.is_empty());
21902 assert!(!empty.replica_state.needs_repair);
21903 assert!(empty.replica_state.repair_write_safe);
21904
21905 let err = select_pool_meta_replica(vec![
21906 PoolMetaReplica::Missing,
21907 PoolMetaReplica::Unreadable(Arc::new(Error::other("read quorum unavailable"))),
21908 ])
21909 .expect_err("an unreadable replica must not be treated as a new deployment");
21910 assert!(err.to_string().contains("no valid committed replica is available"));
21911 assert!(err.to_string().contains("pool 1 is unreadable"));
21912 }
21913
21914 #[test]
21915 fn pool_meta_replica_selection_blocks_repair_for_unreadable_copy() {
21916 let selection = select_pool_meta_replica(vec![
21917 decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
21918 PoolMetaReplica::Unreadable(Arc::new(Error::other("read quorum unavailable"))),
21919 ])
21920 .expect("a validated replica should remain usable while another copy is unreadable");
21921
21922 assert!(selection.replica_state.needs_repair);
21923 assert!(!selection.replica_state.repair_write_safe);
21924 }
21925
21926 #[test]
21927 fn pool_meta_write_state_remains_blocked_after_unreadable_replica() {
21928 let mut write_state = PoolMetaWriteState::default();
21929 write_state.observe_replicas(PoolMetaReplicaState {
21930 needs_repair: true,
21931 repair_write_safe: false,
21932 });
21933 write_state.observe_replicas(PoolMetaReplicaState {
21934 needs_repair: false,
21935 repair_write_safe: true,
21936 });
21937
21938 let err = write_state
21939 .ensure_write_safe("pool metadata save failed")
21940 .expect_err("a later clean read must not clear the startup write block");
21941 assert!(
21942 err.to_string()
21943 .contains("restart after all replicas are readable and consistent")
21944 );
21945 }
21946
21947 #[test]
21948 fn pool_meta_read_probe_does_not_latch_writer_state() {
21949 let write_state = PoolMetaWriteState::default();
21950 select_pool_meta_replicas_for_read_probe(
21951 &write_state,
21952 vec![PoolMetaReplica::Unreadable(Arc::new(Error::Timeout))],
21953 "capacity probe",
21954 )
21955 .expect_err("an unreadable probe replica must fail the current admission");
21956
21957 assert!(
21958 write_state.ensure_write_safe("ordinary object write").is_ok(),
21959 "a read-only capacity probe must not permanently latch the pool metadata writer"
21960 );
21961 }
21962
21963 #[test]
21964 fn pool_meta_preflight_drop_does_not_arm_but_started_drop_retains_first_failure() {
21965 let state = PoolMetaWriteState::default();
21966 drop(state.arm_transaction());
21967 state.ensure_write_safe("preflight retry").unwrap();
21968 let mut first = state.arm_transaction();
21969 first.phase = Some("identity_cas");
21970 let err = first.failure(Error::Timeout);
21971 drop(first);
21972 let context = state.ensure_write_safe("retry").unwrap_err();
21973 let first_failure = context.pool_metadata_failure().unwrap();
21974 assert_eq!(first_failure.kind, crate::error::PoolMetadataFailure::TransactionUnknown);
21975 assert_eq!(first_failure.phase, "identity_cas");
21976 assert!(matches!(first_failure.source.as_deref(), Some(Error::Timeout)));
21977 assert!(matches!(err.pool_metadata_failure().unwrap().source.as_deref(), Some(Error::Timeout)));
21978 let mut late = state.arm_transaction();
21979 late.phase = Some("commit_cas");
21980 drop(late);
21981 let later = state.ensure_write_safe("retry").unwrap_err();
21982 assert_eq!(later.pool_metadata_failure().unwrap().since, first_failure.since);
21983 assert_eq!(later.pool_metadata_failure().unwrap().phase, "identity_cas");
21984 assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
21985 }
21986
21987 #[tokio::test]
21988 #[serial_test::serial]
21989 async fn pool_meta_writer_retries_after_identity_read_failure_without_restart() {
21990 let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
21991 let mut disks = Vec::new();
21992 for set in &store.pools[1].disk_set {
21993 let mut guard = set.disks.write().await;
21994 let disk_count = guard.len();
21995 let original = std::mem::replace(&mut *guard, vec![None; disk_count]);
21996 disks.push((set.clone(), original));
21997 }
21998 let mut state = store.pool_meta_save_gate.lock().await;
21999 let err = store
22000 .acquire_pool_meta_write_guard(&mut state, "writer preflight")
22001 .await
22002 .unwrap_err();
22003 assert_eq!(
22004 err.pool_metadata_failure().unwrap().kind,
22005 crate::error::PoolMetadataFailure::ReadUnavailable
22006 );
22007 state.ensure_write_safe("retry").unwrap();
22008 for (set, original) in disks {
22009 *set.disks.write().await = original;
22010 }
22011 let (_guard, _) = store.acquire_pool_meta_write_guard(&mut state, "writer retry").await.unwrap();
22012 }
22013
22014 #[tokio::test]
22015 #[serial_test::serial]
22016 async fn pool_meta_runtime_recovery_publishes_durable_state_and_drains_old_workers() {
22017 let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
22018 let before = store.pool_meta.read().await.clone();
22019 let mut requested = before.clone();
22020 requested.pools[0].last_update += Duration::seconds(1);
22021 requested.version = POOL_META_GENERATION_VERSION;
22022 {
22023 let mut state = store.pool_meta_save_gate.lock().await;
22024 let (fence, _) = store.acquire_pool_meta_write_guard(&mut state, "test save").await.unwrap();
22025 let outcome = requested
22026 .save_no_lock_armed(store.pools.clone(), &mut state, fence.lock_lost_signal(), &[0])
22027 .await
22028 .unwrap();
22029 assert_eq!(outcome.committed.pools[0].last_update, requested.pools[0].last_update);
22030 drop(outcome);
22031 }
22032 assert!(!store.pool_meta_writes_ready().await);
22033 let worker = DecommissionCanceler::new(CancellationToken::new());
22034 store.decommission_cancelers.write().await[0] = Some(worker.clone());
22035 assert!(store.recover_pool_meta_transaction().await.is_err());
22036 assert!(worker.is_cancelled());
22037 assert!(!store.pool_meta_writes_ready().await);
22038 assert_eq!(store.pool_meta.read().await.pools[0].last_update, before.pools[0].last_update);
22039 worker.release();
22040 store.pool_meta.write().await.pools[0].last_update = requested.pools[0].last_update + Duration::seconds(10);
22041 let outstanding = store.pool_meta_save_gate.lock().await.arm_transaction();
22043 assert!(!store.recover_pool_meta_transaction().await.unwrap());
22044 drop(outstanding);
22045 assert!(store.recover_pool_meta_transaction().await.unwrap());
22046 assert!(store.pool_meta_writes_ready().await);
22047 let recovered = store.pool_meta_write_gate_status().await;
22048 assert!(recovered.writes_ready);
22049 assert!(!recovered.check_timed_out);
22050 assert!(!recovered.write_blocked);
22051 assert!(!recovered.transaction_aborted);
22052 assert_eq!((recovered.reason, recovered.phase, recovered.since_unix_secs), (None, None, None));
22053 assert_eq!(store.pool_meta.read().await.pools[0].last_update, requested.pools[0].last_update);
22054 assert!(!store.recover_pool_meta_transaction().await.unwrap());
22055 store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
22056 assert!(!store.recover_pool_meta_transaction().await.unwrap());
22057 assert!(!store.pool_meta_writes_ready().await);
22058 }
22059
22060 #[tokio::test]
22061 #[serial_test::serial]
22062 async fn pool_meta_readiness_is_bounded_while_save_gate_is_held() {
22063 let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
22064 let _state = store.pool_meta_save_gate.lock().await;
22065 let result = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_status())
22066 .await
22067 .unwrap();
22068 assert!(matches!(result, Err(Error::Timeout)));
22069 let snapshot = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_gate_status())
22070 .await
22071 .unwrap();
22072 assert!(!snapshot.writes_ready);
22073 assert!(snapshot.check_timed_out);
22074 assert!(!snapshot.write_blocked);
22075 assert!(!snapshot.transaction_aborted);
22076 assert_eq!((snapshot.reason, snapshot.phase, snapshot.since_unix_secs), (None, None, None));
22077 }
22078
22079 #[tokio::test]
22080 #[serial_test::serial]
22081 async fn pool_meta_cancelled_recovery_and_new_integrity_block_never_clear_original_gate() {
22082 let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
22083 let since = OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap();
22084 {
22085 let state = store.pool_meta_save_gate.lock().await;
22086 let mut arm = state.arm_transaction();
22087 arm.phase = Some("publication");
22088 drop(arm);
22089 state.transaction_failure.lock().unwrap().as_mut().unwrap().since = since;
22091 *state.block_started_at.lock().unwrap() = Some(since);
22092 }
22093 let original = store.pool_meta_write_gate_status().await;
22094 assert!(!original.writes_ready);
22095 assert!(!original.check_timed_out);
22096 assert!(!original.write_blocked);
22097 assert!(original.transaction_aborted);
22098 assert_eq!(original.reason, Some("transaction_unknown"));
22099 assert_eq!(original.phase, Some("publication"));
22100 assert_eq!(original.since_unix_secs, Some(since.unix_timestamp()));
22101 assert_eq!(store.pool_meta_write_gate_status().await, original);
22102 let publication_guard = store.pool_meta.write().await;
22103 let recovery = tokio::spawn({
22104 let store = store.clone();
22105 async move { store.recover_pool_meta_transaction().await }
22106 });
22107 tokio::time::timeout(std::time::Duration::from_secs(10), async {
22108 loop {
22109 if store.pool_meta_save_gate.try_lock().is_err() {
22110 break;
22111 }
22112 tokio::task::yield_now().await;
22113 }
22114 })
22115 .await
22116 .unwrap();
22117 recovery.abort();
22118 assert!(recovery.await.unwrap_err().is_cancelled());
22119 drop(publication_guard);
22120 assert!(!store.pool_meta_writes_ready().await);
22121 assert_eq!(store.pool_meta_write_gate_status().await, original);
22122 let start_guard = store.start_gate.lock().await;
22123 let recovery = tokio::spawn({
22124 let store = store.clone();
22125 async move { store.recover_pool_meta_transaction().await }
22126 });
22127 store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
22128 drop(start_guard);
22129 assert!(!recovery.await.unwrap().unwrap());
22130 assert_eq!(
22131 store
22132 .pool_meta_write_status()
22133 .await
22134 .unwrap_err()
22135 .pool_metadata_failure()
22136 .unwrap()
22137 .kind,
22138 crate::error::PoolMetadataFailure::FenceLost
22139 );
22140 let fenced = store.pool_meta_write_gate_status().await;
22141 assert!(!fenced.writes_ready);
22142 assert!(fenced.write_blocked);
22143 assert_eq!(fenced.reason, Some("fence_lost"));
22144 assert_eq!(fenced.phase, Some("format_heal"));
22145 assert_eq!(fenced.since_unix_secs, original.since_unix_secs);
22146 }
22147
22148 #[test]
22149 fn pool_meta_read_probe_rejects_missing_runtime_metadata_without_latching() {
22150 let write_state = PoolMetaWriteState {
22151 expected_cluster_id: Some(uuid::Uuid::new_v4()),
22152 identity_initialized: Some(true),
22153 ..Default::default()
22154 };
22155 select_pool_meta_replicas_for_read_probe(&write_state, vec![PoolMetaReplica::Missing], "capacity probe")
22156 .expect_err("runtime metadata disappearance must reject the current probe");
22157 write_state
22158 .ensure_write_safe("ordinary object write")
22159 .expect("a missing-metadata probe must not permanently latch the writer");
22160 }
22161
22162 #[tokio::test]
22163 #[serial_test::serial]
22164 async fn pool_meta_read_guard_does_not_latch_after_unreadable_replica() {
22165 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
22166 let mut saved_disks = Vec::new();
22167 for set in &store.pools[1].disk_set {
22168 let mut disks = set.disks.write().await;
22169 let original = std::mem::take(&mut *disks);
22170 let disk_count = original.len();
22171 saved_disks.push((set.clone(), original));
22172 *disks = vec![None; disk_count];
22173 }
22174
22175 let write_state = store.pool_meta_save_gate.lock().await;
22176 store
22177 .acquire_pool_meta_read_guard(&write_state, "capacity probe")
22178 .await
22179 .expect_err("an unreadable metadata replica must reject this probe");
22180 write_state
22181 .ensure_write_safe("ordinary object write")
22182 .expect("a failed read-only probe must remain retryable");
22183
22184 for (set, disks) in saved_disks {
22185 *set.disks.write().await = disks;
22186 }
22187 store
22188 .acquire_pool_meta_read_guard(&write_state, "capacity probe retry")
22189 .await
22190 .expect("a read-only probe must succeed after the replica recovers");
22191 }
22192
22193 #[test]
22194 fn pool_meta_writer_read_failure_is_retryable_when_all_replicas_are_unreadable() {
22195 let replicas = vec![
22196 PoolMetaReplica::Unreadable(Arc::new(Error::other("pool 0 read quorum unavailable"))),
22197 PoolMetaReplica::Unreadable(Arc::new(Error::other("pool 1 read quorum unavailable"))),
22198 ];
22199 let mut write_state = PoolMetaWriteState::default();
22200
22201 let err = select_pool_meta_replicas_observing(&mut write_state, replicas)
22202 .expect_err("all unreadable replicas must fail selection");
22203 assert_eq!(
22204 err.pool_metadata_failure().unwrap().kind,
22205 crate::error::PoolMetadataFailure::ReadUnavailable
22206 );
22207 write_state
22208 .ensure_write_safe("pool metadata save failed")
22209 .expect("a read-only preflight has no unknown writes and must remain retryable");
22210 }
22211
22212 #[test]
22213 fn pool_meta_write_state_blocks_on_any_recovery_required_selection() {
22214 fn assert_selection_blocks(replicas: Vec<PoolMetaReplica>) {
22215 let mut write_state = PoolMetaWriteState::default();
22216 select_pool_meta_replicas_observing(&mut write_state, replicas)
22217 .expect_err("recovery-required replicas must fail selection");
22218 write_state
22219 .ensure_write_safe("pool metadata save failed")
22220 .expect_err("a recovery-required selection must latch the write block");
22221 }
22222
22223 assert_selection_blocks(vec![PoolMetaReplica::Corrupt("truncated".to_string())]);
22224 assert_selection_blocks(vec![PoolMetaReplica::Incompatible("future format".to_string())]);
22225 assert_selection_blocks(vec![
22226 PoolMetaReplica::Corrupt("canonical unavailable".to_string()),
22227 decode_pool_meta_replica(pool_meta_replica_test_data("pool-old")),
22228 decode_pool_meta_replica(pool_meta_replica_test_data("pool-new")),
22229 ]);
22230 }
22231
22232 #[test]
22233 fn pool_meta_replica_selection_rejects_same_version_tuple_extension() {
22234 #[derive(Serialize)]
22235 struct FuturePersistedPoolMeta {
22236 version: u16,
22237 pools: Vec<PersistedPoolStatus>,
22238 generation: u64,
22239 }
22240
22241 let mut data = Vec::new();
22242 data.write_u16::<LittleEndian>(POOL_META_FORMAT)
22243 .expect("pool metadata format should encode");
22244 data.write_u16::<LittleEndian>(POOL_META_VERSION)
22245 .expect("pool metadata version should encode");
22246 FuturePersistedPoolMeta {
22247 version: POOL_META_VERSION,
22248 pools: Vec::new(),
22249 generation: 2,
22250 }
22251 .serialize(&mut Serializer::new(&mut data))
22252 .expect("extended tuple pool metadata should encode");
22253
22254 let err = select_pool_meta_replica(vec![
22255 decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
22256 decode_pool_meta_replica(data),
22257 ])
22258 .expect_err("same-version tuple extensions must block fallback repair writes");
22259
22260 assert!(err.to_string().contains("pool 1 is incompatible"));
22261 assert!(err.to_string().contains("version 2 tuple has unsupported field count"));
22262 }
22263
22264 #[test]
22265 fn pool_meta_replica_selection_falls_back_from_truncated_current_tuple() {
22266 let mut truncated = pool_meta_replica_test_data("pool-truncated");
22267 truncated.pop();
22268
22269 let selection = select_pool_meta_replica(vec![
22270 decode_pool_meta_replica(truncated),
22271 decode_pool_meta_replica(pool_meta_replica_test_data("pool-valid")),
22272 ])
22273 .expect("a truncated tuple should not block a validated backup replica");
22274
22275 assert!(selection.replica_state.needs_repair);
22276 assert!(selection.replica_state.repair_write_safe);
22277 assert_eq!(selection.meta.pools[0].cmd_line, "pool-valid");
22278 }
22279
22280 #[test]
22281 fn ensure_pool_not_left_in_cmdline_after_decommission_allows_active_pool() {
22282 assert!(ensure_pool_not_left_in_cmdline_after_decommission(0, "http://node{1...4}/disk{1...4}", false).is_ok());
22283 }
22284
22285 #[test]
22286 fn ensure_pool_not_left_in_cmdline_after_decommission_rejects_completed_pool() {
22287 let err = ensure_pool_not_left_in_cmdline_after_decommission(1, "http://node{1...4}/disk{1...4}", true)
22288 .expect_err("completed decommissioned pool should fail validation");
22289
22290 assert!(
22291 err.to_string()
22292 .contains("pool(2) = http://node{1...4}/disk{1...4} is decommissioned, please remove from server command line")
22293 );
22294 }
22295
22296 #[test]
22297 fn determine_decommission_final_state_marks_failures_and_cancellations() {
22298 assert_eq!(determine_decommission_final_state(0, false), DecommissionFinalState::Complete);
22299 assert_eq!(determine_decommission_final_state(1, false), DecommissionFinalState::Failed);
22300 assert_eq!(determine_decommission_final_state(0, true), DecommissionFinalState::Failed);
22301 }
22302
22303 #[test]
22304 fn lifecycle_action_removes_data_movement_version_rejects_delete_marker_action() {
22305 assert!(!lifecycle_action_removes_data_movement_version(IlmAction::DeleteAction));
22306 }
22307
22308 #[test]
22309 fn lifecycle_action_removes_data_movement_version_accepts_version_delete_actions() {
22310 assert!(lifecycle_action_removes_data_movement_version(IlmAction::DeleteVersionAction));
22311 assert!(lifecycle_action_removes_data_movement_version(IlmAction::DeleteAllVersionsAction));
22312 assert!(lifecycle_action_removes_data_movement_version(
22313 IlmAction::DelMarkerDeleteAllVersionsAction
22314 ));
22315 }
22316
22317 #[test]
22318 fn lifecycle_action_skips_heal_version_for_every_delete_action() {
22319 assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAction));
22320 assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteVersionAction));
22321 assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredAction));
22322 assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredVersionAction));
22323 assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAllVersionsAction));
22324 assert!(lifecycle_action_skips_heal_version(IlmAction::DelMarkerDeleteAllVersionsAction));
22325 assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionAction));
22326 assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionVersionAction));
22327 assert!(!lifecycle_action_skips_heal_version(IlmAction::NoneAction));
22328 }
22329
22330 #[test]
22331 fn resolve_data_movement_lifecycle_expiry_result_allows_dry_run_skip() {
22332 let skip = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, false, false)
22333 .expect("dry-run lifecycle evaluation should not require expiry enqueue");
22334
22335 assert!(skip);
22336 }
22337
22338 #[test]
22339 fn resolve_data_movement_lifecycle_expiry_result_rejects_apply_failure() {
22340 let err = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, true, false)
22341 .expect_err("failed lifecycle expiry enqueue should not be treated as skipped");
22342
22343 assert!(err.to_string().contains("failed to apply lifecycle expiry action"));
22344 }
22345
22346 #[test]
22349 fn decommission_capacity_failure_classification_covers_each_class() {
22350 let gate_busy = decommission_capacity_blocked_error(format!(
22351 "{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}2{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}"
22352 ));
22353 assert_eq!(
22354 classify_decommission_capacity_failure(&gate_busy),
22355 DecommissionCapacityFailure::TargetGateBusy { target_pool_index: 2 }
22356 );
22357
22358 let intent_conflict =
22359 decommission_capacity_blocked_error("decommission target mutation has an unresolved target capacity intent");
22360 assert_eq!(
22361 classify_decommission_capacity_failure(&intent_conflict),
22362 DecommissionCapacityFailure::IntentConflict
22363 );
22364
22365 let missing_source = Error::VersionNotFound("bucket".to_string(), "object".to_string(), "version".to_string());
22366 assert_eq!(
22367 classify_decommission_capacity_failure(&missing_source),
22368 DecommissionCapacityFailure::BenignContention
22369 );
22370
22371 let overwrite = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "v1".to_string());
22372 assert_eq!(
22373 classify_decommission_capacity_failure(&overwrite),
22374 DecommissionCapacityFailure::BenignContention
22375 );
22376
22377 let invalid_upload = Error::InvalidUploadID("bucket".to_string(), "object".to_string(), "upload".to_string());
22378 assert_eq!(
22379 classify_decommission_capacity_failure(&invalid_upload),
22380 DecommissionCapacityFailure::BenignContention
22381 );
22382
22383 assert_eq!(
22384 classify_decommission_capacity_failure(&Error::SlowDown),
22385 DecommissionCapacityFailure::Fatal
22386 );
22387 assert_eq!(
22388 classify_decommission_capacity_failure(&decommission_capacity_blocked_error(
22389 "decommission target mutation reservation identity is stale"
22390 )),
22391 DecommissionCapacityFailure::Fatal
22392 );
22393 }
22394
22395 #[test]
22397 fn decommission_capacity_failure_classification_sees_through_a_stage_wrapper() {
22398 let gate_busy = decommission_capacity_blocked_error(format!(
22399 "{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}1{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}"
22400 ));
22401 let wrapped =
22402 data_movement::data_movement_stage_error_for_test("decommission_object", "put_object_part", "b", "o", gate_busy);
22403
22404 assert_eq!(
22405 classify_decommission_capacity_failure(&wrapped),
22406 DecommissionCapacityFailure::TargetGateBusy { target_pool_index: 1 }
22407 );
22408 assert!(matches!(
22409 resolve_decommission_entry_failure(wrapped),
22410 Ok(DecommissionEntryOutcome::Deferred(_))
22411 ));
22412 }
22413
22414 #[tokio::test]
22415 #[serial_test::serial]
22416 async fn decommission_deferred_directory_object_is_copied_and_removed() {
22417 use crate::object_api::PutObjReader;
22418 let (_temp_dirs, store, _other_store) =
22419 Box::pin(crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None)).await;
22420 let bucket = "deferred-directory-object";
22421 let object = "folder/";
22422 Box::pin(store.make_bucket(bucket, &MakeBucketOptions::default()))
22423 .await
22424 .expect("create bucket");
22425 let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
22426 Box::pin(store.pools[0].put_object(
22427 bucket,
22428 &encode_dir_object(object),
22429 &mut PutObjReader::from_vec(Vec::new()),
22430 &ObjectOptions {
22431 expected_bucket_incarnation_id: Some(incarnation),
22432 ..Default::default()
22433 },
22434 ))
22435 .await
22436 .expect("seed directory object");
22437 assert!(
22438 store.pools[0]
22439 .get_disks_by_key(&encode_dir_object(object))
22440 .load_file_info_versions_exact(bucket, object)
22441 .await
22442 .expect("read seeded directory object")
22443 .is_some()
22444 );
22445 let retry_object = "retry.bin";
22446 Box::pin(store.pools[0].put_object(
22447 bucket,
22448 retry_object,
22449 &mut PutObjReader::from_vec(b"retry contents".to_vec()),
22450 &ObjectOptions {
22451 expected_bucket_incarnation_id: Some(incarnation),
22452 ..Default::default()
22453 },
22454 ))
22455 .await
22456 .expect("seed retry object");
22457 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
22458 set_decommission_capacity_info_overrides_for_test(
22459 store.id,
22460 vec![vec![
22461 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 16_384, 16_384),
22462 DecommissionPoolCapacityInfo::for_test(1, layout, 131_072, 131_072, 0),
22463 ]],
22464 );
22465 Box::pin(store.save_current_pool_meta_for_decommission_start(&[0], Vec::new()))
22466 .await
22467 .expect("activate reservation");
22468 let generation = store.active_decommission_generation(0).await.expect("active generation");
22469 let source_set = store.pools[0].get_disks_by_key(&encode_dir_object(object));
22470 let entry_error = Arc::new(tokio::sync::Mutex::new(None));
22471 let outcome = Box::pin(store.clone().replay_deferred_decommission_entry(
22472 CancellationToken::new(),
22473 0,
22474 0,
22475 generation,
22476 DeferredDecommissionEntry {
22477 name: object.into(),
22478 attempts: 2,
22479 },
22480 bucket.into(),
22481 source_set.clone(),
22482 None,
22483 None,
22484 None,
22485 Some(incarnation),
22486 Arc::new(AtomicUsize::new(0)),
22487 Arc::new(Semaphore::new(1)),
22488 entry_error.clone(),
22489 ))
22490 .await
22491 .expect("replay deferred directory object");
22492 assert!(matches!(outcome, DecommissionEntryOutcome::Complete));
22493 assert!(entry_error.lock().await.is_none());
22494 assert!(
22495 source_set
22496 .load_file_info_versions_exact(bucket, object)
22497 .await
22498 .expect("read exact source versions")
22499 .is_none(),
22500 "source object must be removed"
22501 );
22502 let target = store.pools[1]
22503 .get_disks_by_key(&encode_dir_object(object))
22504 .load_file_info_versions_exact(bucket, object)
22505 .await
22506 .expect("read exact target versions")
22507 .expect("directory object must reach target");
22508 assert_eq!(target.versions.len(), 1);
22509 assert_eq!(target.versions[0].size, 0);
22510 let faults = Arc::new(AtomicUsize::new(0));
22511 let observed = faults.clone();
22512 let _fault = DecommissionTestFaultGuard::install(Arc::new(move |stage, b, o, attempt, success| {
22513 if stage == "object_read" && b == bucket && o == retry_object && attempt == 1 && success {
22514 observed.fetch_add(1, Ordering::Relaxed);
22515 return true;
22516 }
22517 false
22518 }));
22519 let mut attempts = 2;
22520 let result = Box::pin(store.decommission_entry(
22521 CancellationToken::new(),
22522 0,
22523 generation,
22524 MetaCacheEntry {
22525 name: retry_object.into(),
22526 ..Default::default()
22527 },
22528 bucket.into(),
22529 store.pools[0].get_disks_by_key(retry_object),
22530 None,
22531 None,
22532 None,
22533 Some(incarnation),
22534 Arc::new(AtomicUsize::new(0)),
22535 &mut attempts,
22536 ))
22537 .await
22538 .expect("retry migration");
22539 assert!(matches!(result, DecommissionEntryOutcome::Complete));
22540 assert_eq!(faults.load(Ordering::Relaxed), 1, "inject one actual read failure");
22541 assert_eq!(attempts, 4, "carry two attempts, then count the entry pass and copy retry");
22542 }
22543
22544 #[test]
22545 fn decommission_object_attempt_metric_keeps_the_peak() {
22546 let mut attempts = 0;
22547 record_decommission_object_attempt(&mut attempts);
22548 record_decommission_object_attempt(&mut attempts);
22549 let mut deferred = DeferredDecommissionEntry {
22550 name: "object".into(),
22551 attempts,
22552 };
22553 record_decommission_object_attempt(&mut deferred.attempts);
22554 assert_eq!(deferred.attempts, 3);
22555 assert!(*DECOMMISSION_OBJECT_ATTEMPTS_MAX.lock().expect("attempt peak lock") >= 3);
22556 }
22557
22558 #[test]
22561 fn decommission_capacity_gate_retry_delay_is_bounded_by_its_ceiling() {
22562 for attempt in 1..=6 {
22563 let delay = decommission_capacity_gate_retry_delay(attempt);
22564 assert!(
22565 delay <= decommission_retry_backoff_ceiling(DECOMMISSION_CAPACITY_GATE_RETRY_BASE, attempt),
22566 "gate wait must stay within its jittered ceiling"
22567 );
22568 }
22569 }
22570
22571 #[test]
22573 fn decommission_deferred_drain_invariant_rejects_unresolved_entries() {
22574 assert!(ensure_decommission_deferred_drained("bucket-a", 2, 0).is_ok());
22575
22576 let err = ensure_decommission_deferred_drained("bucket-a", 2, 3)
22577 .expect_err("a set with deferred entries remaining must not complete");
22578 let message = err.to_string();
22579 assert!(message.contains("bucket-a"), "{message}");
22580 assert!(message.contains('2'), "{message}");
22581 assert!(message.contains('3'), "{message}");
22582 }
22583
22584 #[test]
22589 fn decommission_entry_failure_resolution_covers_each_class() {
22590 let gate_busy = decommission_capacity_blocked_error(format!(
22591 "{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}1{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}"
22592 ));
22593 let intent_conflict =
22594 decommission_capacity_blocked_error("decommission target mutation has an unresolved target capacity intent");
22595 for err in [gate_busy, intent_conflict] {
22596 let outcome = resolve_decommission_entry_failure(err.clone())
22597 .expect("transient capacity contention must defer, not fail the entry");
22598 match outcome {
22599 DecommissionEntryOutcome::Deferred(retained) => assert_eq!(retained, err),
22600 DecommissionEntryOutcome::Complete => panic!("contention must not complete the entry"),
22601 }
22602 }
22603
22604 for benign in [
22605 Error::ObjectNotFound("bucket".to_string(), "object".to_string()),
22606 Error::VersionNotFound("bucket".to_string(), "object".to_string(), "version".to_string()),
22607 Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "v1".to_string()),
22608 Error::InvalidUploadID("bucket".to_string(), "object".to_string(), "upload".to_string()),
22609 ] {
22610 assert!(
22611 matches!(resolve_decommission_entry_failure(benign.clone()), Ok(DecommissionEntryOutcome::Complete)),
22612 "benign contention must complete without failing the entry: {benign}"
22613 );
22614 }
22615
22616 assert!(resolve_decommission_entry_failure(Error::SlowDown).is_err());
22617 assert!(resolve_decommission_entry_failure(Error::OperationCanceled).is_err());
22618 }
22619
22620 #[test]
22621 fn decommission_copy_cleanup_safe_error_accepts_missing_source_errors() {
22622 assert!(is_decommission_copy_cleanup_safe_error(&Error::ObjectNotFound(
22623 "bucket".to_string(),
22624 "object".to_string()
22625 )));
22626 assert!(is_decommission_copy_cleanup_safe_error(&Error::VersionNotFound(
22627 "bucket".to_string(),
22628 "object".to_string(),
22629 "version".to_string()
22630 )));
22631 }
22632
22633 #[test]
22634 fn decommission_free_version_attempt_treats_missing_source_as_consumed() {
22635 let attempt =
22636 classify_decommission_free_version_attempt(Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())));
22637
22638 assert!(matches!(attempt, DecommissionFreeVersionAttempt::Consumed));
22639 }
22640
22641 #[test]
22642 fn decommission_free_version_attempt_preserves_capacity_failure() {
22643 let attempt = classify_decommission_free_version_attempt(Err(Error::DiskFull));
22644
22645 assert!(matches!(attempt, DecommissionFreeVersionAttempt::CapacityFailure(Error::DiskFull)));
22646 }
22647
22648 #[test]
22649 fn decommission_delete_marker_copy_error_rejects_data_movement_overwrite() {
22650 let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
22651
22652 assert!(!is_decommission_copy_cleanup_safe_error(&err));
22653 }
22654
22655 #[test]
22656 fn decommission_remote_tiered_copy_error_rejects_data_movement_overwrite() {
22657 let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
22658
22659 assert!(!is_decommission_copy_cleanup_safe_error(&err));
22660 }
22661
22662 #[test]
22663 fn decommission_target_capacity_error_accepts_direct_capacity_errors() {
22664 assert!(is_decommission_target_capacity_error(&Error::DiskFull));
22665 assert!(is_decommission_target_capacity_error(&Error::StorageFull));
22666 }
22667
22668 #[test]
22673 fn decommission_classifiers_see_through_a_stage_wrapper() {
22674 let wrap = |inner: Error| {
22675 data_movement::data_movement_stage_error_for_test("decommission_object", "put_object", "bucket-a", "object-a", inner)
22676 };
22677
22678 assert!(is_decommission_target_capacity_error(&wrap(Error::DiskFull)));
22680 assert!(is_decommission_target_capacity_error(&wrap(Error::StorageFull)));
22681 assert!(!is_decommission_target_capacity_error(&wrap(Error::SlowDown)));
22682
22683 let gate_busy = decommission_capacity_blocked_error(format!(
22684 "{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}7{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}"
22685 ));
22686 assert_eq!(decommission_capacity_target_gate_busy_index(&wrap(gate_busy)), Some(7));
22687
22688 let not_found = Error::ObjectNotFound("bucket-a".to_string(), "object-a".to_string());
22692 assert!(is_decommission_copy_cleanup_safe_error(¬_found));
22693 assert!(is_decommission_copy_cleanup_safe_error(&wrap(not_found)));
22694 assert!(!is_decommission_copy_cleanup_safe_error(&wrap(Error::SlowDown)));
22695 }
22696
22697 #[test]
22698 fn decommission_target_gate_retry_recognizes_multipart_part_errors() {
22699 let wrap = |inner: Error| {
22700 data_movement::data_movement_part_stage_error_for_test(
22701 "decommission_object",
22702 "put_object_part",
22703 "bucket-a",
22704 "object-a",
22705 1,
22706 inner,
22707 )
22708 };
22709 let gate_busy_message =
22710 format!("{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}7{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}");
22711 let wrapped = wrap(decommission_capacity_blocked_error(&gate_busy_message));
22712 assert!(is_decommission_capacity_target_gate_busy(&wrapped));
22713 assert_eq!(decommission_capacity_target_gate_busy_index(&wrapped), Some(7));
22714 assert_eq!(
22715 wrapped.to_string(),
22716 format!(
22717 "Io error: decommission_object: put_object_part failed for bucket-a/object-a part 1: {}",
22718 decommission_capacity_blocked_error(&gate_busy_message)
22719 )
22720 );
22721
22722 for unrelated in [
22723 Error::SlowDown,
22724 Error::DiskFull,
22725 decommission_capacity_blocked_error("target capacity is exhausted"),
22726 Error::other(gate_busy_message),
22727 ] {
22728 let wrapped = wrap(unrelated);
22729 assert!(!is_decommission_capacity_target_gate_busy(&wrapped));
22730 assert_eq!(decommission_capacity_target_gate_busy_index(&wrapped), None);
22731 }
22732 for missing_target in [
22733 Error::FileNotFound,
22734 Error::ObjectNotFound("bucket-a".to_string(), "object-a".to_string()),
22735 Error::VersionNotFound("bucket-a".to_string(), "object-a".to_string(), "version-a".to_string()),
22736 ] {
22737 assert!(is_decommission_copy_cleanup_safe_error(&missing_target));
22738 assert!(
22739 !is_decommission_copy_cleanup_safe_error(&wrap(missing_target)),
22740 "a missing target part must never authorize source cleanup"
22741 );
22742 }
22743 assert!(is_decommission_target_capacity_error(&wrap(Error::DiskFull)));
22744 }
22745
22746 #[test]
22747 fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
22748 let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
22749 let storage_full = Error::other(format!(
22750 "decommission_object: put_object failed for bucket/object: {}",
22751 Error::StorageFull
22752 ));
22753
22754 assert!(is_decommission_target_capacity_error(&disk_full));
22755 assert!(is_decommission_target_capacity_error(&storage_full));
22756 }
22757
22758 #[test]
22759 fn decommission_capacity_intent_conflict_accepts_context_wrapped_error() {
22760 let err = with_decommission_entry_context(
22761 "migrate_object",
22762 "bucket",
22763 "object",
22764 decommission_capacity_blocked_error("target has an unresolved target capacity intent"),
22765 );
22766
22767 assert!(is_decommission_capacity_intent_conflict(&err));
22768 assert_eq!(
22769 decommission_capacity_retry_kind(&err, DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS - 1),
22770 Some(DecommissionCapacityRetryKind::IntentConflict)
22771 );
22772 assert_eq!(
22773 decommission_capacity_retry_kind(&err, DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS),
22774 None,
22775 "a durable intent conflict must retain its bounded recovery policy"
22776 );
22777 }
22778
22779 #[test]
22780 fn decommission_target_capacity_error_rejects_unrelated_errors() {
22781 assert!(!is_decommission_target_capacity_error(&Error::SlowDown));
22782 }
22783
22784 #[test]
22785 fn decommission_scanner_backlog_handoff_retry_is_typed_and_bounded() {
22786 use data_movement::scanner_backlog::ScannerPauseBacklogRetirementError;
22787
22788 let handoff = data_movement::data_movement_context_error(
22789 "scanner pause backlog retirement planner failed: competing proofs".to_string(),
22790 ScannerPauseBacklogRetirementError::HandoffConflict {
22791 reason: "competing maximum-membership commit proofs".to_string(),
22792 },
22793 );
22794 assert!(is_decommission_scanner_backlog_handoff_conflict(&handoff));
22795 assert_eq!(
22796 decommission_scanner_backlog_retry_kind(&handoff, DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS - 1),
22797 Some(DecommissionScannerBacklogRetryKind::HandoffConflict)
22798 );
22799 assert_eq!(
22800 decommission_scanner_backlog_retry_kind(&handoff, DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS),
22801 None,
22802 "a divergent handoff must not retry forever"
22803 );
22804
22805 for non_retryable in [
22806 ScannerPauseBacklogRetirementError::AuthorityConflict {
22807 reason: "larger source authority".to_string(),
22808 },
22809 ScannerPauseBacklogRetirementError::InvalidRecord {
22810 reason: "unread member".to_string(),
22811 },
22812 ] {
22813 let err = data_movement::data_movement_context_error("planner failed".to_string(), non_retryable);
22814 assert!(!is_decommission_scanner_backlog_handoff_conflict(&err));
22815 assert_eq!(
22816 decommission_scanner_backlog_retry_kind(&err, 0),
22817 None,
22818 "only a handoff conflict may be retried"
22819 );
22820 }
22821 }
22822
22823 #[test]
22824 fn should_skip_decommission_delete_marker_characterizes_empty_marker_without_replication() {
22825 let version = rustfs_filemeta::FileInfo {
22826 deleted: true,
22827 ..Default::default()
22828 };
22829
22830 assert!(should_skip_decommission_delete_marker(&version, 1, false));
22831 }
22832
22833 #[test]
22834 fn should_skip_decommission_delete_marker_characterizes_replication_configured() {
22835 let version = rustfs_filemeta::FileInfo {
22836 deleted: true,
22837 ..Default::default()
22838 };
22839
22840 assert!(!should_skip_decommission_delete_marker(&version, 1, true));
22841 }
22842
22843 #[test]
22844 fn should_skip_decommission_delete_marker_rejects_non_deleted_versions() {
22845 let version = rustfs_filemeta::FileInfo::default();
22846
22847 assert!(!should_skip_decommission_delete_marker(&version, 1, false));
22848 }
22849
22850 #[test]
22851 fn should_skip_decommission_delete_marker_rejects_multiple_remaining_versions() {
22852 let version = rustfs_filemeta::FileInfo {
22853 deleted: true,
22854 ..Default::default()
22855 };
22856
22857 assert!(!should_skip_decommission_delete_marker(&version, 2, false));
22858 }
22859
22860 #[test]
22861 fn decommission_delete_marker_opts_preserves_replication_state() {
22862 let mod_time = OffsetDateTime::now_utc();
22863 let version = rustfs_filemeta::FileInfo {
22864 mod_time: Some(mod_time),
22865 replication_state_internal: Some(crate::bucket::replication::replication_state_to_filemeta(&ReplicationState {
22866 replica_status: ReplicationStatusType::Replica,
22867 delete_marker: true,
22868 replicate_decision_str: "existing".to_string(),
22869 ..Default::default()
22870 })),
22871 ..Default::default()
22872 };
22873
22874 let incarnation = uuid::Uuid::new_v4();
22875 let opts = decommission_delete_marker_opts(&version, Some("version-id".to_string()), 7, Some(incarnation));
22876 let replication = opts.delete_replication.expect("replication state should be preserved");
22877
22878 assert!(opts.versioned);
22879 assert!(opts.data_movement);
22880 assert!(opts.delete_marker);
22881 assert!(opts.skip_decommissioned);
22882 assert_eq!(opts.src_pool_idx, 7);
22883 assert_eq!(opts.version_id.as_deref(), Some("version-id"));
22884 assert_eq!(opts.mod_time, Some(mod_time));
22885 assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
22886 assert_eq!(replication.replica_status, ReplicationStatusType::Replica);
22887 assert!(replication.delete_marker);
22888 assert_eq!(replication.replicate_decision_str, "existing");
22889 }
22890
22891 #[test]
22892 fn decommission_delete_marker_opts_preserves_suspended_null_version() {
22893 let version = rustfs_filemeta::FileInfo {
22894 name: "object".to_string(),
22895 deleted: true,
22896 ..Default::default()
22897 };
22898 let opts = decommission_delete_marker_opts(&version, None, 7, None);
22899
22900 assert!(!opts.versioned);
22901 assert!(opts.version_suspended);
22902 assert_eq!(opts.version_id.as_deref(), Some(uuid::Uuid::nil().to_string().as_str()));
22903
22904 let owner = DecommissionCapacityOwner {
22905 source_pool_index: 7,
22906 operation_id: uuid::Uuid::new_v4(),
22907 generation: 1,
22908 owner_nonce: uuid::Uuid::new_v4(),
22909 mutation_id: None,
22910 };
22911 assert_eq!(
22912 decommission_capacity_version_mutation_id(owner, "bucket", &version),
22913 decommission_capacity_mutation_id(
22914 owner,
22915 "bucket",
22916 &version.name,
22917 opts.version_id.as_deref(),
22918 opts.delete_marker,
22919 opts.mod_time,
22920 )
22921 );
22922 }
22923
22924 #[test]
22925 fn test_decommission_object_migration_read_opts_are_raw_data_movement() {
22926 let opts = decommission_object_migration_read_opts(Some("vid-1".to_string()));
22927
22928 assert_eq!(opts.version_id.as_deref(), Some("vid-1"));
22929 assert!(opts.no_lock);
22930 assert!(opts.data_movement);
22931 assert!(opts.raw_data_movement_read);
22932 assert!(opts.skip_rebalancing);
22933 assert!(opts.skip_decommissioned);
22934 }
22935
22936 #[test]
22937 fn decommission_remote_tiered_opts_preserves_versioning_context() {
22938 let mod_time = OffsetDateTime::now_utc();
22939 let version = rustfs_filemeta::FileInfo {
22940 mod_time: Some(mod_time),
22941 metadata: HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())]),
22942 ..Default::default()
22943 };
22944
22945 let incarnation = uuid::Uuid::new_v4();
22946 let opts = decommission_remote_tiered_opts(&version, Some("version-id".to_string()), 9, Some(incarnation));
22947
22948 assert!(opts.versioned);
22949 assert!(opts.data_movement);
22950 assert_eq!(opts.src_pool_idx, 9);
22951 assert_eq!(opts.version_id.as_deref(), Some("version-id"));
22952 assert_eq!(opts.mod_time, Some(mod_time));
22953 assert_eq!(opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value"));
22954 assert!(opts.include_part_checksums);
22955 assert!(opts.http_preconditions.is_some());
22956 assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
22957 assert!(!opts.incl_free_versions);
22958
22959 let mut free_version = version;
22960 free_version.set_tier_free_version();
22961 let free_opts = decommission_remote_tiered_opts(&free_version, Some("free-version-id".to_string()), 9, Some(incarnation));
22962 assert!(free_opts.incl_free_versions);
22963 }
22964
22965 #[test]
22966 fn decommission_terminal_state_transitions_update_start_time() {
22967 let start_time = OffsetDateTime::now_utc();
22968 let build_pool_meta = || PoolMeta {
22969 version: POOL_META_VERSION,
22970 pools: vec![PoolStatus {
22971 id: 0,
22972 cmd_line: "/tmp/pool".to_string(),
22973 last_update: start_time,
22974 decommission: Some(PoolDecommissionInfo {
22975 start_time: Some(start_time),
22976 ..Default::default()
22977 }),
22978 }],
22979 dont_save: true,
22980 };
22981
22982 let mut pool_meta = build_pool_meta();
22983 assert!(pool_meta.decommission_failed(0));
22984 assert_eq!(pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), None);
22985
22986 let mut pool_meta = build_pool_meta();
22987 assert!(pool_meta.decommission_complete(0));
22988 assert_eq!(
22989 pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time),
22990 Some(start_time)
22991 );
22992
22993 let mut pool_meta = build_pool_meta();
22994 assert!(pool_meta.decommission_cancel(0));
22995 assert_eq!(pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), None);
22996
22997 let mut pool_meta = build_pool_meta();
22998 assert!(pool_meta.decommission_cancel(0));
22999 assert!(!pool_meta.decommission_complete(0));
23000
23001 let mut pool_meta = build_pool_meta();
23002 assert!(pool_meta.decommission_failed(0));
23003 assert!(!pool_meta.decommission_complete(0));
23004 }
23005
23006 #[test]
23007 fn decommission_capacity_reporting_preserves_existing_disk_layout() {
23008 #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
23010 #[serde(deny_unknown_fields)]
23011 struct PreContentionDecommissionInfo {
23012 #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
23013 pub start_time: Option<OffsetDateTime>,
23014 #[serde(rename = "startSize")]
23015 pub start_size: usize,
23016 #[serde(rename = "totalSize")]
23017 pub total_size: usize,
23018 #[serde(rename = "currentSize")]
23019 pub current_size: usize,
23020 #[serde(rename = "complete")]
23021 pub complete: bool,
23022 #[serde(rename = "failed")]
23023 pub failed: bool,
23024 #[serde(rename = "canceled")]
23025 pub canceled: bool,
23026 #[serde(rename = "queued", default)]
23027 pub queued: bool,
23028 #[serde(rename = "queuedBuckets", default)]
23029 pub queued_buckets: Vec<String>,
23030 #[serde(rename = "decommissionedBuckets", default)]
23031 pub decommissioned_buckets: Vec<String>,
23032 #[serde(rename = "bucket", default)]
23033 pub bucket: String,
23034 #[serde(rename = "prefix", default)]
23035 pub prefix: String,
23036 #[serde(rename = "object", default)]
23037 pub object: String,
23038 #[serde(rename = "objectsDecommissioned")]
23039 pub items_decommissioned: usize,
23040 #[serde(rename = "objectsDecommissionedFailed")]
23041 pub items_decommission_failed: usize,
23042 #[serde(rename = "bytesDecommissioned")]
23043 pub bytes_done: usize,
23044 #[serde(rename = "bytesDecommissionedFailed")]
23045 pub bytes_failed: usize,
23046 #[serde(rename = "terminalReloadAttemptAt", with = "time::serde::rfc3339::option", default)]
23047 pub terminal_reload_attempt_at: Option<OffsetDateTime>,
23048 #[serde(rename = "terminalReloadFailures", default)]
23049 pub terminal_reload_failures: Vec<String>,
23050 #[serde(rename = "unresolvedEntries", default)]
23051 pub unresolved_entries: Vec<DecommissionUnresolvedEntry>,
23052 #[serde(rename = "capacityReservation", default)]
23053 pub capacity_reservation: Option<DecommissionCapacityReservation>,
23054 #[serde(rename = "capacityBlockedReason", default)]
23055 pub capacity_blocked_reason: Option<String>,
23056 }
23057 let current = PersistedPoolDecommissionInfo {
23058 capacity_blocked_reason: Some("target gate busy".into()),
23059 ..Default::default()
23060 };
23061 for bytes in [rmp_serde::to_vec(¤t), rmp_serde::to_vec_named(¤t)] {
23062 let bytes = bytes.expect("serialize current decommission state");
23063 let old: PreContentionDecommissionInfo =
23064 rmp_serde::from_slice(&bytes).expect("existing V2/V3 readers must accept new writes");
23065 assert_eq!(old.capacity_blocked_reason, current.capacity_blocked_reason);
23066 }
23067 }
23068
23069 #[test]
23070 fn pool_meta_persists_decommission_resume_queues() {
23071 let start_time = OffsetDateTime::now_utc();
23072 let unresolved_entry = DecommissionUnresolvedEntry {
23073 bucket: "bucket-b".to_string(),
23074 object: "prefix/unresolved.txt".to_string(),
23075 pool_index: 0,
23076 set_index: 1,
23077 source_generation: start_time,
23078 candidate_count: 2,
23079 disk_error_count: 1,
23080 observed_at: start_time,
23081 reason: "metadata_resolution_failed".to_string(),
23082 };
23083 let pool_meta = PoolMeta {
23084 version: POOL_META_VERSION,
23085 pools: vec![PoolStatus {
23086 id: 1,
23087 cmd_line: "/data/pool1/disk{1...4}".to_string(),
23088 last_update: start_time,
23089 decommission: Some(PoolDecommissionInfo {
23090 start_time: Some(start_time),
23091 queued: true,
23092 queued_buckets: vec!["bucket-a".to_string(), "bucket-b/prefix".to_string()],
23093 decommissioned_buckets: vec!["bucket-done".to_string()],
23094 bucket: "bucket-b".to_string(),
23095 prefix: "prefix".to_string(),
23096 object: "object.txt".to_string(),
23097 items_decommissioned: 7,
23098 items_decommission_failed: 1,
23099 bytes_done: 1024,
23100 bytes_failed: 128,
23101 terminal_reload_attempt_at: Some(start_time),
23102 terminal_reload_failures: vec!["complete_decommission: peer node-a failed".to_string()],
23103 unresolved_entries: vec![unresolved_entry.clone()],
23104 ..Default::default()
23105 }),
23106 }],
23107 dont_save: false,
23108 };
23109
23110 let mut buf = Vec::new();
23111 PersistedPoolMeta::from(&pool_meta)
23112 .serialize(&mut Serializer::new(&mut buf))
23113 .expect("pool meta should serialize");
23114
23115 let mut deserializer = Deserializer::new(Cursor::new(&buf));
23116 let restored: PoolMeta = PersistedPoolMeta::deserialize(&mut deserializer)
23117 .expect("pool meta should deserialize")
23118 .try_into()
23119 .expect("pool meta should validate");
23120
23121 let restored_decommission = restored.pools[0]
23122 .decommission
23123 .as_ref()
23124 .expect("decommission info should survive round-trip");
23125 assert_eq!(
23126 restored_decommission.queued_buckets,
23127 vec!["bucket-a".to_string(), "bucket-b/prefix".to_string()]
23128 );
23129 assert_eq!(restored_decommission.decommissioned_buckets, vec!["bucket-done".to_string()]);
23130 assert_eq!(restored_decommission.bucket, "bucket-b");
23131 assert_eq!(restored_decommission.prefix, "prefix");
23132 assert_eq!(restored_decommission.object, "object.txt");
23133 assert!(restored_decommission.stage.is_empty());
23134 assert_eq!(restored_decommission.items_decommissioned, 7);
23135 assert_eq!(restored_decommission.items_decommission_failed, 1);
23136 assert_eq!(restored_decommission.bytes_done, 1024);
23137 assert_eq!(restored_decommission.bytes_failed, 128);
23138 assert_eq!(restored_decommission.terminal_reload_attempt_at, Some(start_time));
23139 assert_eq!(
23140 restored_decommission.terminal_reload_failures,
23141 vec!["complete_decommission: peer node-a failed".to_string()]
23142 );
23143 assert_eq!(restored_decommission.unresolved_entries, vec![unresolved_entry]);
23144 assert!(restored_decommission.queued);
23145 assert_eq!(restored_decommission.items_since_last_progress_save(), 0);
23146 }
23147
23148 #[test]
23149 fn pool_meta_records_decommission_terminal_reload_failure_once() {
23150 let start_time = OffsetDateTime::now_utc();
23151 let mut pool_meta = PoolMeta {
23152 version: POOL_META_VERSION,
23153 pools: vec![PoolStatus {
23154 id: 1,
23155 cmd_line: "/data/pool1/disk{1...4}".to_string(),
23156 last_update: start_time,
23157 decommission: Some(PoolDecommissionInfo {
23158 complete: true,
23159 ..Default::default()
23160 }),
23161 }],
23162 dont_save: false,
23163 };
23164
23165 assert!(
23166 pool_meta
23167 .record_decommission_terminal_reload_failure(0, "complete_decommission", "peer node-a failed".to_string())
23168 .expect("terminal reload failure should be recorded")
23169 );
23170 assert!(
23171 !pool_meta
23172 .record_decommission_terminal_reload_failure(0, "complete_decommission", "peer node-a failed".to_string())
23173 .expect("duplicate terminal reload failure should be ignored")
23174 );
23175
23176 let decommission = pool_meta.pools[0].decommission.as_ref().expect("decommission should exist");
23177 assert!(decommission.terminal_reload_attempt_at.is_some());
23178 assert_eq!(
23179 decommission.terminal_reload_failures,
23180 vec!["complete_decommission: peer node-a failed".to_string()]
23181 );
23182 }
23183
23184 #[test]
23185 fn pool_meta_decode_supports_legacy_payload() {
23186 let start_time = OffsetDateTime::now_utc();
23187 let legacy_meta = LegacyPoolMeta {
23188 version: POOL_META_V1_VERSION,
23189 pools: vec![LegacyPoolStatus {
23190 id: 3,
23191 cmd_line: "/legacy/pool".to_string(),
23192 last_update: start_time,
23193 decommission: Some(LegacyPoolDecommissionInfo {
23194 start_time: Some(start_time),
23195 items_decommissioned: 9,
23196 items_decommission_failed: 2,
23197 bytes_done: 2048,
23198 bytes_failed: 256,
23199 ..Default::default()
23200 }),
23201 }],
23202 dont_save: true,
23203 };
23204
23205 let mut legacy_payload = Vec::new();
23206 legacy_meta
23207 .serialize(&mut Serializer::new(&mut legacy_payload))
23208 .expect("legacy payload should serialize");
23209
23210 let persisted_decode: std::result::Result<PersistedPoolMeta, _> = rmp_serde::from_slice(&legacy_payload);
23212 assert!(persisted_decode.is_err());
23213
23214 let decoded =
23215 PoolMeta::decode_pool_meta_payload(POOL_META_V1_VERSION, &legacy_payload).expect("legacy payload should decode");
23216 assert_eq!(decoded.version, POOL_META_V1_VERSION);
23217 assert!(!decoded.dont_save, "runtime-only flag should reset on load");
23218 assert_eq!(decoded.pools.len(), 1);
23219 assert_eq!(decoded.pools[0].id, 3);
23220 assert_eq!(decoded.pools[0].cmd_line, "/legacy/pool");
23221 assert_eq!(decoded.pools[0].last_update, start_time);
23222
23223 let decommission = decoded.pools[0].decommission.as_ref().expect("decommission should decode");
23224 assert_eq!(decommission.start_time, Some(start_time));
23225 assert_eq!(decommission.items_decommissioned, 9);
23226 assert_eq!(decommission.items_decommission_failed, 2);
23227 assert_eq!(decommission.bytes_done, 2048);
23228 assert_eq!(decommission.bytes_failed, 256);
23229 assert_eq!(decommission.items_since_last_progress_save(), 0);
23230 assert!(decommission.queued_buckets.is_empty());
23232 assert!(decommission.decommissioned_buckets.is_empty());
23233 assert!(decommission.bucket.is_empty());
23234 assert!(decommission.prefix.is_empty());
23235 assert!(decommission.object.is_empty());
23236 assert!(decommission.unresolved_entries.is_empty());
23237 }
23238
23239 #[test]
23240 fn pool_meta_decode_rejects_unknown_legacy_fields() {
23241 #[derive(Serialize)]
23242 struct LegacyPoolMetaWithUnknownField {
23243 version: u16,
23244 pools: Vec<LegacyPoolStatus>,
23245 dont_save: bool,
23246 unexpected: bool,
23247 }
23248
23249 let payload = rmp_serde::to_vec_named(&LegacyPoolMetaWithUnknownField {
23250 version: POOL_META_V1_VERSION,
23251 pools: Vec::new(),
23252 dont_save: true,
23253 unexpected: true,
23254 })
23255 .expect("legacy pool metadata with unknown field should serialize");
23256
23257 let err = PoolMeta::decode_pool_meta_payload(POOL_META_V1_VERSION, payload.as_slice())
23258 .expect_err("unknown legacy pool metadata field should fail decode");
23259 let rendered = err.to_string();
23260 assert!(rendered.contains("PoolMeta v1 decode failed for both persisted and legacy formats"));
23261 assert!(rendered.contains("unknown field") || rendered.contains("missing field"));
23262 }
23263
23264 #[test]
23265 fn pool_meta_decode_rejects_unknown_persisted_fields() {
23266 #[derive(Serialize)]
23267 struct PersistedPoolMetaWithUnknownField {
23268 version: u16,
23269 pools: Vec<PersistedPoolStatus>,
23270 unexpected: bool,
23271 }
23272
23273 let payload = rmp_serde::to_vec_named(&PersistedPoolMetaWithUnknownField {
23274 version: POOL_META_VERSION,
23275 pools: Vec::new(),
23276 unexpected: true,
23277 })
23278 .expect("pool metadata with unknown field should serialize");
23279
23280 let err = PoolMeta::decode_pool_meta_payload(POOL_META_VERSION, payload.as_slice())
23281 .expect_err("unknown persisted pool metadata field should fail decode");
23282 let rendered = err.to_string();
23283 assert!(rendered.contains("PoolMeta v2 decode failed"));
23284 assert!(rendered.contains("unknown field") || rendered.contains("missing field"));
23285 }
23286
23287 #[test]
23288 fn pool_meta_decode_rejects_missing_critical_persisted_fields() {
23289 #[derive(Serialize)]
23290 struct PersistedPoolMetaWithoutPools {
23291 version: u16,
23292 }
23293
23294 let payload = rmp_serde::to_vec_named(&PersistedPoolMetaWithoutPools {
23295 version: POOL_META_VERSION,
23296 })
23297 .expect("pool metadata without pools should serialize");
23298
23299 let err = PoolMeta::decode_pool_meta_payload(POOL_META_VERSION, payload.as_slice())
23300 .expect_err("missing persisted pool metadata pools should fail decode");
23301 assert!(err.to_string().contains("PoolMeta v2 decode failed"));
23302 }
23303
23304 #[test]
23305 fn pool_meta_decode_rejects_unknown_decommission_fields() {
23306 #[derive(Serialize)]
23307 struct PersistedPoolStatusWithUnknownDecommission {
23308 #[serde(rename = "id")]
23309 id: usize,
23310 #[serde(rename = "cmdline")]
23311 cmd_line: String,
23312 #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
23313 last_update: OffsetDateTime,
23314 #[serde(rename = "decommissionInfo")]
23315 decommission: Option<PersistedPoolDecommissionInfoWithUnknownField>,
23316 }
23317
23318 #[derive(Serialize)]
23319 struct PersistedPoolDecommissionInfoWithUnknownField {
23320 #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
23321 start_time: Option<OffsetDateTime>,
23322 #[serde(rename = "startSize")]
23323 start_size: usize,
23324 #[serde(rename = "totalSize")]
23325 total_size: usize,
23326 #[serde(rename = "currentSize")]
23327 current_size: usize,
23328 #[serde(rename = "complete")]
23329 complete: bool,
23330 #[serde(rename = "failed")]
23331 failed: bool,
23332 #[serde(rename = "canceled")]
23333 canceled: bool,
23334 #[serde(rename = "queuedBuckets")]
23335 queued_buckets: Vec<String>,
23336 #[serde(rename = "decommissionedBuckets")]
23337 decommissioned_buckets: Vec<String>,
23338 #[serde(rename = "bucket")]
23339 bucket: String,
23340 #[serde(rename = "prefix")]
23341 prefix: String,
23342 #[serde(rename = "object")]
23343 object: String,
23344 #[serde(rename = "objectsDecommissioned")]
23345 items_decommissioned: usize,
23346 #[serde(rename = "objectsDecommissionedFailed")]
23347 items_decommission_failed: usize,
23348 #[serde(rename = "bytesDecommissioned")]
23349 bytes_done: usize,
23350 #[serde(rename = "bytesDecommissionedFailed")]
23351 bytes_failed: usize,
23352 #[serde(rename = "unexpected")]
23353 unexpected: bool,
23354 }
23355
23356 #[derive(Serialize)]
23357 struct PersistedPoolMetaWithUnknownDecommission {
23358 version: u16,
23359 pools: Vec<PersistedPoolStatusWithUnknownDecommission>,
23360 }
23361
23362 let start_time = OffsetDateTime::now_utc();
23363 let payload = rmp_serde::to_vec_named(&PersistedPoolMetaWithUnknownDecommission {
23364 version: POOL_META_VERSION,
23365 pools: vec![PersistedPoolStatusWithUnknownDecommission {
23366 id: 0,
23367 cmd_line: "/data/pool".to_string(),
23368 last_update: start_time,
23369 decommission: Some(PersistedPoolDecommissionInfoWithUnknownField {
23370 start_time: Some(start_time),
23371 start_size: 0,
23372 total_size: 0,
23373 current_size: 0,
23374 complete: false,
23375 failed: false,
23376 canceled: false,
23377 queued_buckets: Vec::new(),
23378 decommissioned_buckets: Vec::new(),
23379 bucket: String::new(),
23380 prefix: String::new(),
23381 object: String::new(),
23382 items_decommissioned: 0,
23383 items_decommission_failed: 0,
23384 bytes_done: 0,
23385 bytes_failed: 0,
23386 unexpected: true,
23387 }),
23388 }],
23389 })
23390 .expect("pool metadata with unknown decommission field should serialize");
23391
23392 let err = PoolMeta::decode_pool_meta_payload(POOL_META_VERSION, payload.as_slice())
23393 .expect_err("unknown persisted decommission metadata field should fail decode");
23394 assert!(err.to_string().contains("PoolMeta v2 decode failed"));
23395 }
23396
23397 #[test]
23398 fn pool_meta_decode_rejects_invalid_decommission_terminal_state() {
23399 let start_time = OffsetDateTime::now_utc();
23400 let persisted_meta = PersistedPoolMeta {
23401 version: POOL_META_VERSION,
23402 pools: vec![PersistedPoolStatus {
23403 id: 1,
23404 cmd_line: "/data/pool1/disk{1...4}".to_string(),
23405 last_update: start_time,
23406 decommission: Some(PersistedPoolDecommissionInfo {
23407 start_time: Some(start_time),
23408 complete: true,
23409 failed: true,
23410 canceled: false,
23411 ..Default::default()
23412 }),
23413 }],
23414 };
23415
23416 let mut payload = Vec::new();
23417 persisted_meta
23418 .serialize(&mut Serializer::new(&mut payload))
23419 .expect("persisted payload should serialize");
23420
23421 let err = PoolMeta::decode_pool_meta_payload(POOL_META_VERSION, &payload)
23422 .expect_err("invalid terminal state should fail decode");
23423 assert!(err.to_string().contains("invalid decommission terminal state"));
23424 }
23425
23426 #[test]
23427 fn pool_meta_decode_rejects_invalid_legacy_decommission_terminal_state() {
23428 let start_time = OffsetDateTime::now_utc();
23429 let legacy_meta = LegacyPoolMeta {
23430 version: POOL_META_V1_VERSION,
23431 pools: vec![LegacyPoolStatus {
23432 id: 1,
23433 cmd_line: "/legacy/pool".to_string(),
23434 last_update: start_time,
23435 decommission: Some(LegacyPoolDecommissionInfo {
23436 start_time: Some(start_time),
23437 complete: true,
23438 failed: false,
23439 canceled: true,
23440 ..Default::default()
23441 }),
23442 }],
23443 dont_save: false,
23444 };
23445
23446 let mut payload = Vec::new();
23447 legacy_meta
23448 .serialize(&mut Serializer::new(&mut payload))
23449 .expect("legacy payload should serialize");
23450
23451 let err = PoolMeta::decode_pool_meta_payload(POOL_META_V1_VERSION, &payload)
23452 .expect_err("invalid legacy terminal state should fail decode");
23453 assert!(err.to_string().contains("invalid decommission terminal state"));
23454 }
23455}
23456
23457pub type ListCallback = Arc<dyn Fn(MetaCacheEntry) -> BoxFuture<'static, ()> + Send + Sync + 'static>;
23460
23461const DECOMMISSION_ENTRY_QUEUE_HARD_CAP: usize = 256;
23462
23463struct QueuedDecommissionEntry {
23464 entry: MetaCacheEntry,
23465 queue_permit: OwnedSemaphorePermit,
23466}
23467
23468enum DecommissionEntryEnqueueResult {
23469 Enqueued,
23470 Canceled,
23471 Closed,
23472}
23473
23474fn decommission_entry_queue_capacity(worker_limit: usize) -> usize {
23475 worker_limit.saturating_mul(2).clamp(1, DECOMMISSION_ENTRY_QUEUE_HARD_CAP)
23476}
23477
23478async fn enqueue_decommission_entry(
23479 rx: &CancellationToken,
23480 outstanding: &Arc<Semaphore>,
23481 tx: &mpsc::Sender<QueuedDecommissionEntry>,
23482 entry: MetaCacheEntry,
23483) -> DecommissionEntryEnqueueResult {
23484 let queue_permit = match tokio::select! {
23485 biased;
23486 _ = rx.cancelled() => return DecommissionEntryEnqueueResult::Canceled,
23487 permit = outstanding.clone().acquire_owned() => permit,
23488 } {
23489 Ok(permit) => permit,
23490 Err(_) => return DecommissionEntryEnqueueResult::Closed,
23491 };
23492
23493 let queued = QueuedDecommissionEntry { entry, queue_permit };
23494 tokio::select! {
23495 biased;
23496 _ = rx.cancelled() => DecommissionEntryEnqueueResult::Canceled,
23497 result = tx.send(queued) => {
23498 if result.is_ok() {
23499 DecommissionEntryEnqueueResult::Enqueued
23500 } else {
23501 DecommissionEntryEnqueueResult::Closed
23502 }
23503 }
23504 }
23505}
23506
23507async fn drain_decommission_entry_queue(rx: &CancellationToken, outstanding: &Arc<Semaphore>, capacity: usize) -> bool {
23508 let Ok(permits) = u32::try_from(capacity) else {
23509 return true;
23510 };
23511
23512 tokio::select! {
23513 _ = rx.cancelled() => true,
23514 result = outstanding.acquire_many(permits) => result.is_err(),
23515 }
23516}
23517
23518async fn record_decommission_entry_error(
23519 entry_error: &Arc<tokio::sync::Mutex<Option<Error>>>,
23520 rx: &CancellationToken,
23521 err: Error,
23522) -> bool {
23523 if rx.is_cancelled() {
23524 return false;
23525 }
23526
23527 let mut first_err = entry_error.lock().await;
23528 if first_err.is_none() && !rx.is_cancelled() {
23529 *first_err = Some(err);
23530 rx.cancel();
23531 return true;
23532 }
23533 false
23534}
23535
23536fn ensure_decommission_unresolved_verification_disk_count(
23537 expected: usize,
23538 actual: usize,
23539 pool_index: usize,
23540 set_index: usize,
23541) -> Result<()> {
23542 if actual == expected {
23543 return Ok(());
23544 }
23545 Err(Error::other(format!(
23546 "decommission unresolved-entry verification for pool {pool_index} set {set_index} requires all {expected} source disks, but only {actual} are online"
23547 )))
23548}
23549
23550impl SetDisks {
23551 async fn decommission_unresolved_entry_absent_on_all_disks(
23552 &self,
23553 pool_index: usize,
23554 entry: &DecommissionUnresolvedEntry,
23555 ) -> Result<bool> {
23556 let (disks, _) = self.get_online_disks_with_healing(false).await;
23557 ensure_decommission_unresolved_verification_disk_count(self.set_drive_count, disks.len(), pool_index, entry.set_index)?;
23558
23559 let object = encode_dir_object(&entry.object);
23560 let reads = join_all(disks.iter().map(|disk| disk.read_xl(&entry.bucket, &object, false))).await;
23561 let mut found = false;
23562 for read in reads {
23563 match read {
23564 Ok(_) => found = true,
23565 Err(DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound) => {}
23566 Err(err) => {
23567 return Err(Error::other(format!(
23568 "decommission unresolved-entry verification failed for pool {pool_index} set {} path {}/{}: {err}",
23569 entry.set_index, entry.bucket, entry.object
23570 )));
23571 }
23572 }
23573 }
23574 Ok(!found)
23575 }
23576
23577 #[tracing::instrument(skip(self, store, rx, cb_func, entry_error))]
23578 #[allow(clippy::too_many_arguments)]
23579 async fn list_objects_to_decommission(
23580 self: &Arc<Self>,
23581 store: Arc<ECStore>,
23582 rx: CancellationToken,
23583 bucket_info: DecomBucketInfo,
23584 cb_func: ListCallback,
23585 entry_error: Arc<tokio::sync::Mutex<Option<Error>>>,
23586 pool_index: usize,
23587 set_index: usize,
23588 source_generation: OffsetDateTime,
23589 require_all_disks: bool,
23590 ) -> Result<()> {
23591 let (disks, _) = self.get_online_disks_with_healing(false).await;
23592 ensure_decommission_listing_disks_available(!disks.is_empty(), &bucket_info.name)?;
23593 if require_all_disks {
23594 ensure_decommission_unresolved_verification_disk_count(self.set_drive_count, disks.len(), pool_index, set_index)?;
23595 }
23596
23597 let listing_quorum = self.set_drive_count.div_ceil(2);
23598
23599 let resolver = MetadataResolutionParams {
23600 dir_quorum: listing_quorum,
23601 obj_quorum: listing_quorum,
23602 bucket: bucket_info.name.clone(),
23603 ..Default::default()
23604 };
23605
23606 let cb1 = cb_func.clone();
23607 let unresolved_error = entry_error.clone();
23608 let unresolved_rx = rx.clone();
23609 let unresolved_bucket = bucket_info.name.clone();
23610 let unresolved_prefix = bucket_info.prefix.clone();
23611 let unresolved_pool_index = pool_index;
23612 let unresolved_set_index = set_index;
23613 let unresolved_generation = source_generation;
23614 let unresolved_store = store;
23615
23616 list_path_raw(
23617 rx,
23618 ListPathRawOptions {
23619 disks: disks.iter().cloned().map(Some).collect(),
23620 bucket: bucket_info.name.clone(),
23621 path: bucket_info.prefix.clone(),
23622 recursive: true,
23623 min_disks: listing_quorum,
23624 skip_walkdir_total_timeout: true,
23625 walkdir_stall_timeout: Some(DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT),
23626 agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
23627 partial: Some(Box::new(move |entries: MetaCacheEntries, errs: &[Option<DiskError>]| {
23628 let resolver = resolver.clone();
23629 let cb_func = cb_func.clone();
23630 let bucket = unresolved_bucket.clone();
23631 let prefix = unresolved_prefix.clone();
23632 let unresolved_error = unresolved_error.clone();
23633 let unresolved_rx = unresolved_rx.clone();
23634 let unresolved_store = unresolved_store.clone();
23635 let pool_index = unresolved_pool_index;
23636 let set_index = unresolved_set_index;
23637 let source_generation = unresolved_generation;
23638 let disk_error_count = errs.iter().flatten().count();
23639 if unresolved_rx.is_cancelled() {
23640 return Box::pin(async {});
23641 }
23642
23643 match resolve_decommission_partial_listing_entry(
23644 entries,
23645 resolver,
23646 &bucket,
23647 &prefix,
23648 disk_error_count,
23649 pool_index,
23650 set_index,
23651 source_generation,
23652 ) {
23653 Ok(entry) => {
23654 warn!("decommission_pool: list_objects_to_decommission get {}", &entry.name);
23655 Box::pin(async move {
23656 cb_func(entry).await;
23657 })
23658 }
23659 Err(unresolved_entry) => Box::pin(async move {
23660 if unresolved_rx.is_cancelled() {
23661 return;
23662 }
23663 let err = decommission_unresolved_listing_error(&unresolved_entry);
23664 warn!(
23665 event = EVENT_DECOMMISSION_BUCKET,
23666 component = LOG_COMPONENT_ECSTORE,
23667 subsystem = LOG_SUBSYSTEM_POOLS,
23668 bucket = %bucket,
23669 prefix = %prefix,
23670 state = "unresolved_entry",
23671 error = %err,
23672 "Decommission listing failed closed on unresolved metadata"
23673 );
23674 let err = match unresolved_store
23675 .persist_decommission_unresolved_entry(pool_index, source_generation, unresolved_entry)
23676 .await
23677 {
23678 Ok(()) => err,
23679 Err(ledger_err) => Error::other(format!("{err}; {ledger_err}")),
23680 };
23681 record_decommission_entry_error(&unresolved_error, &unresolved_rx, err).await;
23682 }),
23683 }
23684 })),
23685 ..Default::default()
23686 },
23687 )
23688 .await?;
23689
23690 if let Some(err) = entry_error.lock().await.clone() {
23691 return Err(err);
23692 }
23693
23694 Ok(())
23695 }
23696}
23697
23698fn is_disk_online_state(state: &str) -> bool {
23699 let s = state.trim().to_lowercase();
23704 if s.is_empty() {
23705 return true;
23706 }
23707 if s.contains("offline") {
23708 return false;
23709 }
23710 if s.contains("not found") || s.contains("disk not found") {
23711 return false;
23712 }
23713 true
23714}
23715
23716fn decommission_physical_pool_capacity(
23717 disks: &[rustfs_madmin::Disk],
23718 pool_index: usize,
23719 layout: DecommissionErasureLayout,
23720 logical: PoolSpaceInfo,
23721 owned_physical_used: Option<&HashMap<i32, usize>>,
23722) -> (usize, usize, usize) {
23723 let width = layout.width();
23724 let mut sets: HashMap<i32, Vec<&rustfs_madmin::Disk>> = HashMap::new();
23725 let mut seen = HashSet::new();
23726 for disk in disks {
23727 let state = disk.state.trim().to_ascii_lowercase();
23728 if disk.pool_index != pool_index as i32
23729 || disk.set_index < 0
23730 || disk.disk_index < 0
23731 || disk.disk_index as usize >= width
23732 || !matches!(state.as_str(), "ok" | "online")
23733 {
23734 continue;
23735 }
23736 let identity = (disk.set_index, disk.disk_index);
23737 if seen.insert(identity) {
23738 sets.entry(disk.set_index).or_default().push(disk);
23739 }
23740 }
23741
23742 let mut physical_total = 0usize;
23743 let mut physical_free = 0usize;
23744 let mut physical_used = 0usize;
23745 let mut observed_set = false;
23746 for set_disks in sets.values() {
23747 if set_disks.is_empty() {
23748 continue;
23749 }
23750 observed_set = true;
23751 let min_total = set_disks
23752 .iter()
23753 .map(|disk| disk.total_space as usize)
23754 .min()
23755 .unwrap_or_default();
23756 let min_free = set_disks
23757 .iter()
23758 .map(|disk| disk.available_space as usize)
23759 .min()
23760 .unwrap_or_default();
23761 let physical_used_for_set = owned_physical_used
23762 .and_then(|owned| owned.get(&set_disks[0].set_index).copied())
23763 .unwrap_or_else(|| {
23764 set_disks
23765 .iter()
23766 .map(|disk| {
23767 (disk.used_space as usize).max((disk.total_space as usize).saturating_sub(disk.available_space as usize))
23768 })
23769 .max()
23770 .unwrap_or_default()
23771 .saturating_mul(width)
23772 });
23773 physical_total = physical_total.saturating_add(min_total.saturating_mul(width));
23774 physical_used = physical_used.saturating_add(physical_used_for_set);
23775 if set_disks.len() >= width {
23776 physical_free = physical_free.saturating_add(min_free.saturating_mul(width));
23777 }
23778 }
23779
23780 if observed_set {
23781 return (physical_total, physical_free, physical_used);
23782 }
23783
23784 let fallback_total = capacity_mul_div_ceil(logical.total, width, layout.data);
23785 let fallback_used = capacity_mul_div_ceil(logical.used, width, layout.data)
23786 .max(fallback_total.saturating_sub(capacity_mul_div_ceil(logical.free, width, layout.data)));
23787 (fallback_total, 0, fallback_used)
23788}
23789
23790#[deprecated(since = "0.1.0", note = "Use fallback_total_capacity_dedup instead")]
23791#[allow(
23792 dead_code,
23793 reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
23794)]
23795fn fallback_total_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
23796 fallback_total_capacity_dedup(disks)
23797}
23798
23799#[deprecated(since = "0.1.0", note = "Use fallback_free_capacity_dedup instead")]
23800#[allow(
23801 dead_code,
23802 reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
23803)]
23804fn fallback_free_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
23805 fallback_free_capacity_dedup(disks)
23806}
23807
23808pub fn get_total_usable_capacity(disks: &[rustfs_madmin::Disk], info: &rustfs_madmin::StorageInfo) -> usize {
23809 if info.backend.standard_sc_data.is_empty() {
23811 return fallback_total_capacity_dedup(disks);
23812 }
23813 let mut capacity = 0usize;
23814 let mut matched_any = false;
23815 let mut counted_disks: HashSet<String> = HashSet::new();
23816
23817 for disk in disks.iter() {
23818 if disk.pool_index < 0 {
23819 continue;
23820 }
23821 let pool_idx = disk.pool_index as usize;
23822 if info.backend.standard_sc_data.len() <= pool_idx {
23823 continue;
23824 }
23825
23826 let usable_disks_per_set = info.backend.standard_sc_data[pool_idx];
23827 if usable_disks_per_set == 0 {
23828 continue;
23829 }
23830
23831 if (disk.disk_index as usize) < usable_disks_per_set {
23832 let disk_key = format!(
23834 "{}|{}|p{}s{}d{}",
23835 disk.endpoint, disk.drive_path, disk.pool_index, disk.set_index, disk.disk_index );
23841 debug!("get_total_usable_capacity disk_key: {}", disk_key);
23842 if counted_disks.insert(disk_key) {
23844 matched_any = true;
23845 capacity += disk.total_space as usize;
23846 } else {
23847 warn!(
23849 "Duplicate disk detected in capacity calculation: {} at {}",
23850 disk.endpoint, disk.drive_path
23851 );
23852 }
23853 }
23854 }
23855
23856 if matched_any {
23857 capacity
23858 } else {
23859 fallback_total_capacity_dedup(disks)
23862 }
23863}
23864
23865pub fn get_total_usable_capacity_free(disks: &[rustfs_madmin::Disk], info: &rustfs_madmin::StorageInfo) -> usize {
23866 if info.backend.standard_sc_data.is_empty() {
23867 return fallback_free_capacity_dedup(disks);
23868 }
23869
23870 let mut capacity = 0usize;
23871 let mut matched_any = false;
23872 let mut counted_disks: HashSet<String> = HashSet::new();
23873
23874 for disk in disks.iter() {
23875 if disk.pool_index < 0 {
23876 continue;
23877 }
23878 let pool_idx = disk.pool_index as usize;
23879 if info.backend.standard_sc_data.len() <= pool_idx {
23880 continue;
23881 }
23882
23883 let usable_disks_per_set = info.backend.standard_sc_data[pool_idx];
23884 if usable_disks_per_set == 0 {
23885 continue;
23886 }
23887
23888 if (disk.disk_index as usize) < usable_disks_per_set {
23889 let disk_key = format!(
23890 "{}|{}|p{}s{}d{}",
23891 disk.endpoint, disk.drive_path, disk.pool_index, disk.set_index, disk.disk_index
23892 );
23893
23894 if counted_disks.insert(disk_key) {
23895 matched_any = true;
23896 capacity += disk.available_space as usize;
23897 }
23898 }
23899 }
23900
23901 if matched_any {
23902 capacity
23903 } else {
23904 fallback_free_capacity_dedup(disks)
23905 }
23906}
23907
23908pub(crate) fn fallback_total_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usize {
23912 let mut counted_disks: HashSet<String> = HashSet::new();
23913 let mut total = 0usize;
23914
23915 for disk in disks.iter() {
23916 if !is_disk_online_state(&disk.state) {
23918 continue;
23919 }
23920
23921 let disk_key = format!("{}|{}", disk.endpoint, disk.drive_path);
23923
23924 if counted_disks.insert(disk_key) {
23926 total += disk.total_space as usize;
23927 }
23928 }
23929
23930 total
23931}
23932
23933pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usize {
23937 let mut counted_disks: HashSet<String> = HashSet::new();
23938 let mut total = 0usize;
23939
23940 for disk in disks.iter() {
23941 if !is_disk_online_state(&disk.state) {
23942 continue;
23943 }
23944
23945 let disk_key = format!("{}|{}", disk.endpoint, disk.drive_path);
23946
23947 if counted_disks.insert(disk_key) {
23948 total += disk.available_space as usize;
23949 }
23950 }
23951
23952 total
23953}
23954
23955#[cfg(test)]
23956mod pools_tests {
23957 use std::collections::HashMap;
23958
23959 use super::DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
23960 use super::persist_v3_pool_meta_for_test;
23961 use super::record_decommission_entry_error;
23962 use super::resolve_decommission_listing_error;
23963 use super::resolve_decommission_partial_listing_entry;
23964 use super::{
23965 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, DECOMMISSION_CAPACITY_MODEL_VERSION, DECOMMISSION_CAPACITY_RELEASE_CANCELED,
23966 DECOMMISSION_CAPACITY_RELEASE_COMPLETED, DECOMMISSION_CAPACITY_RELEASE_FAILED, DECOMMISSION_CAPACITY_RESERVATION_TTL,
23967 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
23968 DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP, DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP, DECOMMISSION_ENTRY_QUEUE_HARD_CAP,
23969 DECOMMISSION_META_PREFIXES, DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
23970 DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT, DecomBucketInfo, DecommissionCanceler, DecommissionCapacityTarget,
23971 DecommissionDurableIlmReceipt, DecommissionEntryEnqueueResult, DecommissionErasureLayout, DecommissionPoolCapacityInfo,
23972 DecommissionStartPoolState, DecommissionTargetConsumption, DecommissionTerminalState, DecommissionUnresolvedEntry,
23973 ListCallback, POOL_META_GENERATION_VERSION, POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_V1_VERSION,
23974 POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolMetaCasToken, PoolMetaPersistenceFence, PoolSpaceInfo, PoolStatus,
23975 QueuedDecommissionEntry, REBAL_META_NAME, acquire_pool_rebalance_activation_locks, active_decommission_source_indices,
23976 apply_decommission_status_space_info, await_decommission_worker, bind_decommission_cancelers,
23977 bind_missing_decommission_cancelers, build_decommission_capacity_reservation,
23978 build_decommission_capacity_reservation_with_model, cancel_decommission_canceler, clamp_decommission_entry_concurrency,
23979 classify_decommission_terminal_state, count_decommission_item, decommission_cancel_signal_result,
23980 decommission_durable_ilm_receipt_path, decommission_durable_ilm_receipt_run_prefix,
23981 decommission_durable_ilm_receipt_run_token, decommission_entry_queue_capacity, decommission_item_size,
23982 decommission_meta_bucket_options, decommission_physical_pool_capacity, decommission_retry_backoff_ceiling,
23983 decommission_retry_backoff_delay, decommission_start_pool_state, decommission_unresolved_listing_error, dedup_indices,
23984 default_decommission_bucket_concurrency, default_decommission_entry_concurrency, drain_decommission_entry_queue,
23985 enqueue_decommission_entry, ensure_decommission_cancel_allowed, ensure_decommission_capacity_reservations_available,
23986 ensure_decommission_clear_allowed, ensure_decommission_generation, ensure_decommission_listing_disks_available,
23987 ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool,
23988 ensure_decommission_start_local_leader, ensure_decommission_start_pool_states,
23989 ensure_decommission_start_rebalance_meta_allowed, ensure_decommission_start_target_capacity,
23990 ensure_decommission_terminal_operation_supported, ensure_decommission_unresolved_verification_disk_count,
23991 ensure_local_decommission_pool_leaders, ensure_pool_meta_write_fence, ensure_valid_decommission_pool_index, get_by_index,
23992 grow_decommission_target_reservation, guard_decommission_cancelers, has_active_decommission_canceler,
23993 is_decommission_active, is_decommission_cancel_requested, load_decommission_entry_versions,
23994 local_decommission_queue_prefix, mark_decommission_bucket_done, merge_decommission_durable_ilm_receipts,
23995 merge_pool_meta_updates_for_save, merge_pool_status_refresh, missing_decommission_worker_prefix,
23996 next_decommission_capacity_generation, observe_decommission_terminal_reload_result,
23997 parse_decommission_durable_ilm_receipt_path, pool_meta_has_active_decommission, publish_pool_meta_updates,
23998 read_pool_meta_replica, reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion,
23999 record_decommission_unresolved_entry, recover_decommission_capacity_reservations,
24000 renew_decommission_capacity_reservation, require_decommission_store, reserve_decommission_start_cancelers,
24001 reserve_decommission_start_target_capacity, resolve_decommission_bucket_state,
24002 resolve_decommission_check_after_list_result, resolve_decommission_entry_cleanup_delete_result,
24003 resolve_decommission_entry_exact_versions, resolve_decommission_entry_reload_result,
24004 resolve_decommission_listing_worker_result, resolve_decommission_optional_bucket_config_result,
24005 resolve_decommission_pool_meta_reload_result, resolve_decommission_preflight_heal_result,
24006 resolve_decommission_progress_save_result, resolve_decommission_terminal_mark_after_error_result,
24007 resolve_decommission_terminal_mark_result, resolve_decommission_update_after_result,
24008 resolve_start_decommission_pool_meta_reload_result, resumable_decommission_queue_indices,
24009 rollback_start_decommission_pool_meta, run_decommission_buckets_bounded, run_decommission_listing_with_retry,
24010 run_decommission_listing_with_retry_and_drain, run_decommission_phases, run_decommission_side_effect,
24011 save_pool_meta_object_cas, select_decommission_capacity_model, should_cleanup_decommission_source_entry,
24012 should_continue_decommission_queue, should_count_decommission_version_complete,
24013 should_fail_decommission_pool_after_exhausted_source_changed, should_preserve_decommission_canceled_state,
24014 should_reject_decommission_cancel_as_terminal, should_retry_decommission_cancel_reload,
24015 should_retry_decommission_listing, should_skip_canceled_decommission_routine, spawn_decommission_index_cancelers,
24016 split_decommission_buckets, take_and_cancel_decommission_canceler, take_decommission_canceler,
24017 track_decommission_current_object, track_decommission_current_object_stage, update_decommission_for_operation,
24018 validate_start_decommission_request, wait_decommission_retry_backoff, wait_decommission_worker_drain,
24019 with_decommission_entry_context,
24020 };
24021 use super::{
24022 DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS, DECOMMISSION_RETRY_BACKOFF_CAP, DecommissionCapacityAdmission,
24023 DecommissionCapacityOwner, DecommissionCapacityReleaseProof, DecommissionCapacityReservation,
24024 DecommissionCapacityTemporaryMutation, decommission_capacity_mutation_id,
24025 decommission_capacity_target_gate_retry_exhausted, ensure_decommission_target_owner_admission,
24026 ensure_exact_delete_capacity_namespace_fences, ensure_external_decommission_target_admission,
24027 is_decommission_capacity_blocked_error, plan_exact_delete_capacity_reconciliations,
24028 record_decommission_target_consumption, release_decommission_target_inflight, reserve_decommission_target_pending,
24029 resolve_decommission_target_pending, set_decommission_capacity_info_overrides_for_test,
24030 };
24031 use crate::bucket::lifecycle::{
24032 DurableIlmRecordCheckpoint,
24033 bucket_lifecycle_ops::{ManualTransitionQueueSnapshot, ManualTransitionRunOptions},
24034 manual_transition_job::{ManualTransitionJobRecord, manual_transition_job_record_object_name},
24035 validate_durable_ilm_record,
24036 };
24037 use crate::bucket::metadata_sys;
24038 use crate::data_movement;
24039 use crate::disk::{STORAGE_FORMAT_FILE, endpoint::Endpoint};
24040 use crate::error::{Error, StorageError};
24041 use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
24042 use crate::object_api::{ObjectInfo, ObjectOptions};
24043 use crate::runtime::instance::InstanceContext;
24044 use crate::services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats};
24045 use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
24046 use crate::storage_api_contracts::object::HTTPPreconditions;
24047 use crate::storage_api_contracts::{object::ObjectIO, range::HTTPRangeSpec};
24048 use crate::store::ECStore;
24049 use byteorder::{ByteOrder, LittleEndian};
24050 use rmp_serde::Serializer;
24051 use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntry, ObjectPartInfo};
24052 use rustfs_filemeta::{MetaCacheEntries, MetadataResolutionParams};
24053 use rustfs_lock::{GlobalLockManager, LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
24054 use rustfs_rio::Index;
24055 use serde::Serialize;
24056 use std::future::Future;
24057 use std::io::Cursor;
24058 use std::sync::{
24059 Arc, Mutex as StdMutex,
24060 atomic::{AtomicBool, AtomicUsize, Ordering},
24061 };
24062 use std::task::{Context, Poll};
24063 use std::time::Duration as StdDuration;
24064 use time::{Duration, OffsetDateTime};
24065 use tokio::io::AsyncReadExt;
24066 use tokio::sync::Semaphore;
24067 use tokio_util::sync::CancellationToken;
24068
24069 #[derive(Debug)]
24070 struct ActivationLockRecorder {
24071 lock_manager: Arc<rustfs_lock::GlobalLockManager>,
24072 owner: &'static str,
24073 resources: StdMutex<Vec<String>>,
24074 }
24075
24076 #[async_trait::async_trait]
24077 impl crate::storage_api_contracts::namespace::NamespaceLocking for ActivationLockRecorder {
24078 type Error = Error;
24079 type NamespaceLock = rustfs_lock::NamespaceLockWrapper;
24080
24081 async fn new_ns_lock(&self, bucket: &str, object: &str) -> crate::error::Result<Self::NamespaceLock> {
24082 self.resources
24083 .lock()
24084 .expect("activation lock recorder should not be poisoned")
24085 .push(object.to_string());
24086 Ok(rustfs_lock::NamespaceLockWrapper::new(
24087 rustfs_lock::NamespaceLock::with_local_manager(
24088 "activation-lock-test".to_string(),
24089 Arc::clone(&self.lock_manager),
24090 ),
24091 rustfs_lock::ObjectKey::new(bucket, object),
24092 self.owner.to_string(),
24093 ))
24094 }
24095 }
24096
24097 fn noop_decommission_list_callback() -> ListCallback {
24098 Arc::new(|_| Box::pin(async {}))
24099 }
24100
24101 fn decommission_worker_test_store(pool_meta: PoolMeta, cancelers: Vec<Option<DecommissionCanceler>>) -> Arc<ECStore> {
24102 let ctx = Arc::new(InstanceContext::new());
24103 let endpoint_pools = EndpointServerPools::default();
24104 Arc::new(ECStore {
24105 id: uuid::Uuid::new_v4(),
24106 disk_map: std::collections::HashMap::new(),
24107 pools: Vec::new(),
24108 peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
24109 pool_meta: tokio::sync::RwLock::new(pool_meta),
24110 rebalance_meta: tokio::sync::RwLock::new(None),
24111 decommission_cancelers: tokio::sync::RwLock::new(cancelers),
24112 start_gate: tokio::sync::Mutex::new(()),
24113 pool_meta_save_gate: tokio::sync::Mutex::new(super::PoolMetaWriteState::for_test_bootstrap()),
24114 ctx,
24115 bucket_fence_registry: Arc::default(),
24116 })
24117 }
24118
24119 #[derive(Debug, Default)]
24120 struct PartialPoolMetaWriteStorage {
24121 fail_write: bool,
24122 fail_after_first_write: bool,
24123 pending_write: bool,
24124 write_started: tokio::sync::Notify,
24125 wrote: AtomicBool,
24126 revision: AtomicUsize,
24127 stored: StdMutex<Option<(Vec<u8>, String)>>,
24128 identity: StdMutex<Option<(Vec<u8>, String)>>,
24129 }
24130
24131 #[test]
24132 fn decommission_target_gate_retry_is_bounded() {
24133 assert!(!decommission_capacity_target_gate_retry_exhausted(0));
24134 assert!(!decommission_capacity_target_gate_retry_exhausted(
24135 DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS - 2
24136 ));
24137 assert!(decommission_capacity_target_gate_retry_exhausted(
24138 DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS - 1
24139 ));
24140 }
24141
24142 #[tokio::test]
24143 async fn pool_meta_recovery_reconciles_prepare_and_partial_commit_without_format_downgrade() {
24144 for previous_version in [POOL_META_V1_VERSION, POOL_META_VERSION, POOL_META_GENERATION_VERSION] {
24145 for commit_first in [false, true] {
24146 let cluster_id = uuid::Uuid::new_v4();
24147 let previous_meta = PoolMeta {
24148 version: previous_version,
24149 pools: vec![decommission_test_pool_status(0, None)],
24150 ..Default::default()
24151 };
24152 let previous_revision = if previous_version == POOL_META_GENERATION_VERSION {
24153 super::PoolMetaRevision {
24154 version: previous_version,
24155 cluster_id: Some(cluster_id),
24156 epoch: 1,
24157 generation: 9,
24158 transaction_id: Some(uuid::Uuid::new_v4()),
24159 }
24160 } else {
24161 super::PoolMetaRevision::legacy(previous_version)
24162 };
24163 let previous_data = if previous_revision.is_generation_protocol() {
24164 super::encode_pool_meta_v3_envelope(&previous_meta, previous_revision, true, None).unwrap()
24165 } else {
24166 previous_meta.encode_config_data_for_v2_gate(true).unwrap()
24167 };
24168 let previous = super::PoolMetaCommittedCandidate {
24169 canonical: previous_data,
24170 meta: previous_meta.clone(),
24171 revision: previous_revision,
24172 };
24173 let mut next = previous_meta.clone();
24174 next.version = POOL_META_GENERATION_VERSION;
24175 next.pools[0].last_update += Duration::seconds(1);
24176 let revision = super::PoolMetaRevision {
24177 version: POOL_META_GENERATION_VERSION,
24178 cluster_id: Some(cluster_id),
24179 epoch: 1,
24180 generation: previous_revision.generation + 1,
24181 transaction_id: Some(uuid::Uuid::new_v4()),
24182 };
24183 let pending = super::encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous)).unwrap();
24184 let durable = super::encode_pool_meta_v3_envelope(&next, revision, true, None).unwrap();
24185 let identity = super::initialized_pool_meta_identity_for_test(cluster_id, 1).unwrap();
24186 let pools = [if commit_first { durable } else { pending.clone() }, pending]
24187 .into_iter()
24188 .map(|data| {
24189 Arc::new(PartialPoolMetaWriteStorage {
24190 stored: StdMutex::new(Some((data, "initial".to_owned()))),
24191 identity: StdMutex::new(Some((identity.clone(), "identity".to_owned()))),
24192 ..Default::default()
24193 })
24194 })
24195 .collect::<Vec<_>>();
24196 let mut state = super::PoolMetaWriteState::for_startup(cluster_id, false);
24197 let selection = super::load_pool_meta_for_transaction_recovery(pools.clone(), &mut state)
24198 .await
24199 .unwrap();
24200 assert_eq!(
24201 selection.meta.pools[0].last_update,
24202 if commit_first {
24203 next.pools[0].last_update
24204 } else {
24205 previous_meta.pools[0].last_update
24206 }
24207 );
24208 let outcome = super::repair_pool_meta_transaction(
24209 pools.clone(),
24210 &mut state,
24211 selection,
24212 &PoolMetaPersistenceFence::Distributed(None),
24213 )
24214 .await
24215 .unwrap();
24216 assert_eq!(outcome.committed.version, POOL_META_GENERATION_VERSION);
24217 assert_eq!(
24218 outcome.committed.pools[0].last_update,
24219 if commit_first {
24220 next.pools[0].last_update
24221 } else {
24222 previous_meta.pools[0].last_update
24223 }
24224 );
24225 outcome.disarm();
24226 let confirmed = super::load_pool_meta_for_transaction_recovery(pools, &mut state)
24227 .await
24228 .unwrap();
24229 assert!(!confirmed.replica_state.needs_repair);
24230 state.ensure_write_safe("repaired").unwrap();
24231 }
24232 }
24233 }
24234
24235 #[tokio::test]
24236 async fn pool_meta_recovery_rejects_missing_corrupt_and_incompatible_replicas_without_writes() {
24237 let cluster_id = uuid::Uuid::new_v4();
24238 let identity = super::initialized_pool_meta_identity_for_test(cluster_id, 1).unwrap();
24239 for payload in [None, Some(Vec::new()), Some(vec![1, 0, 255, 255, 0])] {
24240 let pool = Arc::new(PartialPoolMetaWriteStorage {
24241 stored: StdMutex::new(payload.map(|data| (data, "initial".to_owned()))),
24242 identity: StdMutex::new(Some((identity.clone(), "identity".to_owned()))),
24243 ..Default::default()
24244 });
24245 let mut state = super::PoolMetaWriteState::for_startup(cluster_id, false);
24246 super::load_pool_meta_for_transaction_recovery(vec![pool.clone()], &mut state)
24247 .await
24248 .unwrap_err();
24249 assert!(!pool.wrote.load(Ordering::SeqCst));
24250 }
24251 }
24252
24253 #[tokio::test]
24254 async fn pool_meta_first_cas_rejection_is_retryable_but_later_rejection_keeps_prior_write_armed() {
24255 let pool = Arc::new(PartialPoolMetaWriteStorage::default());
24256 let state = super::PoolMetaWriteState::default();
24257 let fence = PoolMetaPersistenceFence::Distributed(None);
24258 let mut arm = state.arm_transaction();
24259 save_pool_meta_object_cas(
24260 pool.clone(),
24261 POOL_META_NAME,
24262 vec![1],
24263 &PoolMetaCasToken::Existing("missing".to_owned()),
24264 &fence,
24265 "prepare_cas",
24266 &mut arm,
24267 )
24268 .await
24269 .unwrap_err();
24270 drop(arm);
24271 state.ensure_write_safe("first rejection").unwrap();
24272 let mut arm = state.arm_transaction();
24273 save_pool_meta_object_cas(
24274 pool.clone(),
24275 POOL_META_NAME,
24276 vec![1],
24277 &PoolMetaCasToken::Missing,
24278 &fence,
24279 "prepare_cas",
24280 &mut arm,
24281 )
24282 .await
24283 .unwrap();
24284 save_pool_meta_object_cas(pool, POOL_META_NAME, vec![2], &PoolMetaCasToken::Missing, &fence, "commit_cas", &mut arm)
24285 .await
24286 .unwrap_err();
24287 drop(arm);
24288 let err = state.ensure_write_safe("later rejection").unwrap_err();
24289 assert_eq!(err.pool_metadata_failure().unwrap().phase, "prepare_cas");
24290 assert!(matches!(
24291 err.pool_metadata_failure().unwrap().source.as_deref(),
24292 Some(Error::PreconditionFailed)
24293 ));
24294 }
24295
24296 #[tokio::test]
24297 async fn pool_meta_recovery_cas_conflict_after_repair_write_keeps_transaction_blocked() {
24298 let base = PoolMeta {
24299 version: POOL_META_VERSION,
24300 pools: vec![decommission_test_pool_status(0, None)],
24301 ..Default::default()
24302 };
24303 let mut current = base.clone();
24304 current.pools[0].last_update += Duration::seconds(1);
24305 let first = Arc::new(PartialPoolMetaWriteStorage {
24306 stored: StdMutex::new(Some((current.encode_config_data_for_v2_gate(true).unwrap(), "first".to_owned()))),
24307 ..Default::default()
24308 });
24309 let second = Arc::new(PartialPoolMetaWriteStorage {
24310 stored: StdMutex::new(Some((base.encode_config_data_for_v2_gate(true).unwrap(), "second".to_owned()))),
24311 ..Default::default()
24312 });
24313 let mut state = super::PoolMetaWriteState::default();
24314 let selection = super::load_pool_meta_for_transaction_recovery(vec![first.clone(), second.clone()], &mut state)
24315 .await
24316 .unwrap();
24317 let raced = base.encode_config_data_for_v2_gate(true).unwrap();
24318 *second.stored.lock().unwrap() = Some((raced.clone(), "raced".to_owned()));
24319 let result = super::repair_pool_meta_transaction(
24320 vec![first.clone(), second.clone()],
24321 &mut state,
24322 selection,
24323 &PoolMetaPersistenceFence::Distributed(None),
24324 )
24325 .await;
24326 assert!(matches!(result, Err(Error::PreconditionFailed)));
24327 assert!(first.wrote.load(Ordering::SeqCst));
24328 assert!(!second.wrote.load(Ordering::SeqCst));
24329 assert_eq!(second.stored.lock().unwrap().as_ref().unwrap().0, raced);
24330 let err = state.ensure_write_safe("recovery retry").unwrap_err();
24331 assert_eq!(err.pool_metadata_failure().unwrap().phase, "recovery_cas");
24332 }
24333
24334 #[tokio::test]
24335 async fn pool_meta_cas_failure_retains_the_original_source_for_caller_and_latch() {
24336 let pool = Arc::new(PartialPoolMetaWriteStorage {
24337 fail_write: true,
24338 ..Default::default()
24339 });
24340 let state = super::PoolMetaWriteState::default();
24341 let mut arm = state.arm_transaction();
24342 let err = save_pool_meta_object_cas(
24343 pool,
24344 POOL_META_NAME,
24345 Vec::new(),
24346 &PoolMetaCasToken::Missing,
24347 &PoolMetaPersistenceFence::Distributed(None),
24348 "commit_cas",
24349 &mut arm,
24350 )
24351 .await
24352 .unwrap_err();
24353 let source = err.pool_metadata_failure().unwrap().source.as_ref().unwrap();
24354 assert!(matches!(source.as_ref(), Error::Timeout));
24355 assert!(Arc::ptr_eq(source, arm.source.as_ref().unwrap()));
24356 drop(arm);
24357 let blocked = state.ensure_write_safe("after CAS failure").unwrap_err();
24358 assert!(Arc::ptr_eq(source, blocked.pool_metadata_failure().unwrap().source.as_ref().unwrap()));
24359 }
24360
24361 #[tokio::test]
24362 async fn pool_meta_cancelled_identity_dispatch_is_armed_before_pool_bin_writes() {
24363 let pool = Arc::new(PartialPoolMetaWriteStorage {
24364 pending_write: true,
24365 ..Default::default()
24366 });
24367 let state = Arc::new(tokio::sync::Mutex::new(super::PoolMetaWriteState::for_startup(
24368 uuid::Uuid::new_v4(),
24369 true,
24370 )));
24371 let task = tokio::spawn({
24372 let pool = pool.clone();
24373 let state = state.clone();
24374 async move {
24375 let mut state = state.lock().await;
24376 super::persist_pool_meta_identity_for_startup(vec![pool], &mut state, false).await
24377 }
24378 });
24379 pool.write_started.notified().await;
24380 task.abort();
24381 assert!(task.await.unwrap_err().is_cancelled());
24382 let state = state.lock().await;
24383 let err = state.ensure_write_safe("identity retry").unwrap_err();
24384 assert_eq!(err.pool_metadata_failure().unwrap().phase, "identity_cas");
24385 assert_eq!(
24386 err.pool_metadata_failure().unwrap().kind,
24387 crate::error::PoolMetadataFailure::TransactionUnknown
24388 );
24389 assert!(pool.stored.lock().unwrap().is_none());
24390 }
24391
24392 #[async_trait::async_trait]
24393 impl ObjectIO for PartialPoolMetaWriteStorage {
24394 type Error = Error;
24395 type RangeSpec = HTTPRangeSpec;
24396 type HeaderMap = http::HeaderMap;
24397 type ObjectOptions = crate::object_api::ObjectOptions;
24398 type ObjectInfo = crate::object_api::ObjectInfo;
24399 type GetObjectReader = crate::object_api::GetObjectReader;
24400 type PutObjectReader = crate::object_api::PutObjReader;
24401
24402 async fn get_object_reader(
24403 &self,
24404 bucket: &str,
24405 object: &str,
24406 _range: Option<Self::RangeSpec>,
24407 _h: Self::HeaderMap,
24408 _opts: &Self::ObjectOptions,
24409 ) -> std::result::Result<Self::GetObjectReader, Error> {
24410 let stored = if object == POOL_META_IDENTITY_NAME {
24411 &self.identity
24412 } else {
24413 &self.stored
24414 };
24415 let Some((data, etag)) = stored
24416 .lock()
24417 .expect("pool metadata test storage should not be poisoned")
24418 .clone()
24419 else {
24420 return Err(Error::FileNotFound);
24421 };
24422 Ok(crate::object_api::GetObjectReader {
24423 stream: Box::new(Cursor::new(data.clone())),
24424 object_info: crate::object_api::ObjectInfo {
24425 bucket: bucket.to_string(),
24426 name: object.to_string(),
24427 size: data.len() as i64,
24428 etag: Some(etag),
24429 ..Default::default()
24430 },
24431 buffered_body: None,
24432 body_source: Default::default(),
24433 })
24434 }
24435
24436 async fn put_object(
24437 &self,
24438 _bucket: &str,
24439 object: &str,
24440 data: &mut Self::PutObjectReader,
24441 opts: &Self::ObjectOptions,
24442 ) -> std::result::Result<Self::ObjectInfo, Error> {
24443 self.write_started.notify_one();
24444 if self.pending_write {
24445 std::future::pending().await
24446 }
24447 if object == POOL_META_NAME
24448 && (self.fail_write || (self.fail_after_first_write && self.revision.load(Ordering::SeqCst) > 0))
24449 {
24450 return Err(Error::Timeout);
24451 }
24452 let stored = if object == POOL_META_IDENTITY_NAME {
24453 &self.identity
24454 } else {
24455 &self.stored
24456 };
24457 let current_etag = stored
24458 .lock()
24459 .expect("pool metadata test storage should not be poisoned")
24460 .as_ref()
24461 .map(|(_, etag)| etag.clone());
24462 if opts
24463 .http_preconditions
24464 .as_ref()
24465 .and_then(HTTPPreconditions::if_none_match_value)
24466 == Some("*")
24467 && current_etag.is_some()
24468 {
24469 return Err(Error::PreconditionFailed);
24470 }
24471 if let Some(expected) = opts.http_preconditions.as_ref().and_then(HTTPPreconditions::if_match_value)
24472 && current_etag.as_deref() != Some(expected)
24473 {
24474 return Err(Error::PreconditionFailed);
24475 }
24476 let mut payload = Vec::new();
24477 data.stream.read_to_end(&mut payload).await?;
24478 let etag = format!("pool-meta-test-{}", self.revision.fetch_add(1, Ordering::SeqCst) + 1);
24479 *stored.lock().expect("pool metadata test storage should not be poisoned") = Some((payload, etag.clone()));
24480 self.wrote.store(true, Ordering::SeqCst);
24481 Ok(crate::object_api::ObjectInfo {
24482 etag: Some(etag),
24483 ..Default::default()
24484 })
24485 }
24486 }
24487
24488 mod phase_tests {
24489 use super::super as implementation;
24490 use super::*;
24491
24492 #[derive(Clone, Copy, Debug, Default)]
24493 enum WriteFault {
24494 #[default]
24495 None,
24496 Reject,
24497 Timeout,
24498 TimeoutAfterWrite,
24499 EmptyEtag,
24500 OmitWrite,
24501 }
24502
24503 #[derive(Clone, Debug, Default)]
24504 struct WriteStep {
24505 gate: Option<Arc<tokio::sync::Semaphore>>,
24506 fault: WriteFault,
24507 }
24508
24509 #[derive(Clone, Debug)]
24510 struct WriteCall {
24511 pool: usize,
24512 phase: usize,
24513 if_match: Option<String>,
24514 }
24515
24516 #[derive(Debug, Default)]
24517 struct WriteTrace {
24518 calls: StdMutex<Vec<WriteCall>>,
24519 active: AtomicUsize,
24520 maximum: AtomicUsize,
24521 finished: AtomicUsize,
24522 changed: tokio::sync::Notify,
24523 }
24524
24525 impl WriteTrace {
24526 async fn wait_for(&self, calls: usize, finished: usize) {
24527 tokio::time::timeout(StdDuration::from_secs(3), async {
24528 loop {
24529 let changed = self.changed.notified();
24530 if self.calls.lock().expect("trace lock").len() >= calls
24531 && self.finished.load(Ordering::SeqCst) >= finished
24532 {
24533 return;
24534 }
24535 changed.await;
24536 }
24537 })
24538 .await
24539 .expect("phase must reach the requested barrier");
24540 }
24541
24542 fn commit_count(&self) -> usize {
24543 self.calls
24544 .lock()
24545 .expect("trace lock")
24546 .iter()
24547 .filter(|call| call.phase == 1)
24548 .count()
24549 }
24550 }
24551
24552 struct ActiveWrite<'a>(&'a WriteTrace);
24553 impl Drop for ActiveWrite<'_> {
24554 fn drop(&mut self) {
24555 self.0.active.fetch_sub(1, Ordering::SeqCst);
24556 self.0.finished.fetch_add(1, Ordering::SeqCst);
24557 self.0.changed.notify_one();
24558 }
24559 }
24560
24561 #[derive(Debug)]
24562 struct PhaseStorage {
24563 inner: PartialPoolMetaWriteStorage,
24564 pool: usize,
24565 steps: [WriteStep; 2],
24566 trace: Arc<WriteTrace>,
24567 }
24568
24569 #[async_trait::async_trait]
24570 impl ObjectIO for PhaseStorage {
24571 type Error = Error;
24572 type RangeSpec = HTTPRangeSpec;
24573 type HeaderMap = http::HeaderMap;
24574 type ObjectOptions = crate::object_api::ObjectOptions;
24575 type ObjectInfo = crate::object_api::ObjectInfo;
24576 type GetObjectReader = crate::object_api::GetObjectReader;
24577 type PutObjectReader = crate::object_api::PutObjReader;
24578
24579 async fn get_object_reader(
24580 &self,
24581 bucket: &str,
24582 object: &str,
24583 range: Option<Self::RangeSpec>,
24584 headers: Self::HeaderMap,
24585 opts: &Self::ObjectOptions,
24586 ) -> Result<Self::GetObjectReader, Error> {
24587 self.inner.get_object_reader(bucket, object, range, headers, opts).await
24588 }
24589
24590 async fn put_object(
24591 &self,
24592 bucket: &str,
24593 object: &str,
24594 data: &mut Self::PutObjectReader,
24595 opts: &Self::ObjectOptions,
24596 ) -> Result<Self::ObjectInfo, Error> {
24597 if object != POOL_META_NAME {
24598 return self.inner.put_object(bucket, object, data, opts).await;
24599 }
24600 let mut payload = Vec::new();
24601 data.stream.read_to_end(&mut payload).await?;
24602 let phase = match implementation::decode_pool_meta_replica(payload.clone()) {
24603 implementation::PoolMetaReplica::Valid { committed, .. } => usize::from(committed),
24604 _ => panic!("phase fixture must receive valid pool metadata"),
24605 };
24606 assert!(opts.max_parity && opts.no_lock);
24607 assert_eq!(opts.write_completion, crate::object_api::WriteCompletion::TailDrained);
24608 self.trace.calls.lock().expect("trace lock").push(WriteCall {
24609 pool: self.pool,
24610 phase,
24611 if_match: opts
24612 .http_preconditions
24613 .as_ref()
24614 .and_then(HTTPPreconditions::if_match_value)
24615 .map(str::to_owned),
24616 });
24617 let active = self.trace.active.fetch_add(1, Ordering::SeqCst) + 1;
24618 self.trace.maximum.fetch_max(active, Ordering::SeqCst);
24619 let _active = ActiveWrite(&self.trace);
24620 self.trace.changed.notify_one();
24621 let step = &self.steps[phase];
24622 if let Some(gate) = &step.gate {
24623 gate.acquire().await.expect("phase gate stays open").forget();
24624 }
24625 match step.fault {
24626 WriteFault::Reject => return Err(Error::PreconditionFailed),
24627 WriteFault::Timeout => return Err(Error::Timeout),
24628 WriteFault::OmitWrite => {
24629 return Ok(crate::object_api::ObjectInfo {
24630 etag: Some("uncommitted".to_owned()),
24631 ..Default::default()
24632 });
24633 }
24634 _ => {}
24635 }
24636 let mut data = crate::object_api::PutObjReader::from_vec(payload);
24637 let mut result = self.inner.put_object(bucket, object, &mut data, opts).await?;
24638 match step.fault {
24639 WriteFault::TimeoutAfterWrite => return Err(Error::Timeout),
24640 WriteFault::EmptyEtag => result.etag = Some(" ".to_owned()),
24641 _ => {}
24642 }
24643 Ok(result)
24644 }
24645 }
24646
24647 struct Fixture {
24648 pools: Vec<Arc<PhaseStorage>>,
24649 trace: Arc<WriteTrace>,
24650 previous: implementation::PoolMetaCommittedCandidate,
24651 desired: PoolMeta,
24652 pending: Vec<u8>,
24653 durable: Vec<u8>,
24654 revision: implementation::PoolMetaRevision,
24655 cluster_id: uuid::Uuid,
24656 }
24657
24658 impl Fixture {
24659 fn new(steps: Vec<[WriteStep; 2]>) -> Self {
24660 let cluster_id = uuid::Uuid::new_v4();
24661 let meta = PoolMeta {
24662 version: POOL_META_GENERATION_VERSION,
24663 pools: (0..steps.len())
24664 .map(|index| {
24665 decommission_test_pool_status(
24666 index,
24667 (index == 0).then(|| PoolDecommissionInfo {
24668 start_time: Some(OffsetDateTime::UNIX_EPOCH),
24669 start_size: 4096,
24670 total_size: 16384,
24671 current_size: 8192,
24672 items_decommissioned: 7,
24673 items_decommission_failed: 1,
24674 bytes_done: 4096,
24675 bytes_failed: 512,
24676 queued_buckets: vec!["remaining-a".to_owned(), "remaining-b".to_owned()],
24677 decommissioned_buckets: vec!["completed".to_owned()],
24678 bucket: "remaining-a".to_owned(),
24679 prefix: "objects/".to_owned(),
24680 object: "objects/007".to_owned(),
24681 ..Default::default()
24682 }),
24683 )
24684 })
24685 .collect(),
24686 ..Default::default()
24687 };
24688 let previous_revision = implementation::PoolMetaRevision {
24689 version: POOL_META_GENERATION_VERSION,
24690 cluster_id: Some(cluster_id),
24691 epoch: 1,
24692 generation: 7,
24693 transaction_id: Some(uuid::Uuid::new_v4()),
24694 };
24695 let previous = implementation::PoolMetaCommittedCandidate {
24696 canonical: implementation::encode_pool_meta_v3_envelope(&meta, previous_revision, true, None)
24697 .expect("committed predecessor"),
24698 meta: meta.clone(),
24699 revision: previous_revision,
24700 };
24701 let mut desired = meta;
24702 desired.pools[0].last_update += Duration::seconds(1);
24703 let progress = desired.pools[0].decommission.as_mut().expect("active decommission");
24704 progress.items_decommissioned += 1;
24705 progress.bytes_done += 512;
24706 progress.current_size += 512;
24707 progress.object = "objects/008".to_owned();
24708 let revision = implementation::PoolMetaRevision {
24709 generation: 8,
24710 transaction_id: Some(uuid::Uuid::new_v4()),
24711 ..previous_revision
24712 };
24713 let pending = implementation::encode_pool_meta_v3_envelope(&desired, revision, false, Some(&previous))
24714 .expect("pending candidate");
24715 let durable =
24716 implementation::encode_pool_meta_v3_envelope(&desired, revision, true, None).expect("committed candidate");
24717 let identity = implementation::initialized_pool_meta_identity_for_test(cluster_id, 1).expect("identity");
24718 let trace = Arc::new(WriteTrace::default());
24719 let pools = steps
24720 .into_iter()
24721 .enumerate()
24722 .map(|(index, steps)| {
24723 Arc::new(PhaseStorage {
24724 inner: PartialPoolMetaWriteStorage {
24725 stored: StdMutex::new(Some((previous.canonical.clone(), format!("initial-{index}")))),
24726 identity: StdMutex::new(Some((identity.clone(), "identity".to_owned()))),
24727 revision: AtomicUsize::new(index * 10),
24728 ..Default::default()
24729 },
24730 pool: index,
24731 steps,
24732 trace: trace.clone(),
24733 })
24734 })
24735 .collect();
24736 Self {
24737 pools,
24738 trace,
24739 previous,
24740 desired,
24741 pending,
24742 durable,
24743 revision,
24744 cluster_id,
24745 }
24746 }
24747
24748 fn state(&self) -> implementation::PoolMetaWriteState {
24749 implementation::PoolMetaWriteState::for_startup(self.cluster_id, false)
24750 }
24751
24752 fn tokens(&self) -> Vec<PoolMetaCasToken> {
24753 self.pools
24754 .iter()
24755 .map(|pool| {
24756 PoolMetaCasToken::Existing(
24757 pool.inner
24758 .stored
24759 .lock()
24760 .expect("fixture storage")
24761 .as_ref()
24762 .expect("existing replica")
24763 .1
24764 .clone(),
24765 )
24766 })
24767 .collect()
24768 }
24769 }
24770
24771 fn steps(count: usize) -> Vec<[WriteStep; 2]> {
24772 (0..count).map(|_| [WriteStep::default(), WriteStep::default()]).collect()
24773 }
24774
24775 fn signal() -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
24776 Some(Arc::default())
24777 }
24778
24779 async fn prepare(
24780 fixture: &Fixture,
24781 fence: &PoolMetaPersistenceFence<'_>,
24782 arm: &mut implementation::PoolMetaTransactionArm,
24783 ) -> Vec<Result<crate::object_api::ObjectInfo, Error>> {
24784 implementation::save_pool_meta_phase(
24785 &fixture.pools,
24786 &fixture.pending,
24787 &fixture.tokens(),
24788 fence,
24789 "prepare_cas",
24790 arm,
24791 #[cfg(feature = "e2e-test-hooks")]
24792 None,
24793 )
24794 .await
24795 }
24796
24797 #[tokio::test]
24798 async fn pool_meta_v3_phase_concurrency_is_bounded() {
24799 let gate = Arc::new(tokio::sync::Semaphore::new(0));
24800 let mut config = steps(6);
24801 for step in &mut config {
24802 step[0].gate = Some(gate.clone());
24803 }
24804 let fixture = Fixture::new(config);
24805 let state = fixture.state();
24806 let mut arm = state.arm_transaction();
24807 let fence = PoolMetaPersistenceFence::Distributed(signal());
24808 let mut pending = Box::pin(prepare(&fixture, &fence, &mut arm));
24809 tokio::select! { _ = fixture.trace.wait_for(4, 0) => {}, _ = &mut pending => panic!("blocked writes finished") }
24810 assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 4);
24811 gate.add_permits(1);
24812 tokio::select! { _ = fixture.trace.wait_for(5, 1) => {}, _ = &mut pending => panic!("unreleased writes finished") }
24813 assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 5);
24814 gate.add_permits(5);
24815 assert!(pending.await.into_iter().all(|result| result.is_ok()));
24816 assert_eq!(fixture.trace.maximum.load(Ordering::SeqCst), 4);
24817 assert_eq!(fixture.trace.active.load(Ordering::SeqCst), 0);
24818 arm.disarm();
24819 }
24820
24821 #[tokio::test]
24822 async fn pool_meta_v3_commit_waits_for_all_prepare_etags() {
24823 let gate = Arc::new(tokio::sync::Semaphore::new(0));
24824 let mut config = steps(4);
24825 config[3][0].gate = Some(gate.clone());
24826 let fixture = Fixture::new(config);
24827 let mut state = fixture.state();
24828 let mut save = Box::pin(
24829 fixture
24830 .desired
24831 .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]),
24832 );
24833 tokio::select! { _ = fixture.trace.wait_for(4, 3) => {}, _ = &mut save => panic!("commit crossed the last prepare") }
24834 assert_eq!(fixture.trace.commit_count(), 0);
24835 gate.add_permits(1);
24836 save.await.expect("all prepares and commits succeed").disarm();
24837 let calls = fixture.trace.calls.lock().expect("trace");
24838 for call in calls.iter().filter(|call| call.phase == 1) {
24839 assert_eq!(call.if_match, Some(format!("pool-meta-test-{}", call.pool * 10 + 1)));
24840 }
24841 assert_eq!(calls.iter().filter(|call| call.phase == 1).count(), 4);
24842 }
24843
24844 #[tokio::test]
24845 async fn pool_meta_v3_empty_prepare_etag_never_starts_commit() {
24846 let mut config = steps(4);
24847 config[3][0].fault = WriteFault::EmptyEtag;
24848 let fixture = Fixture::new(config);
24849 let mut state = fixture.state();
24850 let err = fixture
24851 .desired
24852 .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0])
24853 .await
24854 .expect_err("a missing conditional-write revision must fail");
24855 assert!(err.to_string().contains("without a conditional-write revision"));
24856 assert_eq!(fixture.trace.finished.load(Ordering::SeqCst), 4);
24857 assert_eq!(fixture.trace.commit_count(), 0);
24858 assert!(state.ensure_write_safe("empty etag wrote pending data").is_err());
24859 }
24860
24861 #[tokio::test]
24862 async fn pool_meta_v3_prepare_failure_drains_late_writes() {
24863 let gate = Arc::new(tokio::sync::Semaphore::new(0));
24864 let mut config = steps(4);
24865 config[1][0].fault = WriteFault::Timeout;
24866 config[3][0].gate = Some(gate.clone());
24867 let fixture = Fixture::new(config);
24868 let mut state = fixture.state();
24869 let mut save = Box::pin(
24870 fixture
24871 .desired
24872 .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]),
24873 );
24874 tokio::select! { _ = fixture.trace.wait_for(4, 3) => {}, _ = &mut save => panic!("phase returned with a live replica") }
24875 assert_eq!(fixture.trace.commit_count(), 0);
24876 gate.add_permits(1);
24877 assert!(save.await.is_err());
24878 assert_eq!(fixture.trace.finished.load(Ordering::SeqCst), 4);
24879 assert!(fixture.pools[3].inner.wrote.load(Ordering::SeqCst));
24880 assert!(state.ensure_write_safe("ambiguous prepare").is_err());
24881 }
24882
24883 #[tokio::test]
24884 async fn pool_meta_v3_rejections_restore_only_an_unmutated_phase() {
24885 for previous in [None, Some("identity_cas")] {
24886 let mut config = steps(4);
24887 for step in &mut config {
24888 step[0].fault = WriteFault::Reject;
24889 }
24890 let fixture = Fixture::new(config);
24891 let state = fixture.state();
24892 let mut arm = state.arm_transaction();
24893 arm.phase = previous;
24894 let results = prepare(&fixture, &PoolMetaPersistenceFence::Distributed(signal()), &mut arm).await;
24895 assert!(
24896 results
24897 .into_iter()
24898 .all(|result| matches!(result, Err(Error::PreconditionFailed)))
24899 );
24900 assert_eq!(arm.phase, previous);
24901 drop(arm);
24902 assert_eq!(state.ensure_write_safe("all rejected").is_ok(), previous.is_none());
24903 }
24904 let mut config = steps(4);
24905 for step in &mut config[1..] {
24906 step[0].fault = WriteFault::Reject;
24907 }
24908 let fixture = Fixture::new(config);
24909 let state = fixture.state();
24910 let mut arm = state.arm_transaction();
24911 let results = prepare(&fixture, &PoolMetaPersistenceFence::Distributed(signal()), &mut arm).await;
24912 assert!(results[0].is_ok());
24913 assert_eq!(arm.phase, Some("prepare_cas"));
24914 drop(arm);
24915 assert!(state.ensure_write_safe("one sibling wrote").is_err());
24916 }
24917
24918 #[tokio::test]
24919 async fn pool_meta_v3_parallel_errors_keep_original_source_pointers() {
24920 let mut config = steps(4);
24921 config[0][0].fault = WriteFault::Timeout;
24922 config[1][0].fault = WriteFault::Timeout;
24923 config[3][0].fault = WriteFault::Reject;
24924 let fixture = Fixture::new(config);
24925 let state = fixture.state();
24926 let mut arm = state.arm_transaction();
24927 let results = prepare(&fixture, &PoolMetaPersistenceFence::Distributed(signal()), &mut arm).await;
24928 let err = results[0].as_ref().expect_err("first pool error stays first");
24929 let source = err
24930 .pool_metadata_failure()
24931 .expect("typed error")
24932 .source
24933 .as_ref()
24934 .expect("original source");
24935 assert!(matches!(source.as_ref(), Error::Timeout));
24936 assert!(Arc::ptr_eq(source, arm.source.as_ref().expect("arm source")));
24937 drop(arm);
24938 let blocked = state
24939 .ensure_write_safe("after parallel errors")
24940 .expect_err("recovery required");
24941 assert!(Arc::ptr_eq(
24942 source,
24943 blocked
24944 .pool_metadata_failure()
24945 .expect("typed latch")
24946 .source
24947 .as_ref()
24948 .expect("latch source")
24949 ));
24950 }
24951
24952 #[tokio::test]
24953 async fn pool_meta_v3_retry_rejections_keep_the_previous_io_source() {
24954 let mut config = steps(4);
24955 config[0][0].fault = WriteFault::Reject;
24956 config[1][0].fault = WriteFault::Timeout;
24957 let first = Fixture::new(config);
24958 let state = first.state();
24959 let mut arm = state.arm_transaction();
24960 let fence = PoolMetaPersistenceFence::Distributed(signal());
24961 let results = prepare(&first, &fence, &mut arm).await;
24962 assert!(matches!(&results[0], Err(Error::PreconditionFailed)));
24963 let original = arm.source.as_ref().expect("first attempt I/O source").clone();
24964 assert!(matches!(original.as_ref(), Error::Timeout));
24965 let mut config = steps(4);
24966 for step in &mut config {
24967 step[0].fault = WriteFault::Reject;
24968 }
24969 let retry = Fixture::new(config);
24970 assert!(
24971 prepare(&retry, &fence, &mut arm)
24972 .await
24973 .into_iter()
24974 .all(|result| matches!(result, Err(Error::PreconditionFailed)))
24975 );
24976 assert!(Arc::ptr_eq(&original, arm.source.as_ref().expect("source survives phase retry")));
24977 implementation::save_pool_meta_object_cas(
24978 retry.pools[0].clone(),
24979 POOL_META_NAME,
24980 retry.pending.clone(),
24981 &retry.tokens()[0],
24982 &fence,
24983 "prepare_cas",
24984 &mut arm,
24985 )
24986 .await
24987 .expect_err("serial retry rejected");
24988 assert!(Arc::ptr_eq(&original, arm.source.as_ref().expect("source survives serial retry")));
24989 drop(arm);
24990 let blocked = state
24991 .ensure_write_safe("cancelled after retry rejection")
24992 .expect_err("recovery required");
24993 assert!(Arc::ptr_eq(
24994 &original,
24995 blocked
24996 .pool_metadata_failure()
24997 .expect("typed latch")
24998 .source
24999 .as_ref()
25000 .expect("source")
25001 ));
25002 }
25003
25004 #[tokio::test]
25005 async fn pool_meta_v3_cancelled_drain_does_not_replace_io_error_with_cas_rejection() {
25006 let gate = Arc::new(tokio::sync::Semaphore::new(0));
25007 let mut config = steps(4);
25008 config[0][0].gate = Some(gate.clone());
25009 config[2][0].gate = Some(gate);
25010 config[1][0].fault = WriteFault::Timeout;
25011 config[3][0].fault = WriteFault::Reject;
25012 let fixture = Fixture::new(config);
25013 let state = fixture.state();
25014 let mut arm = state.arm_transaction();
25015 let fence = PoolMetaPersistenceFence::Distributed(signal());
25016 let mut phase = Box::pin(prepare(&fixture, &fence, &mut arm));
25017 tokio::select! { _ = fixture.trace.wait_for(4, 2) => {}, _ = &mut phase => panic!("blocked phase finished") }
25018 assert!(futures::poll!(phase.as_mut()).is_pending(), "drain remains parked on the two slow pools");
25019 drop(phase);
25020 let source = arm.source.as_ref().expect("observed I/O source").clone();
25021 assert!(matches!(source.as_ref(), Error::Timeout));
25022 drop(arm);
25023 let blocked = state
25024 .ensure_write_safe("cancelled after mixed results")
25025 .expect_err("recovery required");
25026 assert!(Arc::ptr_eq(
25027 &source,
25028 blocked
25029 .pool_metadata_failure()
25030 .expect("typed latch")
25031 .source
25032 .as_ref()
25033 .expect("source")
25034 ));
25035 assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
25036 }
25037
25038 #[tokio::test]
25039 async fn pool_meta_v3_cancelled_prepare_commit_and_publication_keep_gate_blocked() {
25040 for phase in [0, 1] {
25041 let gate = Arc::new(tokio::sync::Semaphore::new(0));
25042 let mut config = steps(4);
25043 for step in &mut config {
25044 step[phase].gate = Some(gate.clone());
25045 }
25046 let fixture = Fixture::new(config);
25047 let mut state = fixture.state();
25048 let mut save = Box::pin(
25049 fixture
25050 .desired
25051 .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]),
25052 );
25053 tokio::select! { _ = fixture.trace.wait_for(4 * (phase + 1), 4 * phase) => {}, _ = &mut save => panic!("blocked phase finished") }
25054 drop(save);
25055 assert!(state.ensure_write_safe("cancelled phase").is_err());
25056 assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
25057 assert_eq!(fixture.trace.active.load(Ordering::SeqCst), 0);
25058 }
25059 let fixture = Fixture::new(steps(4));
25060 let mut state = fixture.state();
25061 let outcome = fixture
25062 .desired
25063 .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0])
25064 .await
25065 .expect("durable save before publication");
25066 drop(outcome);
25067 assert!(state.ensure_write_safe("publication was cancelled").is_err());
25068 assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
25069 }
25070
25071 #[tokio::test]
25072 async fn pool_meta_v3_fence_loss_blocks_queued_dispatch_and_marks_late_writes() {
25073 let lock = NamespaceLock::with_clients_and_quorum(
25074 "parallel-fence".to_owned(),
25075 vec![Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())))],
25076 1,
25077 );
25078 let request = LockRequest::new(
25079 ObjectKey::new(super::super::RUSTFS_META_BUCKET, POOL_META_NAME),
25080 LockType::Exclusive,
25081 "phase-test",
25082 )
25083 .with_acquire_timeout(StdDuration::from_secs(1))
25084 .with_ttl(StdDuration::from_secs(1))
25085 .with_refresh_interval(StdDuration::from_secs(1));
25086 let guard = lock
25087 .acquire_guard(&request)
25088 .await
25089 .expect("lease request")
25090 .expect("lease acquired");
25091 let gate = Arc::new(tokio::sync::Semaphore::new(0));
25092 let mut config = steps(6);
25093 for step in &mut config {
25094 step[0].gate = Some(gate.clone());
25095 }
25096 let fixture = Fixture::new(config);
25097 let state = fixture.state();
25098 let mut arm = state.arm_transaction();
25099 let fence = PoolMetaPersistenceFence::Distributed(guard.lock_lost_signal());
25100 let mut pending = Box::pin(prepare(&fixture, &fence, &mut arm));
25101 tokio::select! { _ = fixture.trace.wait_for(4, 0) => {}, _ = &mut pending => panic!("gate ignored") }
25102 tokio::time::timeout(StdDuration::from_secs(3), guard.lock_lost_notified())
25103 .await
25104 .expect("lease expires");
25105 gate.add_permits(6);
25106 assert!(pending.await.into_iter().all(|result| result.is_err()));
25107 assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 4, "queued pools must not reach storage");
25108 assert!(fixture.pools[..4].iter().all(|pool| pool.inner.wrote.load(Ordering::SeqCst)));
25109 assert_eq!(arm.phase, Some("prepare_cas"));
25110 drop(arm);
25111 assert!(state.ensure_write_safe("writes landed after lease loss").is_err());
25112 }
25113
25114 #[tokio::test]
25115 async fn pool_meta_v3_all_commit_errors_still_require_canonical_reread() {
25116 for fault in [WriteFault::TimeoutAfterWrite, WriteFault::OmitWrite] {
25117 let mut config = steps(4);
25118 for step in &mut config {
25119 step[1].fault = fault;
25120 }
25121 let fixture = Fixture::new(config);
25122 let mut state = fixture.state();
25123 let result = fixture
25124 .desired
25125 .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0])
25126 .await;
25127 assert_eq!(fixture.trace.commit_count(), 4);
25128 if matches!(fault, WriteFault::TimeoutAfterWrite) {
25129 let outcome = result.expect("reread proves commits whose replies timed out");
25130 assert_eq!(outcome.committed.pools[0].last_update, fixture.desired.pools[0].last_update);
25131 outcome.disarm();
25132 state
25133 .ensure_write_safe("confirmed ambiguous commits")
25134 .expect("confirmed transaction is safe");
25135 } else {
25136 assert!(result.is_err(), "successful replies cannot replace canonical proof");
25137 assert!(state.ensure_write_safe("no committed canonical generation").is_err());
25138 }
25139 }
25140 }
25141
25142 #[tokio::test]
25143 async fn pool_meta_v3_none_and_legacy_paths_remain_serial() {
25144 for (legacy, needs_repair) in [(false, false), (true, false), (false, true)] {
25145 let gate = Arc::new(tokio::sync::Semaphore::new(0));
25146 let mut config = steps(4);
25147 for step in &mut config {
25148 step[0].gate = Some(gate.clone());
25149 step[1].gate = Some(gate.clone());
25150 }
25151 let mut fixture = Fixture::new(config);
25152 if legacy {
25153 let mut meta = fixture.previous.meta.clone();
25154 meta.version = POOL_META_VERSION;
25155 let data = meta.encode_config_data_for_v2_gate(true).expect("legacy metadata");
25156 for pool in &fixture.pools {
25157 pool.inner.stored.lock().expect("fixture").as_mut().expect("replica").0 = data.clone();
25158 }
25159 fixture.desired.version = POOL_META_VERSION;
25160 }
25161 if needs_repair {
25162 fixture.pools[3]
25163 .inner
25164 .stored
25165 .lock()
25166 .expect("fixture")
25167 .as_mut()
25168 .expect("replica")
25169 .0 = fixture.pending.clone();
25170 }
25171 let mut state = fixture.state();
25172 let mut save = Box::pin(fixture.desired.save_no_lock_armed(
25173 fixture.pools.clone(),
25174 &mut state,
25175 if legacy || needs_repair { signal() } else { None },
25176 &[0],
25177 ));
25178 tokio::select! { _ = fixture.trace.wait_for(1, 0) => {}, _ = &mut save => panic!("serial write gate ignored") }
25179 assert!(futures::poll!(save.as_mut()).is_pending());
25180 assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 1);
25181 gate.add_permits(16);
25182 save.await.expect("serial save").disarm();
25183 assert_eq!(fixture.trace.maximum.load(Ordering::SeqCst), 1);
25184 }
25185 }
25186
25187 #[cfg(feature = "e2e-test-hooks")]
25188 #[tokio::test]
25189 async fn pool_meta_phase_barrier_leaves_noop_arm_for_the_next_changed_transaction() {
25190 for case in [
25191 implementation::PoolMetaPhaseBarrierCase::PrepareSubset,
25192 implementation::PoolMetaPhaseBarrierCase::PreparedAll,
25193 implementation::PoolMetaPhaseBarrierCase::CommitOne,
25194 implementation::PoolMetaPhaseBarrierCase::BeforePublish,
25195 ] {
25196 let fixture = Fixture::new(steps(4));
25197 let directory = tempfile::TempDir::new().expect("barrier directory");
25198 tokio::fs::write(
25199 directory.path().join("arm.json"),
25200 serde_json::to_vec(&serde_json::json!({"nonce": uuid::Uuid::new_v4(), "case": case})).expect("arm"),
25201 )
25202 .await
25203 .expect("write arm");
25204 let noop = implementation::encode_pool_meta_v3_envelope(&fixture.previous.meta, fixture.revision, true, None)
25205 .expect("new revision with unchanged pools");
25206 assert_ne!(noop, fixture.previous.canonical);
25207 let barrier = implementation::PoolMetaPhaseBarrier::bind_in_directory(
25208 directory.path().to_path_buf(),
25209 &fixture.previous,
25210 &fixture.previous.meta,
25211 fixture.revision,
25212 &noop,
25213 )
25214 .await
25215 .expect("unchanged pools must not consume the arm");
25216 assert!(barrier.is_none());
25217 assert!(!directory.path().join("claimed").exists());
25218 assert!(!directory.path().join("events.jsonl").exists());
25219
25220 let previous = implementation::PoolMetaCommittedCandidate {
25221 meta: fixture.previous.meta.clone(),
25222 revision: fixture.revision,
25223 canonical: noop,
25224 };
25225 let mut candidate = previous.meta.clone();
25226 let progress = candidate.pools[0].decommission.as_mut().expect("progress");
25227 progress.items_decommissioned += 1;
25228 progress.bytes_done += 512;
25229 let revision = implementation::PoolMetaRevision {
25230 generation: previous.revision.generation + 1,
25231 transaction_id: Some(uuid::Uuid::new_v4()),
25232 ..previous.revision
25233 };
25234 let durable = implementation::encode_pool_meta_v3_envelope(&candidate, revision, true, None)
25235 .expect("changed progress without a timestamp change");
25236 let barrier = implementation::PoolMetaPhaseBarrier::bind_in_directory(
25237 directory.path().to_path_buf(),
25238 &previous,
25239 &candidate,
25240 revision,
25241 &durable,
25242 )
25243 .await
25244 .expect("next changed transaction must bind")
25245 .expect("bound barrier");
25246 assert_eq!(barrier.transaction_id, revision.transaction_id.expect("transaction"));
25247 assert!(directory.path().join("claimed").is_file());
25248 let events = tokio::fs::read_to_string(directory.path().join("events.jsonl"))
25249 .await
25250 .expect("armed event");
25251 assert_eq!(events.lines().count(), 1);
25252 let event: serde_json::Value = serde_json::from_str(events.trim()).expect("armed event JSON");
25253 assert_eq!(event["kind"], "armed");
25254 assert_eq!(event["previous"]["generation"], previous.revision.generation);
25255 assert_eq!(event["candidate"]["generation"], revision.generation);
25256 assert_ne!(event["previous"]["persisted_pools_sha256"], event["candidate"]["persisted_pools_sha256"]);
25257 assert!(
25258 implementation::PoolMetaPhaseBarrier::bind_in_directory(
25259 directory.path().to_path_buf(),
25260 &previous,
25261 &candidate,
25262 revision,
25263 &durable,
25264 )
25265 .await
25266 .expect("already claimed arm")
25267 .is_none()
25268 );
25269 }
25270 }
25271
25272 #[cfg(feature = "e2e-test-hooks")]
25273 #[tokio::test]
25274 async fn pool_meta_phase_barrier_uses_persisted_responsibility_not_runtime_progress() {
25275 let fixture = Fixture::new(steps(4));
25276 let directory = tempfile::TempDir::new().expect("barrier directory");
25277 tokio::fs::write(
25278 directory.path().join("arm.json"),
25279 serde_json::to_vec(&serde_json::json!({"nonce": uuid::Uuid::new_v4(), "case": "prepare_subset"})).expect("arm"),
25280 )
25281 .await
25282 .expect("write arm");
25283 let mut candidate = fixture.previous.meta.clone();
25284 candidate.pools[0].decommission.as_mut().expect("progress").stage = "entry_started".to_owned();
25285 for changed in [false, true] {
25286 if changed {
25287 candidate.pools[0]
25288 .decommission
25289 .as_mut()
25290 .expect("progress")
25291 .queued_buckets
25292 .push("new-responsibility".to_owned());
25293 }
25294 let durable =
25295 implementation::encode_pool_meta_v3_envelope(&candidate, fixture.revision, true, None).expect("candidate");
25296 let barrier = implementation::PoolMetaPhaseBarrier::bind_in_directory(
25297 directory.path().to_path_buf(),
25298 &fixture.previous,
25299 &candidate,
25300 fixture.revision,
25301 &durable,
25302 )
25303 .await
25304 .expect("binding by persisted state");
25305 assert_eq!(barrier.is_some(), changed);
25306 assert_eq!(directory.path().join("claimed").exists(), changed);
25307 assert_eq!(directory.path().join("events.jsonl").exists(), changed);
25308 }
25309 }
25310
25311 fn persisted_pools(meta: &PoolMeta) -> serde_json::Value {
25312 serde_json::to_value(
25313 meta.pools
25314 .iter()
25315 .map(implementation::PersistedPoolStatus::from)
25316 .collect::<Vec<_>>(),
25317 )
25318 .expect("persisted pool state")
25319 }
25320
25321 #[tokio::test]
25322 async fn pool_meta_v3_all_prepare_subsets_preserve_previous_snapshot() {
25323 for previous_version in [POOL_META_VERSION, POOL_META_GENERATION_VERSION] {
25324 for mask in 0usize..16 {
25325 let mut fixture = Fixture::new(steps(4));
25326 if previous_version != POOL_META_GENERATION_VERSION {
25327 fixture.previous.meta.version = previous_version;
25328 fixture.previous.revision = implementation::PoolMetaRevision::legacy(previous_version);
25329 fixture.revision.generation = 1;
25330 fixture.previous.canonical = fixture
25331 .previous
25332 .meta
25333 .encode_config_data_for_v2_gate(true)
25334 .expect("legacy predecessor");
25335 fixture.pending = implementation::encode_pool_meta_v3_envelope(
25336 &fixture.desired,
25337 fixture.revision,
25338 false,
25339 Some(&fixture.previous),
25340 )
25341 .expect("legacy pending envelope");
25342 }
25343 let payloads = (0..4)
25344 .map(|pool| {
25345 if mask & (1 << pool) != 0 {
25346 fixture.pending.clone()
25347 } else {
25348 fixture.previous.canonical.clone()
25349 }
25350 })
25351 .collect::<Vec<_>>();
25352 let selected = implementation::select_pool_meta_replica(
25353 payloads
25354 .iter()
25355 .cloned()
25356 .map(implementation::decode_pool_meta_replica)
25357 .collect(),
25358 )
25359 .expect("every pending subset selects its predecessor");
25360 assert_eq!(
25361 persisted_pools(&selected.meta),
25362 persisted_pools(&fixture.previous.meta),
25363 "mask={mask:04b}"
25364 );
25365 assert_eq!(selected.revision, fixture.previous.revision);
25366 if previous_version == POOL_META_GENERATION_VERSION {
25367 for (pool, data) in fixture.pools.iter().zip(payloads) {
25368 pool.inner.stored.lock().expect("fixture").as_mut().expect("replica").0 = data;
25369 }
25370 let mut state = fixture.state();
25371 let selected = implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state)
25372 .await
25373 .expect("production recovery selection");
25374 let outcome = implementation::repair_pool_meta_transaction(
25375 fixture.pools.clone(),
25376 &mut state,
25377 selected,
25378 &PoolMetaPersistenceFence::Distributed(None),
25379 )
25380 .await
25381 .expect("production recovery repair");
25382 assert_eq!(persisted_pools(&outcome.committed), persisted_pools(&fixture.previous.meta));
25383 outcome.disarm();
25384 let confirmed =
25385 implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state)
25386 .await
25387 .expect("repaired reread");
25388 assert_eq!(persisted_pools(&confirmed.meta), persisted_pools(&fixture.previous.meta));
25389 assert!(!confirmed.replica_state.needs_repair);
25390 }
25391 }
25392 }
25393 }
25394
25395 #[tokio::test]
25396 async fn pool_meta_v3_every_sole_commit_position_recovers_new_snapshot() {
25397 for committed_pool in 0..4 {
25398 let fixture = Fixture::new(steps(4));
25399 for (index, pool) in fixture.pools.iter().enumerate() {
25400 pool.inner.stored.lock().expect("fixture").as_mut().expect("replica").0 = if index == committed_pool {
25401 fixture.durable.clone()
25402 } else {
25403 fixture.pending.clone()
25404 };
25405 }
25406 let mut state = fixture.state();
25407 let selected = implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state)
25408 .await
25409 .expect("sole commit must select new generation");
25410 assert_eq!(selected.revision, fixture.revision);
25411 assert_eq!(persisted_pools(&selected.meta), persisted_pools(&fixture.desired));
25412 let outcome = implementation::repair_pool_meta_transaction(
25413 fixture.pools.clone(),
25414 &mut state,
25415 selected,
25416 &PoolMetaPersistenceFence::Distributed(None),
25417 )
25418 .await
25419 .expect("repair partial commit");
25420 assert_eq!(persisted_pools(&outcome.committed), persisted_pools(&fixture.desired));
25421 outcome.disarm();
25422 let confirmed = implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state)
25423 .await
25424 .expect("repaired sole commit reread");
25425 assert_eq!(persisted_pools(&confirmed.meta), persisted_pools(&fixture.desired));
25426 assert!(!confirmed.replica_state.needs_repair);
25427 let fork = implementation::encode_pool_meta_v3_envelope(
25428 &fixture.desired,
25429 implementation::PoolMetaRevision {
25430 transaction_id: Some(uuid::Uuid::new_v4()),
25431 ..fixture.revision
25432 },
25433 true,
25434 None,
25435 )
25436 .expect("same generation fork");
25437 assert!(
25438 implementation::select_pool_meta_replica(vec![
25439 implementation::decode_pool_meta_replica(fixture.durable.clone()),
25440 implementation::decode_pool_meta_replica(fork)
25441 ])
25442 .is_err()
25443 );
25444 }
25445 }
25446 }
25447
25448 #[tokio::test]
25449 async fn test_pool_meta_cas_deterministically_rejects_stale_writer() {
25450 let storage = Arc::new(PartialPoolMetaWriteStorage {
25451 fail_write: false,
25452 fail_after_first_write: false,
25453 pending_write: false,
25454 write_started: tokio::sync::Notify::new(),
25455 wrote: AtomicBool::new(false),
25456 revision: AtomicUsize::new(0),
25457 stored: StdMutex::new(None),
25458 identity: StdMutex::new(None),
25459 });
25460 let stale_token = read_pool_meta_replica(storage.clone(), true).await.cas;
25461 assert!(matches!(&stale_token, PoolMetaCasToken::Missing));
25462 let winner = PoolMeta {
25463 version: POOL_META_VERSION,
25464 pools: vec![decommission_test_pool_status(0, None)],
25465 ..Default::default()
25466 }
25467 .encode_config_data_for_test()
25468 .expect("winning pool metadata should encode");
25469 let stale = PoolMeta {
25470 version: POOL_META_VERSION,
25471 pools: vec![decommission_test_pool_status(
25472 0,
25473 Some(PoolDecommissionInfo {
25474 complete: true,
25475 ..Default::default()
25476 }),
25477 )],
25478 ..Default::default()
25479 }
25480 .encode_config_data_for_test()
25481 .expect("stale pool metadata should encode");
25482 let fence = PoolMetaPersistenceFence::Distributed(None);
25483
25484 let mut arm = super::PoolMetaWriteState::default().arm_transaction();
25485 save_pool_meta_object_cas(
25486 storage.clone(),
25487 POOL_META_NAME,
25488 winner.clone(),
25489 &stale_token,
25490 &fence,
25491 "prepare_cas",
25492 &mut arm,
25493 )
25494 .await
25495 .expect("the first writer should create pool metadata");
25496 let err =
25497 save_pool_meta_object_cas(storage.clone(), POOL_META_NAME, stale, &stale_token, &fence, "prepare_cas", &mut arm)
25498 .await
25499 .expect_err("the second writer must not reuse the stale missing-object revision");
25500
25501 assert_eq!(err, Error::PreconditionFailed);
25502 let stored = storage
25503 .stored
25504 .lock()
25505 .expect("pool metadata test storage should not be poisoned")
25506 .as_ref()
25507 .map(|(data, _)| data.clone())
25508 .expect("the winning pool metadata should remain stored");
25509 assert_eq!(stored, winner);
25510 }
25511
25512 #[tokio::test]
25513 async fn test_pool_meta_v3_single_replica_commit_failure_recovers_on_restart() {
25514 let committed_replica = Arc::new(PartialPoolMetaWriteStorage {
25515 fail_write: false,
25516 fail_after_first_write: false,
25517 pending_write: false,
25518 write_started: tokio::sync::Notify::new(),
25519 wrote: AtomicBool::new(false),
25520 revision: AtomicUsize::new(0),
25521 stored: StdMutex::new(None),
25522 identity: StdMutex::new(None),
25523 });
25524 let pending_replica = Arc::new(PartialPoolMetaWriteStorage {
25525 fail_write: false,
25526 fail_after_first_write: true,
25527 pending_write: false,
25528 write_started: tokio::sync::Notify::new(),
25529 wrote: AtomicBool::new(false),
25530 revision: AtomicUsize::new(0),
25531 stored: StdMutex::new(None),
25532 identity: StdMutex::new(None),
25533 });
25534 let cluster_id = uuid::Uuid::new_v4();
25535 let fresh_bootstrap_nonce = uuid::Uuid::new_v4();
25536 let identity = super::encode_pool_meta_identity(super::PersistedPoolMetaIdentity {
25537 version: super::POOL_META_IDENTITY_VERSION,
25538 cluster_id,
25539 epoch: super::POOL_META_INITIAL_EPOCH,
25540 initialized: false,
25541 fresh_bootstrap_nonce: Some(fresh_bootstrap_nonce),
25542 })
25543 .expect("pending bootstrap identity should encode");
25544 *committed_replica
25545 .identity
25546 .lock()
25547 .expect("identity storage should not be poisoned") = Some((identity.clone(), "identity-0".to_string()));
25548 *pending_replica
25549 .identity
25550 .lock()
25551 .expect("identity storage should not be poisoned") = Some((identity, "identity-0".to_string()));
25552 let mut write_state = super::PoolMetaWriteState::for_startup(cluster_id, true);
25553 let snapshot = PoolMeta {
25554 version: super::POOL_META_GENERATION_VERSION,
25555 pools: vec![decommission_test_pool_status(
25556 0,
25557 Some(PoolDecommissionInfo {
25558 queued: true,
25559 ..Default::default()
25560 }),
25561 )],
25562 ..Default::default()
25563 };
25564
25565 snapshot
25566 .save_no_lock_observing(vec![committed_replica.clone(), pending_replica.clone()], &mut write_state)
25567 .await
25568 .expect("one committed replica should durably complete the V3 transaction");
25569
25570 let mut restarted = PoolMeta::default();
25571 let replica_state = restarted
25572 .load_no_lock_from_replicas(vec![committed_replica, pending_replica.clone()])
25573 .await
25574 .expect("restart should select the committed generation over a pending replica");
25575 assert_eq!(restarted.version, super::POOL_META_GENERATION_VERSION);
25576 assert!(restarted.pools[0].decommission.as_ref().is_some_and(|info| info.queued));
25577 assert!(replica_state.needs_repair);
25578
25579 let mut pending_only = PoolMeta::default();
25580 pending_only
25581 .load_no_lock_from_replicas(vec![pending_replica])
25582 .await
25583 .expect("a prepare-only replica should expose the embedded predecessor");
25584 assert!(pending_only.pools.is_empty());
25585 }
25586
25587 #[tokio::test]
25588 async fn test_partial_pool_meta_save_failure_blocks_following_side_effect() {
25589 let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
25590 let committed = Arc::new(PartialPoolMetaWriteStorage {
25591 fail_write: false,
25592 fail_after_first_write: false,
25593 pending_write: false,
25594 write_started: tokio::sync::Notify::new(),
25595 wrote: AtomicBool::new(false),
25596 revision: AtomicUsize::new(0),
25597 stored: StdMutex::new(None),
25598 identity: StdMutex::new(None),
25599 });
25600 let failed = Arc::new(PartialPoolMetaWriteStorage {
25601 fail_write: true,
25602 fail_after_first_write: false,
25603 pending_write: false,
25604 write_started: tokio::sync::Notify::new(),
25605 wrote: AtomicBool::new(false),
25606 revision: AtomicUsize::new(0),
25607 stored: StdMutex::new(None),
25608 identity: StdMutex::new(None),
25609 });
25610 let snapshot = PoolMeta {
25611 version: super::POOL_META_VERSION,
25612 pools: vec![decommission_test_pool_status(0, None)],
25613 ..Default::default()
25614 };
25615
25616 {
25617 let mut save_guard = store.pool_meta_save_gate.lock().await;
25618 snapshot
25619 .save_no_lock_observing(vec![committed.clone(), failed], &mut save_guard)
25620 .await
25621 .expect_err("the second replica write should fail after the first commits");
25622 }
25623 assert!(committed.wrote.load(Ordering::SeqCst));
25624
25625 let ran = Arc::new(AtomicBool::new(false));
25626 let ran_by_operation = ran.clone();
25627 let movement_gate = store.ctx.data_movement_operation_gate();
25628 let result: std::result::Result<(), Error> = store
25629 .run_guarded_decommission_side_effect(&CancellationToken::new(), &movement_gate, move || async move {
25630 ran_by_operation.store(true, Ordering::SeqCst);
25631 Ok(())
25632 })
25633 .await;
25634
25635 assert!(result.is_err(), "a partial pool metadata save must latch the sticky safety gate");
25636 assert!(!ran.load(Ordering::SeqCst), "the side effect must not run after a partial save");
25637 }
25638
25639 #[tokio::test]
25640 async fn test_cancelled_pool_meta_save_blocks_following_side_effect() {
25641 let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
25642 let committed = Arc::new(PartialPoolMetaWriteStorage {
25643 fail_write: false,
25644 fail_after_first_write: false,
25645 pending_write: false,
25646 write_started: tokio::sync::Notify::new(),
25647 wrote: AtomicBool::new(false),
25648 revision: AtomicUsize::new(0),
25649 stored: StdMutex::new(None),
25650 identity: StdMutex::new(None),
25651 });
25652 let pending = Arc::new(PartialPoolMetaWriteStorage {
25653 fail_write: false,
25654 fail_after_first_write: false,
25655 pending_write: true,
25656 write_started: tokio::sync::Notify::new(),
25657 wrote: AtomicBool::new(false),
25658 revision: AtomicUsize::new(0),
25659 stored: StdMutex::new(None),
25660 identity: StdMutex::new(None),
25661 });
25662 let snapshot = PoolMeta {
25663 version: super::POOL_META_VERSION,
25664 pools: vec![decommission_test_pool_status(0, None)],
25665 ..Default::default()
25666 };
25667 let save_store = store.clone();
25668 let save_committed = committed.clone();
25669 let save_pending = pending.clone();
25670 let save_task = tokio::spawn(async move {
25671 let mut save_guard = save_store.pool_meta_save_gate.lock().await;
25672 snapshot
25673 .save_no_lock_observing(vec![save_committed, save_pending], &mut save_guard)
25674 .await
25675 });
25676
25677 tokio::time::timeout(StdDuration::from_secs(1), pending.write_started.notified())
25678 .await
25679 .expect("the second replica write should start");
25680 assert!(committed.wrote.load(Ordering::SeqCst));
25681 save_task.abort();
25682 assert!(
25683 save_task.await.expect_err("the save task should be cancelled").is_cancelled(),
25684 "the pending replica write should be aborted"
25685 );
25686
25687 {
25688 let save_guard = store.pool_meta_save_gate.lock().await;
25689 save_guard
25690 .ensure_write_safe("cancelled pool metadata save")
25691 .expect_err("a cancelled replica update must latch the sticky safety gate");
25692 }
25693
25694 let ran = Arc::new(AtomicBool::new(false));
25695 let ran_by_operation = ran.clone();
25696 let movement_gate = store.ctx.data_movement_operation_gate();
25697 let result: std::result::Result<(), Error> = store
25698 .run_guarded_decommission_side_effect(&CancellationToken::new(), &movement_gate, move || async move {
25699 ran_by_operation.store(true, Ordering::SeqCst);
25700 Ok(())
25701 })
25702 .await;
25703
25704 assert!(result.is_err(), "a cancelled pool metadata save must block following side effects");
25705 assert!(!ran.load(Ordering::SeqCst), "the side effect must not run after a cancelled save");
25706 }
25707
25708 #[tokio::test]
25709 async fn test_cancelled_pool_meta_publish_keeps_write_gate_blocked() {
25710 let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
25711 let committed = Arc::new(PartialPoolMetaWriteStorage {
25712 fail_write: false,
25713 fail_after_first_write: false,
25714 pending_write: false,
25715 write_started: tokio::sync::Notify::new(),
25716 wrote: AtomicBool::new(false),
25717 revision: AtomicUsize::new(0),
25718 stored: StdMutex::new(None),
25719 identity: StdMutex::new(None),
25720 });
25721 let snapshot = PoolMeta {
25722 version: super::POOL_META_VERSION,
25723 pools: vec![decommission_test_pool_status(0, None)],
25724 ..Default::default()
25725 };
25726 let publish_started = Arc::new(tokio::sync::Notify::new());
25727 let publish_release = Arc::new(tokio::sync::Notify::new());
25728 let save_store = store.clone();
25729 let save_committed = committed.clone();
25730 let task_publish_started = publish_started.clone();
25731 let task_publish_release = publish_release.clone();
25732 let save_task = tokio::spawn(async move {
25733 let mut save_guard = save_store.pool_meta_save_gate.lock().await;
25734 let outcome = snapshot
25735 .save_no_lock_armed(vec![save_committed], &mut save_guard, None, &[0])
25736 .await?;
25737 task_publish_started.notify_one();
25738 task_publish_release.notified().await;
25739 outcome.disarm();
25740 Ok::<(), Error>(())
25741 });
25742
25743 tokio::time::timeout(StdDuration::from_secs(1), publish_started.notified())
25744 .await
25745 .expect("the replica save should complete before publication");
25746 assert!(committed.wrote.load(Ordering::SeqCst));
25747 save_task.abort();
25748 assert!(
25749 save_task
25750 .await
25751 .expect_err("the publication task should be cancelled")
25752 .is_cancelled(),
25753 "the task should be aborted while publication is pending"
25754 );
25755
25756 let save_guard = store.pool_meta_save_gate.lock().await;
25757 save_guard
25758 .ensure_write_safe("cancelled pool metadata publication")
25759 .expect_err("cancellation after replica save but before publication must keep writes blocked");
25760 }
25761
25762 #[tokio::test]
25763 async fn test_lost_pool_meta_fence_rejects_replica_write() {
25764 let client = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
25765 let lock = NamespaceLock::with_clients_and_quorum("pool-meta-fence-loss".to_string(), vec![client], 1);
25766 let request = LockRequest::new(
25767 ObjectKey::new(super::RUSTFS_META_BUCKET, super::POOL_META_NAME),
25768 LockType::Exclusive,
25769 "stale-writer",
25770 )
25771 .with_acquire_timeout(StdDuration::from_secs(1))
25772 .with_ttl(StdDuration::from_millis(50))
25773 .with_refresh_interval(StdDuration::from_millis(50));
25774 let guard = lock
25775 .acquire_guard(&request)
25776 .await
25777 .expect("pool metadata fence acquisition should not error")
25778 .expect("the stale writer should acquire the pool metadata fence");
25779 tokio::time::timeout(StdDuration::from_secs(2), guard.lock_lost_notified())
25780 .await
25781 .expect("the stale writer lease should expire");
25782
25783 let storage = Arc::new(PartialPoolMetaWriteStorage {
25784 fail_write: false,
25785 fail_after_first_write: false,
25786 pending_write: false,
25787 write_started: tokio::sync::Notify::new(),
25788 wrote: AtomicBool::new(false),
25789 revision: AtomicUsize::new(0),
25790 stored: StdMutex::new(None),
25791 identity: StdMutex::new(None),
25792 });
25793 let snapshot = PoolMeta {
25794 version: super::POOL_META_VERSION,
25795 pools: vec![decommission_test_pool_status(0, None)],
25796 ..Default::default()
25797 };
25798 let mut write_state = super::PoolMetaWriteState::default();
25799 let err = snapshot
25800 .save_no_lock_armed(vec![storage.clone()], &mut write_state, guard.lock_lost_signal(), &[0])
25801 .await
25802 .expect_err("a writer must not persist after losing the distributed pool metadata fence");
25803
25804 assert!(err.to_string().contains("distributed fence was lost"));
25805 assert!(!storage.wrote.load(Ordering::SeqCst), "the stale writer must not reach replica storage");
25806 write_state
25807 .ensure_write_safe("lost pool metadata fence")
25808 .expect("fence loss before dispatch has no unknown durable write and must remain retryable");
25809 }
25810
25811 #[tokio::test]
25812 async fn test_pool_meta_fence_loss_after_publish_keeps_write_gate_blocked() {
25813 let client = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
25814 let lock = NamespaceLock::with_clients_and_quorum("pool-meta-publish-fence-loss".to_string(), vec![client], 1);
25815 let request = LockRequest::new(
25816 ObjectKey::new(super::RUSTFS_META_BUCKET, super::POOL_META_NAME),
25817 LockType::Exclusive,
25818 "publishing-writer",
25819 )
25820 .with_acquire_timeout(StdDuration::from_secs(1))
25821 .with_ttl(StdDuration::from_secs(1))
25822 .with_refresh_interval(StdDuration::from_secs(1));
25823 let guard = lock
25824 .acquire_guard(&request)
25825 .await
25826 .expect("pool metadata fence acquisition should not error")
25827 .expect("the publishing writer should acquire the pool metadata fence");
25828 let storage = Arc::new(PartialPoolMetaWriteStorage {
25829 fail_write: false,
25830 fail_after_first_write: false,
25831 pending_write: false,
25832 write_started: tokio::sync::Notify::new(),
25833 wrote: AtomicBool::new(false),
25834 revision: AtomicUsize::new(0),
25835 stored: StdMutex::new(None),
25836 identity: StdMutex::new(None),
25837 });
25838 let mut saved = PoolMeta {
25839 version: super::POOL_META_VERSION,
25840 pools: vec![decommission_test_pool_status(0, None)],
25841 ..Default::default()
25842 };
25843 saved.pools[0].last_update = OffsetDateTime::UNIX_EPOCH + Duration::seconds(1);
25844 let mut current = saved.clone();
25845 current.pools[0].last_update = OffsetDateTime::UNIX_EPOCH;
25846 let mut write_state = super::PoolMetaWriteState::for_test_bootstrap();
25847 let outcome = saved
25848 .save_no_lock_armed(vec![storage], &mut write_state, guard.lock_lost_signal(), &[0])
25849 .await
25850 .expect("the replica save should finish while the fence is valid");
25851
25852 ensure_pool_meta_write_fence(&guard, "test pool metadata publish")
25853 .expect("the fence should remain valid before publication");
25854 publish_pool_meta_updates(&mut current, &saved, &[0]);
25855 tokio::time::timeout(StdDuration::from_secs(2), guard.lock_lost_notified())
25856 .await
25857 .expect("the fence should expire after publication");
25858 ensure_pool_meta_write_fence(&guard, "test pool metadata publish")
25859 .expect_err("the post-publication fence check must observe the loss");
25860
25861 assert_eq!(current.pools[0].last_update, saved.pools[0].last_update);
25862 drop(outcome);
25863 write_state
25864 .ensure_write_safe("lost pool metadata publish fence")
25865 .expect_err("the sticky write gate must remain armed after post-publication fence loss");
25866 }
25867
25868 #[tokio::test]
25869 async fn test_clear_decommission_transaction_survives_caller_abort() {
25870 let pool_meta = PoolMeta {
25871 version: super::POOL_META_VERSION,
25872 pools: vec![decommission_test_pool_status(
25873 0,
25874 Some(PoolDecommissionInfo {
25875 failed: true,
25876 ..Default::default()
25877 }),
25878 )],
25879 ..Default::default()
25880 };
25881 let store = decommission_worker_test_store(pool_meta, vec![None]);
25882 let save_started = Arc::new(tokio::sync::Notify::new());
25883 let save_release = Arc::new(tokio::sync::Notify::new());
25884 let save_completed = Arc::new(AtomicBool::new(false));
25885 let caller_store = store.clone();
25886 let caller_save_started = save_started.clone();
25887 let caller_save_release = save_release.clone();
25888 let caller_save_completed = save_completed.clone();
25889 let caller_task = tokio::spawn(async move {
25890 caller_store
25891 .clear_decommission_with_save(0, move || async move {
25892 caller_save_started.notify_one();
25893 caller_save_release.notified().await;
25894 caller_save_completed.store(true, Ordering::SeqCst);
25895 Ok(())
25896 })
25897 .await
25898 });
25899
25900 tokio::time::timeout(StdDuration::from_secs(1), save_started.notified())
25901 .await
25902 .expect("the clear transaction should reach persistence");
25903 caller_task.abort();
25904 assert!(
25905 caller_task
25906 .await
25907 .expect_err("the RPC waiter should be cancelled")
25908 .is_cancelled(),
25909 "the clear caller should be aborted while persistence is pending"
25910 );
25911 save_release.notify_one();
25912
25913 let _start_guard = tokio::time::timeout(StdDuration::from_secs(1), store.start_gate.lock())
25914 .await
25915 .expect("the detached clear transaction should finish");
25916 assert!(
25917 save_completed.load(Ordering::SeqCst),
25918 "the detached transaction must finish persistence after the caller is aborted"
25919 );
25920 let pool_meta = store.pool_meta.read().await;
25921 assert!(
25922 pool_meta.pools[0]
25923 .decommission
25924 .as_ref()
25925 .is_some_and(|info| !info.has_decommission_state())
25926 );
25927 }
25928
25929 fn decommission_test_pool_endpoint(idx: usize, is_local: bool) -> PoolEndpoints {
25930 let port = 9000usize + idx;
25931 let mut endpoint =
25932 Endpoint::try_from(format!("http://127.0.0.1:{port}/disk").as_str()).expect("test endpoint should parse");
25933 endpoint.is_local = is_local;
25934 endpoint.pool_idx = i32::try_from(idx).expect("test pool index should fit i32");
25935
25936 PoolEndpoints {
25937 legacy: false,
25938 set_count: 1,
25939 drives_per_set: 1,
25940 endpoints: Endpoints::from(vec![endpoint]),
25941 cmd_line: format!("pool-{idx}"),
25942 platform: String::new(),
25943 }
25944 }
25945
25946 fn decommission_test_pool_status(idx: usize, decommission: Option<PoolDecommissionInfo>) -> PoolStatus {
25947 PoolStatus {
25948 id: idx,
25949 cmd_line: format!("pool-{idx}"),
25950 last_update: OffsetDateTime::now_utc(),
25951 decommission,
25952 }
25953 }
25954
25955 fn decommission_test_active_model_meta(model_versions: &[u16]) -> PoolMeta {
25956 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
25957 PoolMeta {
25958 version: POOL_META_VERSION,
25959 pools: model_versions
25960 .iter()
25961 .copied()
25962 .enumerate()
25963 .map(|(idx, model_version)| {
25964 let reservation = build_decommission_capacity_reservation_with_model(
25965 DecommissionPoolCapacityInfo::for_test(idx, layout, 0, 10, 10),
25966 layout,
25967 uuid::Uuid::new_v4(),
25968 u64::try_from(idx).unwrap_or_default() + 1,
25969 OffsetDateTime::UNIX_EPOCH,
25970 model_version,
25971 )
25972 .expect("test capacity model should be supported");
25973 decommission_test_pool_status(
25974 idx,
25975 Some(PoolDecommissionInfo {
25976 start_time: Some(OffsetDateTime::UNIX_EPOCH),
25977 capacity_reservation: Some(reservation),
25978 ..Default::default()
25979 }),
25980 )
25981 })
25982 .collect(),
25983 ..Default::default()
25984 }
25985 }
25986
25987 fn decommission_test_cleanup_meta(
25988 model_version: u16,
25989 temporary_mutations: Vec<DecommissionCapacityTemporaryMutation>,
25990 inflight_physical_bytes: usize,
25991 pending: Option<(uuid::Uuid, usize)>,
25992 ) -> PoolMeta {
25993 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
25994 let mut reservation = build_decommission_capacity_reservation_with_model(
25995 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
25996 layout,
25997 uuid::Uuid::new_v4(),
25998 1,
25999 OffsetDateTime::UNIX_EPOCH,
26000 model_version,
26001 )
26002 .expect("test cleanup reservation should be valid");
26003 let (pending_mutation_id, pending_physical_bytes) = pending.unzip();
26004 let pending_physical_bytes = pending_physical_bytes.unwrap_or_default();
26005 reservation.observed_target_physical_bytes = inflight_physical_bytes;
26006 reservation.inflight_target_physical_bytes = inflight_physical_bytes;
26007 reservation.pending_target_physical_bytes = pending_physical_bytes;
26008 reservation.targets.push(DecommissionCapacityTarget {
26009 pool_index: 1,
26010 layout,
26011 physical_total_at_reservation: 200,
26012 physical_free_at_reservation: 200,
26013 reserved_physical_bytes: reservation.peak_physical_bytes,
26014 consumed_physical_bytes: 0,
26015 observed_physical_bytes: inflight_physical_bytes,
26016 inflight_physical_bytes,
26017 pending_physical_bytes,
26018 pending_mutation_id,
26019 temporary_mutations,
26020 });
26021 PoolMeta {
26022 version: POOL_META_VERSION,
26023 pools: vec![
26024 decommission_test_pool_status(
26025 0,
26026 Some(PoolDecommissionInfo {
26027 start_time: Some(OffsetDateTime::UNIX_EPOCH),
26028 capacity_reservation: Some(reservation),
26029 ..Default::default()
26030 }),
26031 ),
26032 decommission_test_pool_status(1, None),
26033 ],
26034 ..Default::default()
26035 }
26036 }
26037
26038 #[tokio::test]
26039 async fn test_activation_fence_uses_one_lock_order_and_serializes_callers() {
26040 let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
26041 let first = Arc::new(ActivationLockRecorder {
26042 lock_manager: Arc::clone(&manager),
26043 owner: "first",
26044 resources: StdMutex::new(Vec::new()),
26045 });
26046 let second = Arc::new(ActivationLockRecorder {
26047 lock_manager: manager,
26048 owner: "second",
26049 resources: StdMutex::new(Vec::new()),
26050 });
26051
26052 let first_guards = acquire_pool_rebalance_activation_locks(first.clone(), None)
26053 .await
26054 .expect("first activation should acquire both locks");
26055 assert_eq!(
26056 *first
26057 .resources
26058 .lock()
26059 .expect("activation lock recorder should not be poisoned"),
26060 vec![REBAL_META_NAME.to_string(), POOL_META_NAME.to_string()]
26061 );
26062
26063 let mut second_acquire = Box::pin(acquire_pool_rebalance_activation_locks(second.clone(), None));
26064 let mut context = Context::from_waker(futures::task::noop_waker_ref());
26065 assert!(matches!(second_acquire.as_mut().poll(&mut context), Poll::Pending));
26066
26067 drop(first_guards);
26068 second_acquire
26069 .await
26070 .expect("second activation should acquire both locks after the first releases them");
26071 assert_eq!(
26072 *second
26073 .resources
26074 .lock()
26075 .expect("activation lock recorder should not be poisoned"),
26076 vec![REBAL_META_NAME.to_string(), POOL_META_NAME.to_string()]
26077 );
26078 }
26079
26080 #[tokio::test]
26081 async fn test_activation_cancellation_releases_rebalance_fence_while_pool_fence_is_contended() {
26082 use crate::storage_api_contracts::namespace::NamespaceLocking as _;
26083 let pool = Arc::new(ActivationLockRecorder {
26084 lock_manager: Arc::new(rustfs_lock::GlobalLockManager::new()),
26085 owner: "activation-cancellation",
26086 resources: StdMutex::new(Vec::new()),
26087 });
26088 let pool_lock = pool
26089 .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, POOL_META_NAME)
26090 .await
26091 .expect("pool lock should be created");
26092 let pool_reader = pool_lock
26093 .get_read_lock(std::time::Duration::from_secs(5))
26094 .await
26095 .expect("ordinary mutation should hold the pool read fence");
26096 pool.resources.lock().expect("recorder should not be poisoned").clear();
26097 let mut activation = Box::pin(acquire_pool_rebalance_activation_locks(Arc::clone(&pool), None));
26098 assert!(matches!(futures::poll!(&mut activation), Poll::Pending));
26099 assert_eq!(
26100 *pool.resources.lock().expect("recorder should not be poisoned"),
26101 vec![REBAL_META_NAME.to_string(), POOL_META_NAME.to_string()],
26102 "activation must hold the run fence before waiting for the pool fence",
26103 );
26104 drop(activation);
26105 let rebalance_lock = pool
26106 .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME)
26107 .await
26108 .expect("run lock should be created");
26109 let run_writer = rebalance_lock
26110 .get_write_lock(std::time::Duration::from_secs(5))
26111 .await
26112 .expect("cancelling activation must release its already-acquired run fence");
26113 assert!(
26114 !pool_reader.is_released(),
26115 "cancelling activation must not release another caller's pool fence"
26116 );
26117 assert!(!run_writer.is_lock_lost());
26118 }
26119
26120 #[test]
26121 fn decommission_receipt_run_token_changes_with_persisted_start_time() {
26122 let first = OffsetDateTime::from_unix_timestamp(1_000).expect("first run timestamp should be valid");
26123 let second = OffsetDateTime::from_unix_timestamp(2_000).expect("second run timestamp should be valid");
26124 let first_token = decommission_durable_ilm_receipt_run_token("pool-0", first);
26125 let second_token = decommission_durable_ilm_receipt_run_token("pool-0", second);
26126
26127 assert_ne!(first_token, second_token);
26128 assert_eq!(first_token, decommission_durable_ilm_receipt_run_token("pool-0", first));
26129 let operation_id = "a".repeat(64);
26130 let old_receipt = decommission_durable_ilm_receipt_path(
26131 &first_token,
26132 &format!("ilm/tier-delete-journal/{operation_id}.json"),
26133 "operation_id",
26134 &operation_id,
26135 );
26136 assert!(!old_receipt.starts_with(&decommission_durable_ilm_receipt_run_prefix(&second_token)));
26137 }
26138
26139 #[test]
26140 fn decommission_recovery_control_receipt_path_round_trips() {
26141 let run_token = "b".repeat(64);
26142 let control_id = "a".repeat(64);
26143 let source_path = format!(
26144 "ilm/recovery-controls/transition_transaction/{}/{}/{}.json",
26145 &control_id[..2],
26146 &control_id[2..4],
26147 control_id
26148 );
26149 let path = decommission_durable_ilm_receipt_path(&run_token, &source_path, "control_id", &control_id);
26150
26151 let locator = parse_decommission_durable_ilm_receipt_path(&path).expect("recovery control receipt path should parse");
26152
26153 assert_eq!(locator.run_token, run_token);
26154 assert_eq!(locator.source_path, source_path);
26155 assert_eq!(locator.id_kind, "control_id");
26156 assert_eq!(locator.id, control_id);
26157 }
26158
26159 #[test]
26160 fn decommission_receipt_merge_preserves_terminal_proof() {
26161 let operation_id = "a".repeat(64);
26162 let source_path = format!("ilm/tier-delete-journal/{operation_id}.json");
26163 let checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal {
26164 content_sha256: "b".repeat(64),
26165 identity_sha256: "c".repeat(64),
26166 committed: false,
26167 dispatch_identity_sha256: None,
26168 state: None,
26169 };
26170 let terminal_checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal {
26171 content_sha256: "d".repeat(64),
26172 identity_sha256: "c".repeat(64),
26173 committed: true,
26174 dispatch_identity_sha256: None,
26175 state: None,
26176 };
26177 let incoming = DecommissionDurableIlmReceipt {
26178 source_path,
26179 namespace: "tier-delete-journal".to_string(),
26180 id_kind: "operation_id".to_string(),
26181 id: operation_id,
26182 checkpoint: checkpoint.clone(),
26183 terminal_checkpoint: None,
26184 fleet_topology_generation: None,
26185 };
26186 let existing = DecommissionDurableIlmReceipt {
26187 terminal_checkpoint: Some(terminal_checkpoint.clone()),
26188 ..incoming.clone()
26189 };
26190
26191 let merged = merge_decommission_durable_ilm_receipts(&existing, &incoming)
26192 .expect("retry receipt must merge with a terminal receipt");
26193
26194 assert_eq!(merged.checkpoint, checkpoint);
26195 assert_eq!(merged.terminal_checkpoint, Some(terminal_checkpoint.clone()));
26196
26197 let topology_bound = DecommissionDurableIlmReceipt {
26198 fleet_topology_generation: Some("e".repeat(64)),
26199 ..incoming
26200 };
26201 let mixed_error = merge_decommission_durable_ilm_receipts(&existing, &topology_bound)
26202 .expect_err("a topology-bound v6 receipt must not mask an unbound receipt")
26203 .to_string();
26204 assert!(mixed_error.contains("fleet topology conflict"));
26205
26206 let topology_existing = DecommissionDurableIlmReceipt {
26207 terminal_checkpoint: Some(terminal_checkpoint),
26208 ..topology_bound.clone()
26209 };
26210 let topology_merged = merge_decommission_durable_ilm_receipts(&topology_existing, &topology_bound)
26211 .expect("receipts bound to the same fleet topology should merge");
26212 assert_eq!(topology_merged.fleet_topology_generation, Some("e".repeat(64)));
26213 }
26214
26215 #[test]
26216 fn decommission_manual_job_receipt_compacts_large_progress() {
26217 let prefix = "p".repeat(12 * 1024);
26218 let options = ManualTransitionRunOptions {
26219 prefix,
26220 ..Default::default()
26221 };
26222 let mut job = ManualTransitionJobRecord::new(uuid::Uuid::new_v4(), "bounded-receipt-bucket", &options, "owner");
26223 let token_bytes = serde_json::to_vec(&serde_json::json!({
26224 "marker": "m".repeat(12 * 1024),
26225 "version_marker": "opaque-version"
26226 }))
26227 .expect("large continuation token should encode");
26228 let mut report = job.report.clone();
26229 report.scanned = 1;
26230 report.continuation_token = Some(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&token_bytes));
26231 job.update_running_progress(report, ManualTransitionQueueSnapshot::default());
26232 let path = manual_transition_job_record_object_name(job.job_id).expect("manual job path should build");
26233 let job_bytes = job.encode().expect("large manual job should remain within its record limit");
26234 assert!(job_bytes.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE);
26235 let record = validate_durable_ilm_record(&path, &job_bytes).expect("large manual job should validate");
26236 let expected_checkpoint = record.checkpoint.clone();
26237 let mut receipt = DecommissionDurableIlmReceipt::new(&path, &record, None);
26238 receipt.terminal_checkpoint = Some(record.checkpoint);
26239
26240 let encoded = receipt.encode().expect("bounded progress proof should fit the receipt limit");
26241 let decoded = DecommissionDurableIlmReceipt::decode(&encoded).expect("bounded receipt should round trip");
26242
26243 assert!(encoded.len() <= DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE);
26244 assert_eq!(decoded.source_path, path);
26245 assert_eq!(decoded.checkpoint, expected_checkpoint);
26246 assert_eq!(decoded.terminal_checkpoint, Some(expected_checkpoint));
26247 }
26248
26249 #[test]
26250 fn test_apply_decommission_status_space_info_adds_idle_pool_usage() {
26251 let status = apply_decommission_status_space_info(
26252 decommission_test_pool_status(0, None),
26253 PoolSpaceInfo {
26254 free: 25,
26255 total: 100,
26256 used: 75,
26257 },
26258 );
26259
26260 let decommission = status.decommission.expect("idle pool status should include usage info");
26261 assert_eq!(decommission.total_size, 100);
26262 assert_eq!(decommission.current_size, 25);
26263 assert!(decommission.start_time.is_none());
26264 assert!(!decommission.complete);
26265 assert!(!decommission.failed);
26266 assert!(!decommission.canceled);
26267 }
26268
26269 #[test]
26270 fn test_apply_decommission_status_space_info_refreshes_active_decommission_sizes() {
26271 let status = apply_decommission_status_space_info(
26272 decommission_test_pool_status(
26273 0,
26274 Some(PoolDecommissionInfo {
26275 total_size: 1,
26276 current_size: 1,
26277 ..Default::default()
26278 }),
26279 ),
26280 PoolSpaceInfo {
26281 free: 25,
26282 total: 100,
26283 used: 75,
26284 },
26285 );
26286
26287 let decommission = status.decommission.expect("active decommission info should remain present");
26288 assert_eq!(decommission.total_size, 100);
26289 assert_eq!(decommission.current_size, 25);
26290 }
26291
26292 #[test]
26293 fn test_merge_pool_status_refresh_uses_persisted_terminal_decommission() {
26294 let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
26295 let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
26296 let mut current = PoolMeta {
26297 pools: vec![PoolStatus {
26298 id: 0,
26299 cmd_line: "pool-0".to_string(),
26300 last_update: older,
26301 decommission: Some(PoolDecommissionInfo {
26302 start_time: Some(older),
26303 ..Default::default()
26304 }),
26305 }],
26306 ..Default::default()
26307 };
26308 let persisted = PoolMeta {
26309 pools: vec![PoolStatus {
26310 id: 0,
26311 cmd_line: "pool-0".to_string(),
26312 last_update: newer,
26313 decommission: Some(PoolDecommissionInfo {
26314 complete: true,
26315 ..Default::default()
26316 }),
26317 }],
26318 ..Default::default()
26319 };
26320
26321 assert!(merge_pool_status_refresh(&mut current, persisted, &[false]));
26322
26323 let info = current.pools[0]
26324 .decommission
26325 .as_ref()
26326 .expect("decommission info should be present");
26327 assert!(info.complete);
26328 assert!(!info.failed);
26329 assert!(!info.canceled);
26330 }
26331
26332 #[test]
26333 fn test_merge_pool_status_refresh_keeps_newer_local_active_progress() {
26334 let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
26335 let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
26336 let mut current = PoolMeta {
26337 pools: vec![PoolStatus {
26338 id: 0,
26339 cmd_line: "pool-0".to_string(),
26340 last_update: newer,
26341 decommission: Some(PoolDecommissionInfo {
26342 start_time: Some(older),
26343 items_decommissioned: 10,
26344 bytes_done: 1_024,
26345 ..Default::default()
26346 }),
26347 }],
26348 ..Default::default()
26349 };
26350 let persisted = PoolMeta {
26351 pools: vec![PoolStatus {
26352 id: 0,
26353 cmd_line: "pool-0".to_string(),
26354 last_update: older,
26355 decommission: Some(PoolDecommissionInfo {
26356 start_time: Some(older),
26357 items_decommissioned: 1,
26358 bytes_done: 128,
26359 ..Default::default()
26360 }),
26361 }],
26362 ..Default::default()
26363 };
26364
26365 assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
26366
26367 let info = current.pools[0]
26368 .decommission
26369 .as_ref()
26370 .expect("local decommission info should remain present");
26371 assert_eq!(info.items_decommissioned, 10);
26372 assert_eq!(info.bytes_done, 1_024);
26373 }
26374
26375 #[test]
26376 fn test_merge_pool_status_refresh_preserves_observed_v2_floor() {
26377 let timestamp = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
26378 let mut current = PoolMeta {
26379 version: POOL_META_V1_VERSION,
26380 pools: vec![PoolStatus {
26381 id: 0,
26382 cmd_line: "pool-0".to_string(),
26383 last_update: timestamp,
26384 decommission: Some(PoolDecommissionInfo::default()),
26385 }],
26386 dont_save: false,
26387 };
26388 let persisted = PoolMeta {
26389 version: POOL_META_VERSION,
26390 pools: current.pools.clone(),
26391 dont_save: false,
26392 };
26393
26394 assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
26395 assert_eq!(current.version, POOL_META_VERSION);
26396 let encoded = current
26397 .encode_config_data_for_v2_gate(false)
26398 .expect("a peer-observed v2 floor must remain sticky");
26399 assert_eq!(LittleEndian::read_u16(&encoded[2..4]), POOL_META_VERSION);
26400 }
26401
26402 #[test]
26403 fn test_merge_pool_status_refresh_keeps_newer_local_active_over_older_terminal() {
26404 let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
26405 let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
26406 let mut current = PoolMeta {
26407 pools: vec![PoolStatus {
26408 id: 0,
26409 cmd_line: "pool-0".to_string(),
26410 last_update: newer,
26411 decommission: Some(PoolDecommissionInfo {
26412 start_time: Some(older),
26413 items_decommissioned: 10,
26414 bytes_done: 1_024,
26415 ..Default::default()
26416 }),
26417 }],
26418 ..Default::default()
26419 };
26420 let persisted = PoolMeta {
26421 pools: vec![PoolStatus {
26422 id: 0,
26423 cmd_line: "pool-0".to_string(),
26424 last_update: older,
26425 decommission: Some(PoolDecommissionInfo {
26426 failed: true,
26427 ..Default::default()
26428 }),
26429 }],
26430 ..Default::default()
26431 };
26432
26433 assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
26434
26435 let info = current.pools[0]
26436 .decommission
26437 .as_ref()
26438 .expect("local active decommission info should remain present");
26439 assert!(!info.failed);
26440 assert_eq!(info.items_decommissioned, 10);
26441 assert_eq!(info.bytes_done, 1_024);
26442 }
26443
26444 #[test]
26445 fn test_merge_pool_status_refresh_fails_closed_on_missing_persisted_pools() {
26446 let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
26447 let mut current = PoolMeta {
26448 pools: vec![decommission_test_pool_status(
26449 0,
26450 Some(PoolDecommissionInfo {
26451 complete: true,
26452 ..Default::default()
26453 }),
26454 )],
26455 ..Default::default()
26456 };
26457 current.pools[0].last_update = newer;
26458
26459 assert!(
26460 !merge_pool_status_refresh(&mut current, PoolMeta::default(), &[false]),
26461 "an empty persisted snapshot must fail closed instead of replacing local state"
26462 );
26463
26464 let info = current.pools[0]
26465 .decommission
26466 .as_ref()
26467 .expect("local decommission info should survive a missing snapshot");
26468 assert!(info.complete);
26469 assert_eq!(current.pools[0].last_update, newer);
26470 }
26471
26472 #[test]
26473 fn test_merge_pool_status_refresh_ignores_mislabeled_pool_entries() {
26474 let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
26475 let mut current = PoolMeta {
26476 pools: vec![decommission_test_pool_status(0, None)],
26477 ..Default::default()
26478 };
26479 let mut persisted = PoolMeta {
26480 pools: vec![decommission_test_pool_status(0, Some(PoolDecommissionInfo::default()))],
26481 ..Default::default()
26482 };
26483 persisted.pools[0].id = 7;
26484 persisted.pools[0].last_update = older;
26485
26486 assert!(
26487 !merge_pool_status_refresh(&mut current, persisted, &[false]),
26488 "a pool entry whose id does not match its index must be ignored"
26489 );
26490 assert!(current.pools[0].decommission.is_none());
26491 }
26492
26493 #[test]
26494 fn test_pool_meta_save_merge_preserves_newer_untouched_pool() {
26495 let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
26496 let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
26497 let mut current = PoolMeta {
26498 pools: vec![
26499 PoolStatus {
26500 id: 0,
26501 cmd_line: "pool-0".to_string(),
26502 last_update: older,
26503 decommission: Some(PoolDecommissionInfo {
26504 failed: true,
26505 ..Default::default()
26506 }),
26507 },
26508 PoolStatus {
26509 id: 1,
26510 cmd_line: "pool-1".to_string(),
26511 last_update: older,
26512 decommission: None,
26513 },
26514 ],
26515 ..Default::default()
26516 };
26517 let mut persisted = current.clone();
26518 persisted.pools[1].last_update = newer;
26519 persisted.pools[1].decommission = Some(PoolDecommissionInfo {
26520 failed: true,
26521 ..Default::default()
26522 });
26523
26524 assert!(current.clear_decommission(0).expect("terminal decommission should clear"));
26525 merge_pool_meta_updates_for_save(&mut persisted, ¤t, &[0], "clear decommission")
26526 .expect("the target pool update should merge into the latest snapshot");
26527
26528 assert!(
26529 persisted.pools[0]
26530 .decommission
26531 .as_ref()
26532 .is_some_and(|info| !info.has_decommission_state())
26533 );
26534 assert_eq!(persisted.pools[1].last_update, newer);
26535 assert!(persisted.pools[1].decommission.as_ref().is_some_and(|info| info.failed));
26536 }
26537
26538 #[test]
26539 fn test_pool_meta_save_merge_rejects_stale_terminal_resurrection() {
26540 let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
26541 let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
26542 let mut persisted = PoolMeta {
26543 pools: vec![PoolStatus {
26544 id: 0,
26545 cmd_line: "pool-0".to_string(),
26546 last_update: newer,
26547 decommission: Some(PoolDecommissionInfo {
26548 canceled: true,
26549 ..Default::default()
26550 }),
26551 }],
26552 ..Default::default()
26553 };
26554 let stale = PoolMeta {
26555 pools: vec![PoolStatus {
26556 id: 0,
26557 cmd_line: "pool-0".to_string(),
26558 last_update: older,
26559 decommission: Some(PoolDecommissionInfo {
26560 start_time: Some(older),
26561 ..Default::default()
26562 }),
26563 }],
26564 ..Default::default()
26565 };
26566
26567 let err = merge_pool_meta_updates_for_save(&mut persisted, &stale, &[0], "stale writer")
26568 .expect_err("a stale active snapshot must not resurrect a canceled pool");
26569
26570 assert!(err.to_string().contains("stale pool metadata update rejected"));
26571 assert!(persisted.pools[0].decommission.as_ref().is_some_and(|info| info.canceled));
26572 }
26573
26574 #[test]
26575 fn test_pool_meta_save_merge_rejects_clear_over_completed_or_unresolved_state() {
26576 let timestamp = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
26577 for persisted_info in [
26578 PoolDecommissionInfo {
26579 complete: true,
26580 ..Default::default()
26581 },
26582 PoolDecommissionInfo {
26583 failed: true,
26584 unresolved_entries: vec![DecommissionUnresolvedEntry {
26585 bucket: "bucket-a".to_string(),
26586 object: "object-a".to_string(),
26587 pool_index: 0,
26588 set_index: 0,
26589 source_generation: timestamp,
26590 candidate_count: 1,
26591 disk_error_count: 1,
26592 observed_at: timestamp,
26593 reason: "test unresolved entry".to_string(),
26594 }],
26595 ..Default::default()
26596 },
26597 ] {
26598 let mut persisted = PoolMeta {
26599 pools: vec![PoolStatus {
26600 id: 0,
26601 cmd_line: "pool-0".to_string(),
26602 last_update: timestamp,
26603 decommission: Some(persisted_info),
26604 }],
26605 ..Default::default()
26606 };
26607 let cleared = PoolMeta {
26608 pools: vec![PoolStatus {
26609 id: 0,
26610 cmd_line: "pool-0".to_string(),
26611 last_update: OffsetDateTime::now_utc(),
26612 decommission: Some(PoolDecommissionInfo::default()),
26613 }],
26614 ..Default::default()
26615 };
26616
26617 let err = merge_pool_meta_updates_for_save(&mut persisted, &cleared, &[0], "clear decommission")
26618 .expect_err("a stale clear must not erase completed or unresolved state");
26619
26620 assert!(
26621 err.to_string()
26622 .contains("completed or unresolved decommission state cannot be cleared")
26623 );
26624 }
26625 }
26626
26627 #[test]
26628 fn test_pool_meta_publish_preserves_untouched_runtime_progress() {
26629 let mut current = PoolMeta {
26630 pools: vec![
26631 decommission_test_pool_status(0, None),
26632 decommission_test_pool_status(
26633 1,
26634 Some(PoolDecommissionInfo {
26635 items_decommissioned: 10,
26636 bytes_done: 1_024,
26637 ..Default::default()
26638 }),
26639 ),
26640 ],
26641 ..Default::default()
26642 };
26643 let mut saved = current.clone();
26644 saved.pools[0].decommission = Some(PoolDecommissionInfo {
26645 complete: true,
26646 ..Default::default()
26647 });
26648 let saved_pool_1 = saved.pools[1].decommission.as_mut().expect("pool 1 progress should exist");
26649 saved_pool_1.items_decommissioned = 1;
26650 saved_pool_1.bytes_done = 128;
26651
26652 publish_pool_meta_updates(&mut current, &saved, &[0]);
26653
26654 assert!(current.pools[0].decommission.as_ref().is_some_and(|info| info.complete));
26655 let pool_1 = current.pools[1]
26656 .decommission
26657 .as_ref()
26658 .expect("pool 1 progress should remain present");
26659 assert_eq!(pool_1.items_decommissioned, 10);
26660 assert_eq!(pool_1.bytes_done, 1_024);
26661 }
26662
26663 #[test]
26664 fn test_dedup_indices_removes_duplicates_preserving_order() {
26665 assert_eq!(dedup_indices(&[0, 2, 1, 2, 3, 0]), vec![0, 2, 1, 3]);
26666 }
26667
26668 #[test]
26669 fn test_dedup_indices_handles_empty_input() {
26670 let empty: Vec<usize> = Vec::new();
26671 assert!(dedup_indices(&empty).is_empty());
26672 }
26673
26674 #[test]
26675 fn test_default_decommission_bucket_concurrency_is_conservative() {
26676 assert_eq!(default_decommission_bucket_concurrency(0), 1);
26677 assert_eq!(default_decommission_bucket_concurrency(1), 1);
26678 assert_eq!(default_decommission_bucket_concurrency(2), 2);
26679 assert_eq!(default_decommission_bucket_concurrency(8), 4);
26680 }
26681
26682 #[test]
26683 fn test_default_decommission_entry_concurrency_is_conservative() {
26684 assert_eq!(default_decommission_entry_concurrency(0), 1);
26685 assert_eq!(default_decommission_entry_concurrency(1), 1);
26686 assert_eq!(default_decommission_entry_concurrency(4), 4);
26687 assert_eq!(default_decommission_entry_concurrency(16), DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP);
26688 }
26689
26690 #[test]
26691 fn test_decommission_entry_concurrency_clamps_operator_configuration() {
26692 assert_eq!(clamp_decommission_entry_concurrency(0), 1);
26693 assert_eq!(clamp_decommission_entry_concurrency(1), 1);
26694 assert_eq!(
26695 clamp_decommission_entry_concurrency(DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP),
26696 DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP
26697 );
26698 assert_eq!(clamp_decommission_entry_concurrency(usize::MAX), DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP);
26699 }
26700
26701 #[test]
26702 fn test_split_decommission_buckets_keeps_meta_buckets_last() {
26703 let (regular, meta) = split_decommission_buckets(vec![
26704 DecomBucketInfo {
26705 name: "bucket-a".to_string(),
26706 ..Default::default()
26707 },
26708 DecomBucketInfo {
26709 name: crate::disk::RUSTFS_META_BUCKET.to_string(),
26710 prefix: crate::config::com::CONFIG_PREFIX.to_string(),
26711 },
26712 DecomBucketInfo {
26713 name: "bucket-b".to_string(),
26714 ..Default::default()
26715 },
26716 DecomBucketInfo {
26717 name: crate::disk::RUSTFS_META_BUCKET.to_string(),
26718 prefix: crate::disk::BUCKET_META_PREFIX.to_string(),
26719 },
26720 DecomBucketInfo {
26721 name: crate::disk::RUSTFS_META_BUCKET.to_string(),
26722 prefix: crate::bucket::lifecycle::ILM_META_PREFIX.to_string(),
26723 },
26724 ]);
26725
26726 assert_eq!(
26727 regular.iter().map(|bucket| bucket.name.as_str()).collect::<Vec<_>>(),
26728 vec!["bucket-a", "bucket-b",]
26729 );
26730 assert_eq!(
26731 meta.iter().map(|bucket| bucket.prefix.as_str()).collect::<Vec<_>>(),
26732 vec![
26733 crate::config::com::CONFIG_PREFIX,
26734 crate::disk::BUCKET_META_PREFIX,
26735 crate::bucket::lifecycle::ILM_META_PREFIX,
26736 ]
26737 );
26738 }
26739
26740 #[test]
26741 fn test_resume_reconciles_missing_decommission_meta_prefixes() {
26742 let mut meta = PoolMeta {
26743 pools: vec![decommission_test_pool_status(
26744 0,
26745 Some(PoolDecommissionInfo {
26746 queued_buckets: vec![
26747 format!("{}/{}", crate::disk::RUSTFS_META_BUCKET, crate::config::com::CONFIG_PREFIX),
26748 format!("{}/{}", crate::disk::RUSTFS_META_BUCKET, crate::disk::BUCKET_META_PREFIX),
26749 ],
26750 ..Default::default()
26751 }),
26752 )],
26753 ..Default::default()
26754 };
26755
26756 assert!(reconcile_decommission_meta_buckets(&mut meta, 0));
26757 assert_eq!(
26758 meta.pending_buckets(0)
26759 .iter()
26760 .filter(|bucket| bucket.name == crate::disk::RUSTFS_META_BUCKET)
26761 .map(|bucket| bucket.prefix.as_str())
26762 .collect::<Vec<_>>(),
26763 DECOMMISSION_META_PREFIXES
26764 );
26765 assert!(!reconcile_decommission_meta_buckets(&mut meta, 0));
26766 }
26767
26768 #[tokio::test]
26769 async fn test_decommission_metadata_phase_precedes_regular_failure() {
26770 let events = Arc::new(StdMutex::new(Vec::new()));
26771 let err = run_decommission_phases(
26772 CancellationToken::new(),
26773 vec![
26774 DecomBucketInfo {
26775 name: "regular-fails".to_string(),
26776 ..Default::default()
26777 },
26778 DecomBucketInfo {
26779 name: "regular-not-started".to_string(),
26780 ..Default::default()
26781 },
26782 ],
26783 vec![
26784 DecomBucketInfo {
26785 name: crate::disk::RUSTFS_META_BUCKET.to_string(),
26786 prefix: crate::config::com::CONFIG_PREFIX.to_string(),
26787 },
26788 DecomBucketInfo {
26789 name: crate::disk::RUSTFS_META_BUCKET.to_string(),
26790 prefix: crate::disk::BUCKET_META_PREFIX.to_string(),
26791 },
26792 ],
26793 1,
26794 {
26795 let events = Arc::clone(&events);
26796 move |bucket, _rx| {
26797 let events = Arc::clone(&events);
26798 Box::pin(async move {
26799 let event = if bucket.name == crate::disk::RUSTFS_META_BUCKET {
26800 format!("meta:{}", bucket.prefix)
26801 } else {
26802 format!("regular:{}", bucket.name)
26803 };
26804 events.lock().expect("phase event lock should not be poisoned").push(event);
26805 if bucket.name == "regular-fails" {
26806 Err(Error::SlowDown)
26807 } else {
26808 Ok(())
26809 }
26810 })
26811 }
26812 },
26813 )
26814 .await
26815 .expect_err("regular failure should remain fatal after metadata completes");
26816
26817 assert!(matches!(err, Error::SlowDown));
26818 assert_eq!(
26819 *events.lock().expect("phase event lock should not be poisoned"),
26820 vec![
26821 format!("meta:{}", crate::config::com::CONFIG_PREFIX),
26822 format!("meta:{}", crate::disk::BUCKET_META_PREFIX),
26823 "regular:regular-fails".to_string(),
26824 ]
26825 );
26826 }
26827
26828 #[tokio::test]
26829 async fn test_run_decommission_buckets_bounded_respects_limit() {
26830 let rx = CancellationToken::new();
26831 let running = Arc::new(AtomicUsize::new(0));
26832 let max_running = Arc::new(AtomicUsize::new(0));
26833 let started = Arc::new(AtomicUsize::new(0));
26834 let buckets = (0..8)
26835 .map(|idx| DecomBucketInfo {
26836 name: format!("bucket-{idx}"),
26837 ..Default::default()
26838 })
26839 .collect::<Vec<_>>();
26840
26841 run_decommission_buckets_bounded(rx, buckets, 2, {
26842 let running = Arc::clone(&running);
26843 let max_running = Arc::clone(&max_running);
26844 let started = Arc::clone(&started);
26845 move |_bucket, _rx| {
26846 let running = Arc::clone(&running);
26847 let max_running = Arc::clone(&max_running);
26848 let started = Arc::clone(&started);
26849 Box::pin(async move {
26850 started.fetch_add(1, Ordering::SeqCst);
26851 let current = running.fetch_add(1, Ordering::SeqCst) + 1;
26852 max_running.fetch_max(current, Ordering::SeqCst);
26853 tokio::time::sleep(StdDuration::from_millis(10)).await;
26854 running.fetch_sub(1, Ordering::SeqCst);
26855 Ok(())
26856 })
26857 }
26858 })
26859 .await
26860 .expect("bounded bucket scheduler should complete");
26861
26862 assert_eq!(started.load(Ordering::SeqCst), 8);
26863 assert_eq!(max_running.load(Ordering::SeqCst), 2);
26864 assert_eq!(running.load(Ordering::SeqCst), 0);
26865 }
26866
26867 #[tokio::test]
26868 async fn test_run_decommission_buckets_bounded_cancels_and_stops_launching_after_failure() {
26869 let rx = CancellationToken::new();
26870 let started = Arc::new(AtomicUsize::new(0));
26871 let observed_cancel = Arc::new(AtomicBool::new(false));
26872 let buckets = (0..5)
26873 .map(|idx| DecomBucketInfo {
26874 name: format!("bucket-{idx}"),
26875 ..Default::default()
26876 })
26877 .collect::<Vec<_>>();
26878
26879 let err = tokio::time::timeout(
26880 StdDuration::from_secs(2),
26881 run_decommission_buckets_bounded(rx.clone(), buckets, 2, {
26882 let started = Arc::clone(&started);
26883 let observed_cancel = Arc::clone(&observed_cancel);
26884 move |bucket, rx| {
26885 let started = Arc::clone(&started);
26886 let observed_cancel = Arc::clone(&observed_cancel);
26887 Box::pin(async move {
26888 started.fetch_add(1, Ordering::SeqCst);
26889 if bucket.name == "bucket-0" {
26890 while started.load(Ordering::SeqCst) < 2 {
26891 tokio::task::yield_now().await;
26892 }
26893 return Err(Error::SlowDown);
26894 }
26895
26896 rx.cancelled().await;
26897 observed_cancel.store(true, Ordering::SeqCst);
26898 Ok(())
26899 })
26900 }
26901 }),
26902 )
26903 .await
26904 .expect("bucket scheduler should not hang after a bucket failure")
26905 .expect_err("first bucket failure should be returned");
26906
26907 assert!(matches!(err, Error::SlowDown));
26908 assert!(rx.is_cancelled());
26909 assert!(observed_cancel.load(Ordering::SeqCst));
26910 assert_eq!(started.load(Ordering::SeqCst), 2);
26911 }
26912
26913 #[tokio::test]
26914 async fn test_run_decommission_buckets_bounded_external_cancel_stops_pending_buckets() {
26915 let rx = CancellationToken::new();
26916 let started = Arc::new(AtomicUsize::new(0));
26917 let buckets = (0..4)
26918 .map(|idx| DecomBucketInfo {
26919 name: format!("bucket-{idx}"),
26920 ..Default::default()
26921 })
26922 .collect::<Vec<_>>();
26923
26924 let err = run_decommission_buckets_bounded(rx.clone(), buckets, 1, {
26925 let started = Arc::clone(&started);
26926 move |_bucket, rx| {
26927 let started = Arc::clone(&started);
26928 Box::pin(async move {
26929 started.fetch_add(1, Ordering::SeqCst);
26930 rx.cancel();
26931 Ok(())
26932 })
26933 }
26934 })
26935 .await
26936 .expect_err("external cancellation with pending buckets should stop the scheduler");
26937
26938 assert!(matches!(err, Error::OperationCanceled));
26939 assert!(rx.is_cancelled());
26940 assert_eq!(started.load(Ordering::SeqCst), 1);
26941 }
26942
26943 #[tokio::test]
26944 async fn test_wait_decommission_worker_drain_waits_for_entry_permit() {
26945 let workers = Arc::new(Semaphore::new(1));
26946 let permit = workers
26947 .clone()
26948 .acquire_owned()
26949 .await
26950 .expect("test worker permit should acquire");
26951
26952 let drain = tokio::spawn({
26953 let workers = workers.clone();
26954 async move { wait_decommission_worker_drain(&workers, 1).await }
26955 });
26956
26957 tokio::task::yield_now().await;
26958 assert!(!drain.is_finished(), "drain should wait while a worker permit is held");
26959
26960 drop(permit);
26961 let result = tokio::time::timeout(StdDuration::from_secs(1), drain)
26962 .await
26963 .expect("drain should finish after permit release")
26964 .expect("drain task should not panic");
26965 assert!(result.is_ok());
26966 }
26967
26968 #[test]
26969 fn test_decommission_entry_queue_capacity_is_bounded() {
26970 assert_eq!(decommission_entry_queue_capacity(0), 1);
26971 assert_eq!(decommission_entry_queue_capacity(1), 2);
26972 assert_eq!(
26973 decommission_entry_queue_capacity(DECOMMISSION_ENTRY_QUEUE_HARD_CAP),
26974 DECOMMISSION_ENTRY_QUEUE_HARD_CAP
26975 );
26976 assert_eq!(decommission_entry_queue_capacity(usize::MAX), DECOMMISSION_ENTRY_QUEUE_HARD_CAP);
26977 }
26978
26979 #[tokio::test]
26980 async fn test_drain_decommission_entry_queue_waits_for_all_outstanding_entries() {
26981 let outstanding = Arc::new(Semaphore::new(1));
26982 let held = outstanding
26983 .clone()
26984 .acquire_owned()
26985 .await
26986 .expect("test outstanding permit should acquire");
26987 let rx = CancellationToken::new();
26988 let drain = tokio::spawn({
26989 let outstanding = outstanding.clone();
26990 let rx = rx.clone();
26991 async move { drain_decommission_entry_queue(&rx, &outstanding, 1).await }
26992 });
26993
26994 tokio::task::yield_now().await;
26995 assert!(!drain.is_finished(), "queue drain must wait for active entry work");
26996 drop(held);
26997
26998 let drained = tokio::time::timeout(StdDuration::from_secs(1), drain)
26999 .await
27000 .expect("queue drain should finish after entry completion")
27001 .expect("queue drain task should not panic");
27002 assert!(!drained);
27003 }
27004
27005 #[tokio::test]
27006 async fn test_enqueue_decommission_entry_observes_cancellation_when_queue_is_full() {
27007 let outstanding = Arc::new(Semaphore::new(2));
27008 let (tx, mut queue) = tokio::sync::mpsc::channel(1);
27009 let held = outstanding
27010 .clone()
27011 .acquire_owned()
27012 .await
27013 .expect("first queue permit should acquire");
27014 tx.send(QueuedDecommissionEntry {
27015 entry: MetaCacheEntry::default(),
27016 queue_permit: held,
27017 })
27018 .await
27019 .expect("first entry should fill the queue");
27020
27021 let rx = CancellationToken::new();
27022 let enqueue = tokio::spawn({
27023 let rx = rx.clone();
27024 let outstanding = outstanding.clone();
27025 let tx = tx.clone();
27026 async move { enqueue_decommission_entry(&rx, &outstanding, &tx, MetaCacheEntry::default()).await }
27027 });
27028
27029 tokio::task::yield_now().await;
27030 rx.cancel();
27031 let result = tokio::time::timeout(StdDuration::from_secs(1), enqueue)
27032 .await
27033 .expect("full queue enqueue should observe cancellation")
27034 .expect("enqueue task should not panic");
27035 assert!(matches!(result, DecommissionEntryEnqueueResult::Canceled));
27036 drop(queue.recv().await);
27037 }
27038
27039 #[tokio::test]
27040 async fn test_decommission_side_effect_gate_quiesces_before_transition() {
27041 let operation_gate = Arc::new(tokio::sync::RwLock::new(()));
27042 let rx = CancellationToken::new();
27043 let started = Arc::new(tokio::sync::Notify::new());
27044 let release = Arc::new(tokio::sync::Notify::new());
27045 let operation = tokio::spawn({
27046 let operation_gate = operation_gate.clone();
27047 let rx = rx.clone();
27048 let started = started.clone();
27049 let release = release.clone();
27050 async move {
27051 run_decommission_side_effect(&rx, &operation_gate, || async {
27052 started.notify_one();
27053 release.notified().await;
27054 Ok::<_, Error>(())
27055 })
27056 .await
27057 }
27058 });
27059
27060 started.notified().await;
27061 rx.cancel();
27062 let transition = tokio::spawn({
27063 let operation_gate = operation_gate.clone();
27064 async move {
27065 let _guard = operation_gate.write().await;
27066 }
27067 });
27068 tokio::task::yield_now().await;
27069 assert!(!transition.is_finished(), "transition must wait for the in-flight side effect");
27070
27071 release.notify_one();
27072 let operation_result = operation.await.expect("operation task should not panic");
27073 assert!(matches!(operation_result, Err(Error::OperationCanceled)));
27074 transition.await.expect("transition task should not panic");
27075
27076 let called = Arc::new(AtomicBool::new(false));
27077 let result = run_decommission_side_effect(&rx, &operation_gate, {
27078 let called = called.clone();
27079 move || async move {
27080 called.store(true, Ordering::SeqCst);
27081 Ok::<_, Error>(())
27082 }
27083 })
27084 .await;
27085 assert!(matches!(result, Err(Error::OperationCanceled)));
27086 assert!(!called.load(Ordering::SeqCst));
27087 }
27088
27089 #[tokio::test]
27090 async fn test_decommission_side_effect_stops_after_pool_meta_write_block() {
27091 let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
27092 store
27093 .pool_meta_save_gate
27094 .lock()
27095 .await
27096 .observe_replicas(super::PoolMetaReplicaState {
27097 needs_repair: true,
27098 repair_write_safe: false,
27099 });
27100 let called = Arc::new(AtomicBool::new(false));
27101 let operation_gate = store.ctx.data_movement_operation_gate();
27102
27103 let result = store
27104 .run_guarded_decommission_side_effect(&CancellationToken::new(), &operation_gate, {
27105 let called = called.clone();
27106 move || async move {
27107 called.store(true, Ordering::SeqCst);
27108 Ok::<_, Error>(())
27109 }
27110 })
27111 .await;
27112
27113 assert!(
27114 result
27115 .expect_err("sticky pool metadata state must block new movement")
27116 .to_string()
27117 .contains("restart after all replicas are readable and consistent")
27118 );
27119 assert!(!called.load(Ordering::SeqCst));
27120 }
27121
27122 #[tokio::test]
27123 async fn test_decommission_reservation_stops_before_canceler_slot_when_pool_meta_is_blocked() {
27124 let generation = OffsetDateTime::UNIX_EPOCH;
27125 let pool_meta = PoolMeta {
27126 version: super::POOL_META_VERSION,
27127 pools: vec![decommission_test_pool_status(
27128 0,
27129 Some(PoolDecommissionInfo {
27130 start_time: Some(generation),
27131 ..Default::default()
27132 }),
27133 )],
27134 ..Default::default()
27135 };
27136 let store = decommission_worker_test_store(pool_meta, vec![None]);
27137 store
27138 .pool_meta_save_gate
27139 .lock()
27140 .await
27141 .observe_replicas(super::PoolMetaReplicaState {
27142 needs_repair: true,
27143 repair_write_safe: false,
27144 });
27145
27146 let err = store
27147 .reserve_decommission_routines(&CancellationToken::new(), &[0])
27148 .await
27149 .err()
27150 .expect("sticky pool metadata state must block worker reservation");
27151
27152 assert!(
27153 err.to_string()
27154 .contains("restart after all replicas are readable and consistent")
27155 );
27156 assert!(store.decommission_cancelers.read().await[0].is_none());
27157 }
27158
27159 #[tokio::test]
27160 async fn test_v1_unresolved_ledger_rejection_keeps_live_state_and_write_gate_safe() {
27161 let generation = OffsetDateTime::UNIX_EPOCH;
27162 let status = decommission_test_pool_status(
27163 0,
27164 Some(PoolDecommissionInfo {
27165 start_time: Some(generation),
27166 ..Default::default()
27167 }),
27168 );
27169 let last_update = status.last_update;
27170 let store = decommission_worker_test_store(
27171 PoolMeta {
27172 version: POOL_META_V1_VERSION,
27173 pools: vec![status],
27174 ..Default::default()
27175 },
27176 vec![None],
27177 );
27178 let entry = DecommissionUnresolvedEntry {
27179 bucket: "bucket-a".to_string(),
27180 object: "directory/".to_string(),
27181 pool_index: 0,
27182 set_index: 0,
27183 source_generation: generation,
27184 candidate_count: 1,
27185 disk_error_count: 0,
27186 observed_at: generation,
27187 reason: "metadata_resolution_failed".to_string(),
27188 };
27189
27190 let err = store
27191 .persist_decommission_unresolved_entry(0, generation, entry)
27192 .await
27193 .expect_err("V1 must reject the ledger before changing live state");
27194
27195 assert!(matches!(err, Error::InvalidArgument(..)));
27196 let pool_meta = store.pool_meta.read().await;
27197 let status = &pool_meta.pools[0];
27198 assert_eq!(status.last_update, last_update);
27199 assert!(
27200 status
27201 .decommission
27202 .as_ref()
27203 .expect("active decommission metadata should remain present")
27204 .unresolved_entries
27205 .is_empty()
27206 );
27207 drop(pool_meta);
27208 store
27209 .pool_meta_save_gate
27210 .lock()
27211 .await
27212 .ensure_write_safe("V1 unresolved-entry preflight")
27213 .expect("a deterministic capability rejection must not latch recovery");
27214 }
27215
27216 #[tokio::test]
27217 async fn test_v1_runtime_recovery_rejects_worker_but_keeps_cancel_persistable() {
27218 let generation = OffsetDateTime::UNIX_EPOCH;
27219 let store = decommission_worker_test_store(
27220 PoolMeta {
27221 version: POOL_META_V1_VERSION,
27222 pools: vec![decommission_test_pool_status(
27223 0,
27224 Some(PoolDecommissionInfo {
27225 start_time: Some(generation),
27226 ..Default::default()
27227 }),
27228 )],
27229 ..Default::default()
27230 },
27231 vec![None],
27232 );
27233
27234 let err = store
27235 .reserve_decommission_routines(&CancellationToken::new(), &[0])
27236 .await
27237 .err()
27238 .expect("V1 recovery must not install a worker that cannot persist an unresolved ledger");
27239 assert!(matches!(err, Error::InvalidArgument(..)));
27240 assert!(store.decommission_cancelers.read().await[0].is_none());
27241
27242 let save_called = Arc::new(AtomicBool::new(false));
27243 store
27244 .decommission_cancel_with_owner_and_save(0, None, {
27245 let save_called = save_called.clone();
27246 move |snapshot, _| async move {
27247 snapshot.encode_config_data_for_v2_gate(false)?;
27248 save_called.store(true, Ordering::SeqCst);
27249 Ok(())
27250 }
27251 })
27252 .await
27253 .expect("a rejected V1 recovery must remain cancelable without restart");
27254
27255 assert!(save_called.load(Ordering::SeqCst));
27256 let pool_meta = store.pool_meta.read().await;
27257 let info = pool_meta.pools[0]
27258 .decommission
27259 .as_ref()
27260 .expect("cancel metadata should remain present");
27261 assert!(info.canceled);
27262 assert!(!info.failed);
27263 assert!(!info.complete);
27264 }
27265
27266 #[tokio::test]
27267 async fn test_decommission_transition_waits_without_registered_canceler() {
27268 let store = decommission_worker_test_store(PoolMeta::default(), vec![None]);
27269 let operation_gate = store.ctx.data_movement_operation_gate();
27270 let operation_guard = operation_gate.read().await;
27271 let transition = tokio::spawn({
27272 let store = store.clone();
27273 async move { store.cancel_decommission_routines_and_wait(&[0]).await }
27274 });
27275
27276 tokio::task::yield_now().await;
27277 assert!(
27278 !transition.is_finished(),
27279 "a transition must wait for an in-flight side effect even after its canceler slot is gone"
27280 );
27281
27282 drop(operation_guard);
27283 tokio::time::timeout(StdDuration::from_secs(1), transition)
27284 .await
27285 .expect("transition should finish after the side effect")
27286 .expect("transition task should not panic");
27287 }
27288
27289 #[tokio::test]
27290 async fn test_join_error_quiesces_side_effects_before_failure_transition() {
27291 let canceler = DecommissionCanceler::new(CancellationToken::new());
27292 let store = decommission_worker_test_store(PoolMeta::default(), vec![Some(canceler.clone())]);
27293 let operation_gate = store.ctx.data_movement_operation_gate();
27294 let operation_guard = operation_gate.read().await;
27295 let transition = tokio::spawn({
27296 let store = store.clone();
27297 let canceler = canceler.clone();
27298 async move { store.quiesce_decommission_worker_after_join_error(&canceler).await }
27299 });
27300
27301 tokio::time::timeout(StdDuration::from_secs(1), canceler.token().cancelled())
27302 .await
27303 .expect("join error handling should cancel the detached worker");
27304 assert!(
27305 !transition.is_finished(),
27306 "failure transition must wait for the detached worker's in-flight side effect"
27307 );
27308
27309 drop(operation_guard);
27310 tokio::time::timeout(StdDuration::from_secs(1), transition)
27311 .await
27312 .expect("failure transition should finish after the side effect")
27313 .expect("failure transition task should not panic");
27314 }
27315
27316 #[tokio::test(start_paused = true)]
27317 async fn test_run_decommission_listing_with_retry_drains_before_each_retry() {
27318 let attempts = Arc::new(AtomicUsize::new(0));
27319 let drains = Arc::new(AtomicUsize::new(0));
27320 let err = run_decommission_listing_with_retry_and_drain(
27321 CancellationToken::new(),
27322 "bucket-a".to_string(),
27323 noop_decommission_list_callback(),
27324 1,
27325 2,
27326 2,
27327 {
27328 let attempts = attempts.clone();
27329 move |_| {
27330 let attempts = attempts.clone();
27331 async move {
27332 attempts.fetch_add(1, Ordering::SeqCst);
27333 Err(Error::SlowDown)
27334 }
27335 }
27336 },
27337 {
27338 let drains = drains.clone();
27339 move || {
27340 let drains = drains.clone();
27341 async move {
27342 drains.fetch_add(1, Ordering::SeqCst);
27343 false
27344 }
27345 }
27346 },
27347 )
27348 .await
27349 .expect_err("permanent listing failure must be returned");
27350
27351 assert!(err.to_string().contains("attempt 2/2"));
27352 assert_eq!(attempts.load(Ordering::SeqCst), 2);
27353 assert_eq!(drains.load(Ordering::SeqCst), 2);
27354 }
27355
27356 #[test]
27357 fn test_get_by_index_returns_value_when_in_range() {
27358 let values = vec!["a", "b", "c"];
27359 let value = get_by_index(values.as_slice(), 1, "fetch decommission status").expect("in-range index should return value");
27360 assert_eq!(*value, "b");
27361 }
27362
27363 #[test]
27364 fn test_get_by_index_returns_error_when_out_of_range() {
27365 let values = vec![1_u8];
27366 let err =
27367 get_by_index(values.as_slice(), 2, "load decommission background pool").expect_err("out-of-range index should fail");
27368 assert!(
27369 err.to_string()
27370 .contains("failed to load decommission background pool: invalid decommission pool index 2 for 1 pools")
27371 );
27372 }
27373
27374 #[test]
27375 fn test_pool_meta_is_suspended_returns_false_for_out_of_range() {
27376 let meta = PoolMeta::default();
27377 assert!(!meta.is_suspended(1));
27378 }
27379
27380 #[test]
27381 fn test_rollback_start_decommission_pool_meta_clears_active_state() {
27382 let previous = PoolMeta {
27383 pools: vec![PoolStatus {
27384 id: 0,
27385 cmd_line: "pool-0".to_string(),
27386 last_update: OffsetDateTime::UNIX_EPOCH,
27387 decommission: None,
27388 }],
27389 ..Default::default()
27390 };
27391 let mut active = previous.clone();
27392 let active_update = OffsetDateTime::UNIX_EPOCH + Duration::seconds(1);
27393 active.pools[0].last_update = active_update;
27394 active.pools[0].decommission = Some(PoolDecommissionInfo {
27395 start_time: Some(OffsetDateTime::UNIX_EPOCH),
27396 ..Default::default()
27397 });
27398 let mut peer = active.clone();
27399
27400 assert!(active.is_suspended(0));
27401 assert_eq!(
27402 decommission_start_pool_state(active.pools.first()),
27403 DecommissionStartPoolState::Decommissioning
27404 );
27405
27406 rollback_start_decommission_pool_meta(&mut active, &previous, &[0]);
27407
27408 assert!(!active.is_suspended(0));
27409 assert_eq!(decommission_start_pool_state(active.pools.first()), DecommissionStartPoolState::Active);
27410 assert!(active.pools[0].last_update > active_update);
27411 assert!(merge_pool_status_refresh(&mut peer, active, &[false]));
27412 assert!(peer.pools[0].decommission.is_none());
27413 }
27414
27415 #[test]
27416 fn test_pool_meta_queue_buckets_ignores_out_of_range_index() {
27417 let mut meta = PoolMeta {
27418 pools: vec![PoolStatus {
27419 id: 0,
27420 cmd_line: "pool-0".to_string(),
27421 last_update: OffsetDateTime::UNIX_EPOCH,
27422 decommission: Some(PoolDecommissionInfo::default()),
27423 }],
27424 ..Default::default()
27425 };
27426
27427 meta.queue_buckets(
27428 9,
27429 vec![DecomBucketInfo {
27430 name: "bucket-a".to_string(),
27431 prefix: String::new(),
27432 }],
27433 );
27434
27435 let queued = meta.pools[0]
27436 .decommission
27437 .as_ref()
27438 .expect("pool should have decommission info")
27439 .queued_buckets
27440 .clone();
27441 assert!(queued.is_empty());
27442 }
27443
27444 #[test]
27445 fn test_pool_meta_is_bucket_decommissioned_returns_false_for_out_of_range() {
27446 let meta = PoolMeta::default();
27447 assert!(!meta.is_bucket_decommissioned(7, "bucket-a".to_string()));
27448 }
27449
27450 #[test]
27451 fn test_resolve_decommission_bucket_state_rejects_out_of_range_index() {
27452 let meta = PoolMeta {
27453 pools: vec![PoolStatus {
27454 id: 0,
27455 cmd_line: "pool-0".to_string(),
27456 last_update: OffsetDateTime::UNIX_EPOCH,
27457 decommission: Some(PoolDecommissionInfo::default()),
27458 }],
27459 ..Default::default()
27460 };
27461
27462 let bucket = DecomBucketInfo {
27463 name: "bucket-a".to_string(),
27464 prefix: String::new(),
27465 };
27466 let err =
27467 resolve_decommission_bucket_state(&meta, 3, &bucket).expect_err("out-of-range index should return invalid argument");
27468 assert!(err.to_string().contains("invalid decommission pool index 3 for 1 pools"));
27469 }
27470
27471 #[test]
27472 fn test_resolve_decommission_bucket_state_rejects_missing_decommission_meta() {
27473 let meta = PoolMeta {
27474 pools: vec![PoolStatus {
27475 id: 0,
27476 cmd_line: "pool-0".to_string(),
27477 last_update: OffsetDateTime::UNIX_EPOCH,
27478 decommission: None,
27479 }],
27480 ..Default::default()
27481 };
27482
27483 let bucket = DecomBucketInfo {
27484 name: "bucket-a".to_string(),
27485 prefix: String::new(),
27486 };
27487 let err = resolve_decommission_bucket_state(&meta, 0, &bucket)
27488 .expect_err("missing decommission metadata should return explicit error");
27489 assert!(
27490 err.to_string()
27491 .contains("failed to resolve decommission bucket state: decommission metadata not initialized")
27492 );
27493 }
27494
27495 #[test]
27496 fn test_resolve_decommission_bucket_state_returns_true_for_done_bucket() {
27497 let meta = PoolMeta {
27498 pools: vec![PoolStatus {
27499 id: 0,
27500 cmd_line: "pool-0".to_string(),
27501 last_update: OffsetDateTime::UNIX_EPOCH,
27502 decommission: Some(PoolDecommissionInfo {
27503 decommissioned_buckets: vec!["bucket-a".to_string()],
27504 ..Default::default()
27505 }),
27506 }],
27507 ..Default::default()
27508 };
27509
27510 let bucket = DecomBucketInfo {
27511 name: "bucket-a".to_string(),
27512 prefix: String::new(),
27513 };
27514 let done = resolve_decommission_bucket_state(&meta, 0, &bucket).expect("valid state should resolve");
27515 assert!(done);
27516 }
27517
27518 #[test]
27519 fn test_decommission_capacity_publication_preserves_progress_and_checkpoint_deadline() {
27520 let mut current = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_MODEL_VERSION]);
27521 let start = OffsetDateTime::UNIX_EPOCH;
27522 current.pools[0].last_update = start;
27523 current.count_item(0, 17, false);
27524 let mut saved = current.clone();
27525 saved.pools[0]
27526 .decommission
27527 .as_mut()
27528 .expect("persisted source")
27529 .items_decommissioned = 0;
27530 saved.pools[0].decommission.as_mut().expect("persisted source").bytes_done = 0;
27531 for seconds in [10, 20, 29] {
27532 saved.pools[0].last_update = start + Duration::seconds(seconds);
27533 super::publish_decommission_capacity_update(&mut current, &saved, 0).expect("same-owner capacity publication");
27534 }
27535 let info = current.pools[0].decommission.as_ref().expect("live source");
27536 assert_eq!((info.items_decommissioned, info.bytes_done, info.progress_save_item_baseline), (1, 17, 0));
27537 assert_eq!(info.progress_save_last_at, Some(start));
27538 let checkpoint = current
27539 .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, start + Duration::seconds(30), None)
27540 .expect("checkpoint should remain due despite capacity writes")
27541 .expect("capacity updates must not postpone the 30-second progress checkpoint");
27542 current.count_item(0, 19, false);
27543 assert!(current.commit_decommission_progress_checkpoint(0, checkpoint));
27544 let info = current.pools[0].decommission.as_ref().expect("live source");
27545 assert_eq!(
27546 info.items_since_last_progress_save(),
27547 1,
27548 "only the committed snapshot advances the watermark"
27549 );
27550 assert_eq!(info.progress_save_last_at, Some(checkpoint.checkpoint_at));
27551 assert!(
27552 current
27553 .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, start + Duration::seconds(31), None)
27554 .expect("next checkpoint timing")
27555 .is_none()
27556 );
27557 }
27558
27559 #[test]
27560 fn test_decommission_capacity_publication_rejects_changed_identity_without_mutation() {
27561 let baseline = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_MODEL_VERSION]);
27562 for changed_field in [
27563 "layout",
27564 "start",
27565 "queued",
27566 "terminal",
27567 "operation",
27568 "generation",
27569 "owner",
27570 "model",
27571 "cleared",
27572 ] {
27573 let mut current = baseline.clone();
27574 let mut saved = baseline.clone();
27575 saved.pools[0].last_update += Duration::seconds(1);
27576 let pool = &mut current.pools[0];
27577 let info = pool.decommission.as_mut().expect("current source");
27578 match changed_field {
27579 "layout" => pool.cmd_line.push_str("-replaced"),
27580 "start" => info.start_time = Some(OffsetDateTime::UNIX_EPOCH + Duration::seconds(1)),
27581 "queued" => info.queued = true,
27582 "terminal" => info.canceled = true,
27583 "operation" => info.capacity_reservation.as_mut().expect("owner").operation_id = uuid::Uuid::new_v4(),
27584 "generation" => info.capacity_reservation.as_mut().expect("owner").generation += 1,
27585 "owner" => info.capacity_reservation.as_mut().expect("owner").owner_nonce = uuid::Uuid::new_v4(),
27586 "model" => info.capacity_reservation.as_mut().expect("owner").model_version += 1,
27587 "cleared" => pool.decommission = None,
27588 _ => unreachable!("fixed test matrix"),
27589 }
27590 let before = format!("{current:?}");
27591 let err = super::publish_decommission_capacity_update(&mut current, &saved, 0)
27592 .expect_err("changed operation identity must reject publication");
27593 assert!(matches!(err, Error::StalePoolMetadataUpdate { .. }), "{changed_field}: {err}");
27594 assert_eq!(format!("{current:?}"), before, "{changed_field}: rejected publication mutated live state");
27595 }
27596 }
27597
27598 #[test]
27599 fn test_decommission_capacity_revision_does_not_regress_when_clock_moves_backwards() {
27600 let mut meta = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_MODEL_VERSION, Vec::new(), 0, None);
27601 let revision = meta.pools[0].last_update;
27602 let mutation = uuid::Uuid::new_v4();
27603 reserve_decommission_target_pending(&mut meta, 0, 1, 1, mutation, OffsetDateTime::UNIX_EPOCH)
27604 .expect("clock rollback must not prevent a valid owned reservation");
27605 assert_eq!(meta.pools[0].last_update, revision);
27606 resolve_decommission_target_pending(&mut meta, 0, 1, 1, mutation).expect("resolve the exact intent");
27607 record_decommission_target_consumption(
27608 &mut meta,
27609 0,
27610 1,
27611 DecommissionTargetConsumption {
27612 committed_data_bytes: 1,
27613 target_physical_bytes: 1,
27614 observed_physical_bytes: 1,
27615 },
27616 mutation,
27617 OffsetDateTime::UNIX_EPOCH,
27618 )
27619 .expect("commit consumption during clock rollback");
27620 assert_eq!(meta.pools[0].last_update, revision);
27621 }
27622
27623 #[test]
27624 fn test_mark_decommission_bucket_done_is_idempotent_and_ignores_unknown_buckets() {
27625 let bucket = DecomBucketInfo {
27626 name: "bucket-a".to_string(),
27627 prefix: String::new(),
27628 };
27629 let unknown = DecomBucketInfo {
27630 name: "bucket-b".to_string(),
27631 prefix: String::new(),
27632 };
27633 let mut meta = PoolMeta {
27634 pools: vec![decommission_test_pool_status(
27635 0,
27636 Some(PoolDecommissionInfo {
27637 queued_buckets: vec![bucket.to_string()],
27638 ..Default::default()
27639 }),
27640 )],
27641 ..Default::default()
27642 };
27643 assert!(!mark_decommission_bucket_done(&mut meta, 0, &unknown).expect("an unknown bucket is not completed"));
27644 assert!(
27645 meta.pools[0]
27646 .decommission
27647 .as_ref()
27648 .expect("source progress")
27649 .decommissioned_buckets
27650 .is_empty()
27651 );
27652 assert!(mark_decommission_bucket_done(&mut meta, 0, &bucket).expect("complete the queued bucket once"));
27653 assert!(!mark_decommission_bucket_done(&mut meta, 0, &bucket).expect("repeat completion is a no-op"));
27654 let info = meta.pools[0].decommission.as_ref().expect("source progress");
27655 assert!(info.queued_buckets.is_empty());
27656 assert_eq!(info.decommissioned_buckets, vec![bucket.to_string()]);
27657 }
27658
27659 #[test]
27660 fn test_mark_decommission_bucket_done_rejects_missing_decommission_meta() {
27661 let mut meta = PoolMeta {
27662 pools: vec![PoolStatus {
27663 id: 0,
27664 cmd_line: "pool-0".to_string(),
27665 last_update: OffsetDateTime::UNIX_EPOCH,
27666 decommission: None,
27667 }],
27668 ..Default::default()
27669 };
27670
27671 let bucket = DecomBucketInfo {
27672 name: "bucket-a".to_string(),
27673 prefix: String::new(),
27674 };
27675 let err = mark_decommission_bucket_done(&mut meta, 0, &bucket)
27676 .expect_err("missing decommission metadata should return explicit error");
27677 assert!(
27678 err.to_string()
27679 .contains("failed to mark decommission bucket done: decommission metadata not initialized")
27680 );
27681 }
27682
27683 #[test]
27684 fn test_mark_decommission_bucket_done_rejects_out_of_range_index() {
27685 let mut meta = PoolMeta {
27686 pools: vec![PoolStatus {
27687 id: 0,
27688 cmd_line: "pool-0".to_string(),
27689 last_update: OffsetDateTime::UNIX_EPOCH,
27690 decommission: Some(PoolDecommissionInfo::default()),
27691 }],
27692 ..Default::default()
27693 };
27694
27695 let bucket = DecomBucketInfo {
27696 name: "bucket-a".to_string(),
27697 prefix: String::new(),
27698 };
27699 let err =
27700 mark_decommission_bucket_done(&mut meta, 1, &bucket).expect_err("out-of-range index should return invalid argument");
27701 assert!(err.to_string().contains("invalid decommission pool index 1 for 1 pools"));
27702 }
27703
27704 #[test]
27705 fn test_mark_decommission_bucket_done_pops_bucket_when_present() {
27706 let mut meta = PoolMeta {
27707 pools: vec![PoolStatus {
27708 id: 0,
27709 cmd_line: "pool-0".to_string(),
27710 last_update: OffsetDateTime::UNIX_EPOCH,
27711 decommission: Some(PoolDecommissionInfo {
27712 queued_buckets: vec!["bucket-a".to_string()],
27713 ..Default::default()
27714 }),
27715 }],
27716 ..Default::default()
27717 };
27718
27719 let bucket = DecomBucketInfo {
27720 name: "bucket-a".to_string(),
27721 prefix: String::new(),
27722 };
27723 let popped = mark_decommission_bucket_done(&mut meta, 0, &bucket).expect("valid state should mark bucket done");
27724 assert!(popped);
27725 }
27726
27727 #[test]
27728 fn test_count_decommission_item_rejects_missing_decommission_meta() {
27729 let mut meta = PoolMeta {
27730 pools: vec![PoolStatus {
27731 id: 0,
27732 cmd_line: "pool-0".to_string(),
27733 last_update: OffsetDateTime::UNIX_EPOCH,
27734 decommission: None,
27735 }],
27736 ..Default::default()
27737 };
27738
27739 let err = count_decommission_item(&mut meta, 0, 64, true)
27740 .expect_err("missing decommission metadata should return explicit error");
27741 assert!(
27742 err.to_string()
27743 .contains("failed to count decommission item: decommission metadata not initialized")
27744 );
27745 }
27746
27747 #[test]
27748 fn test_count_decommission_item_updates_done_and_failed_counters() {
27749 let mut meta = PoolMeta {
27750 pools: vec![PoolStatus {
27751 id: 0,
27752 cmd_line: "pool-0".to_string(),
27753 last_update: OffsetDateTime::UNIX_EPOCH,
27754 decommission: Some(PoolDecommissionInfo::default()),
27755 }],
27756 ..Default::default()
27757 };
27758
27759 count_decommission_item(&mut meta, 0, 32, false).expect("success counter should be updated");
27760 count_decommission_item(&mut meta, 0, 16, true).expect("failed counter should be updated");
27761
27762 let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
27763 assert_eq!(info.items_decommissioned, 1);
27764 assert_eq!(info.bytes_done, 32);
27765 assert_eq!(info.items_decommission_failed, 1);
27766 assert_eq!(info.bytes_failed, 16);
27767 }
27768
27769 #[test]
27770 fn test_track_decommission_current_object_rejects_missing_decommission_meta() {
27771 let mut meta = PoolMeta {
27772 pools: vec![PoolStatus {
27773 id: 0,
27774 cmd_line: "pool-0".to_string(),
27775 last_update: OffsetDateTime::UNIX_EPOCH,
27776 decommission: None,
27777 }],
27778 ..Default::default()
27779 };
27780
27781 let err = track_decommission_current_object(&mut meta, 0, "bucket-a", "object-a")
27782 .expect_err("missing decommission metadata should return explicit error");
27783 assert!(
27784 err.to_string()
27785 .contains("failed to track decommission current object: decommission metadata not initialized")
27786 );
27787 }
27788
27789 #[test]
27790 fn test_track_decommission_current_object_updates_bucket_and_object() {
27791 let mut meta = PoolMeta {
27792 pools: vec![PoolStatus {
27793 id: 0,
27794 cmd_line: "pool-0".to_string(),
27795 last_update: OffsetDateTime::UNIX_EPOCH,
27796 decommission: Some(PoolDecommissionInfo::default()),
27797 }],
27798 ..Default::default()
27799 };
27800
27801 track_decommission_current_object(&mut meta, 0, "bucket-a", "object-a").expect("valid state should track bucket/object");
27802
27803 let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
27804 assert_eq!(info.bucket, "bucket-a");
27805 assert_eq!(info.object, "object-a");
27806 assert!(info.stage.is_empty());
27807 }
27808
27809 #[test]
27810 fn test_track_decommission_current_object_stage_updates_stage() {
27811 let mut meta = PoolMeta {
27812 pools: vec![PoolStatus {
27813 id: 0,
27814 cmd_line: "pool-0".to_string(),
27815 last_update: OffsetDateTime::UNIX_EPOCH,
27816 decommission: Some(PoolDecommissionInfo::default()),
27817 }],
27818 ..Default::default()
27819 };
27820
27821 track_decommission_current_object_stage(&mut meta, 0, "bucket-a", "object-a", "cleanup_preflight")
27822 .expect("valid state should track bucket/object stage");
27823
27824 let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
27825 assert_eq!(info.bucket, "bucket-a");
27826 assert_eq!(info.object, "object-a");
27827 assert_eq!(info.stage, "cleanup_preflight");
27828 }
27829
27830 #[test]
27831 fn test_resolve_decommission_update_after_result_passthrough_ok() {
27832 let ok = resolve_decommission_update_after_result(Ok(true)).expect("ok value should pass through");
27833 assert!(ok);
27834 }
27835
27836 #[test]
27837 fn test_resolve_decommission_update_after_result_wraps_error_context() {
27838 let err = resolve_decommission_update_after_result(ensure_valid_decommission_pool_index(0, 0).map(|_| false))
27839 .expect_err("invalid argument should be wrapped with context");
27840 assert!(err.to_string().contains("decommission metadata update failed"));
27841 assert!(err.to_string().contains("invalid decommission pool index 0 for 0 pools"));
27842 }
27843
27844 #[test]
27845 fn test_resolve_decommission_progress_save_result_returns_none_on_success() {
27846 assert!(resolve_decommission_progress_save_result(Ok(())).is_none());
27847 }
27848
27849 #[test]
27850 fn test_resolve_decommission_progress_save_result_returns_error_for_best_effort_failure() {
27851 let err = resolve_decommission_progress_save_result(Err(Error::SlowDown))
27852 .expect("progress save failure should be returned for logging");
27853
27854 assert!(err.to_string().contains("decommission progress save failed"));
27855 assert!(err.to_string().contains(Error::SlowDown.to_string().as_str()));
27856 }
27857
27858 #[test]
27859 fn test_resolve_decommission_preflight_heal_result_passthrough_ok() {
27860 assert!(resolve_decommission_preflight_heal_result::<()>("bucket-a", Ok(())).is_ok());
27861 }
27862
27863 #[test]
27864 fn test_resolve_decommission_preflight_heal_result_wraps_error_context() {
27865 let err = resolve_decommission_preflight_heal_result::<()>("bucket-a", Err(Error::SlowDown))
27866 .expect_err("heal failure should carry preflight context");
27867 assert!(
27868 err.to_string()
27869 .contains("decommission preflight heal failed for bucket bucket-a")
27870 );
27871 }
27872
27873 #[test]
27874 fn test_resolve_decommission_optional_bucket_config_result_passthrough() {
27875 let result = resolve_decommission_optional_bucket_config_result("bucket-a", "replication", Ok(42_u8))
27876 .expect("bucket config should pass through");
27877 assert_eq!(result, Some(42));
27878 }
27879
27880 #[test]
27881 fn test_resolve_decommission_optional_bucket_config_result_returns_none_for_missing_config() {
27882 let result =
27883 resolve_decommission_optional_bucket_config_result::<()>("bucket-a", "versioning", Err(Error::ConfigNotFound))
27884 .expect("missing bucket config should map to None");
27885 assert!(result.is_none());
27886 }
27887
27888 #[test]
27889 fn test_resolve_decommission_optional_bucket_config_result_wraps_other_errors() {
27890 let err = resolve_decommission_optional_bucket_config_result::<()>("bucket-a", "replication", Err(Error::SlowDown))
27891 .expect_err("unexpected bucket config errors should be wrapped with context");
27892 assert!(
27893 err.to_string()
27894 .contains("decommission replication config load failed for bucket bucket-a")
27895 );
27896 }
27897
27898 #[test]
27899 fn test_resolve_decommission_entry_cleanup_delete_result_passthrough_ok() {
27900 assert!(resolve_decommission_entry_cleanup_delete_result(Ok(()), "bucket-a", "obj.txt").is_ok());
27901 }
27902
27903 #[test]
27904 fn test_resolve_decommission_entry_cleanup_delete_result_ignores_not_found() {
27905 assert!(resolve_decommission_entry_cleanup_delete_result::<()>(Err(Error::FileNotFound), "bucket-a", "obj.txt").is_ok());
27906 }
27907
27908 #[test]
27909 fn test_resolve_decommission_entry_cleanup_delete_result_wraps_error_context() {
27910 let err = resolve_decommission_entry_cleanup_delete_result::<()>(Err(Error::SlowDown), "bucket-a", "obj.txt")
27911 .expect_err("cleanup delete failure should be wrapped with explicit context");
27912 assert!(
27913 err.to_string()
27914 .contains("decommission cleanup_delete_object failed for bucket-a/obj.txt")
27915 );
27916 }
27917
27918 #[test]
27919 fn test_resolve_decommission_entry_reload_result_passthrough_ok() {
27920 assert!(resolve_decommission_entry_reload_result(Ok(()), "bucket-a", "obj.txt").is_ok());
27921 }
27922
27923 #[test]
27924 fn test_resolve_decommission_entry_reload_result_wraps_error_context() {
27925 let err = resolve_decommission_entry_reload_result(Err(Error::SlowDown), "bucket-a", "obj.txt")
27926 .expect_err("reload failure should be wrapped with explicit context");
27927 assert!(
27928 err.to_string()
27929 .contains("decommission reload_pool_meta failed for bucket-a/obj.txt")
27930 );
27931 }
27932
27933 #[test]
27934 fn test_resolve_decommission_terminal_mark_result_passthrough_ok() {
27935 assert!(resolve_decommission_terminal_mark_result(Ok(()), "completed", "pool-a").is_ok());
27936 }
27937
27938 #[test]
27939 fn test_resolve_decommission_terminal_mark_result_wraps_error_context() {
27940 let err = resolve_decommission_terminal_mark_result(Err(Error::SlowDown), "failed", "pool-a")
27941 .expect_err("terminal mark failure should include stage and pool context");
27942 let message = err.to_string();
27943 assert!(message.contains("decommission terminal mark failed failed for pool pool-a"));
27944 }
27945
27946 #[test]
27947 fn test_resolve_decommission_terminal_mark_after_error_result_passthrough_ok() {
27948 assert!(resolve_decommission_terminal_mark_after_error_result(Ok(()), 3, &Error::SlowDown).is_ok());
27949 }
27950
27951 #[test]
27952 fn test_resolve_decommission_terminal_mark_after_error_result_wraps_error_context() {
27953 let err = resolve_decommission_terminal_mark_after_error_result(Err(Error::OperationCanceled), 3, &Error::SlowDown)
27954 .expect_err("terminal mark after-error failure should include both errors");
27955 let message = err.to_string();
27956 assert!(message.contains("decommission terminal mark failed after background error on pool 3"));
27957 assert!(message.contains("mark error"));
27958 }
27959
27960 #[test]
27961 fn test_observe_decommission_terminal_reload_result_returns_none_on_success() {
27962 assert!(observe_decommission_terminal_reload_result(Ok(()), "complete_decommission for pool 3").is_none());
27963 }
27964
27965 #[test]
27966 fn test_observe_decommission_terminal_reload_result_keeps_failure_for_logging() {
27967 let err = observe_decommission_terminal_reload_result(Err(Error::SlowDown), "decommission_failed for pool 3")
27968 .expect("reload failure should be observable");
27969 let message = err.to_string();
27970 assert!(message.contains("decommission terminal pool meta reload failed during decommission_failed for pool 3"));
27971 assert!(message.contains(Error::SlowDown.to_string().as_str()));
27972 }
27973
27974 #[test]
27975 fn test_decommission_item_size_converts_positive_values() {
27976 assert_eq!(decommission_item_size(42_i64), 42);
27977 }
27978
27979 #[test]
27980 fn test_decommission_item_size_clamps_negative_values_to_zero() {
27981 assert_eq!(decommission_item_size(-1_i64), 0);
27982 }
27983
27984 #[test]
27985 fn test_new_multipart_abort_flag_defaults_to_abort_enabled() {
27986 let flag = data_movement::new_multipart_abort_flag();
27987 assert!(data_movement::should_abort_multipart_upload(&flag));
27988 }
27989
27990 #[test]
27991 fn test_mark_multipart_upload_completed_disables_abort_cleanup() {
27992 let flag = data_movement::new_multipart_abort_flag();
27993 data_movement::mark_multipart_upload_completed(&flag);
27994 assert!(!data_movement::should_abort_multipart_upload(&flag));
27995 }
27996
27997 #[test]
27998 fn test_decode_part_index_returns_some_for_valid_payload() {
27999 let mut index = Index::new();
28000 index.add(0, 0).expect("first index entry should be accepted");
28001 index
28002 .add(2_097_152, 2_097_152)
28003 .expect("second index entry should advance totals");
28004
28005 let encoded = index.into_vec();
28006 let decoded = data_movement::decode_part_index(Some(&encoded)).expect("valid index payload should decode");
28007
28008 assert_eq!(decoded.total_uncompressed, 2_097_152);
28009 assert_eq!(decoded.total_compressed, 2_097_152);
28010 }
28011
28012 #[test]
28013 fn test_with_decommission_entry_context_formats_stage_bucket_and_object() {
28014 let err = with_decommission_entry_context("update_after", "bucket-a", "obj.txt", Error::SlowDown);
28015 let message = err.to_string();
28016 assert!(message.contains("decommission entry update_after failed"));
28017 assert!(message.contains("bucket bucket-a"));
28018 assert!(message.contains("object obj.txt"));
28019 }
28020
28021 #[test]
28022 fn test_load_decommission_entry_versions_wraps_parse_errors_with_context() {
28023 let entry = MetaCacheEntry {
28024 name: "obj.txt".to_string(),
28025 metadata: vec![1, 2, 3],
28026 cached: None,
28027 reusable: false,
28028 };
28029
28030 let err = load_decommission_entry_versions(&entry, "bucket-a", "check_after_decommission.file_info_versions")
28031 .expect_err("invalid metadata should fail");
28032 let message = err.to_string();
28033 assert!(message.contains("decommission entry check_after_decommission.file_info_versions failed"));
28034 assert!(message.contains("bucket bucket-a"));
28035 assert!(message.contains("object obj.txt"));
28036 }
28037
28038 #[test]
28039 fn test_resolve_decommission_entry_exact_versions_preserves_full_parts() {
28040 let entry = MetaCacheEntry {
28041 name: "obj.txt".to_string(),
28042 metadata: Vec::new(),
28043 cached: None,
28044 reusable: false,
28045 };
28046 let fivs = FileInfoVersions {
28047 volume: "bucket-a".to_string(),
28048 name: "obj.txt".to_string(),
28049 versions: vec![FileInfo {
28050 name: "obj.txt".to_string(),
28051 parts: vec![ObjectPartInfo {
28052 number: 1,
28053 etag: "part-etag".to_string(),
28054 size: 128,
28055 actual_size: 128,
28056 ..Default::default()
28057 }],
28058 ..Default::default()
28059 }],
28060 ..Default::default()
28061 };
28062
28063 let resolved = resolve_decommission_entry_exact_versions(Ok(Some(fivs)), &entry, "bucket-a", "file_info_versions")
28064 .expect("exact versions should be preserved");
28065
28066 assert_eq!(resolved.versions[0].parts.len(), 1);
28067 assert_eq!(resolved.versions[0].parts[0].etag, "part-etag");
28068 }
28069
28070 #[test]
28071 fn test_resolve_decommission_entry_exact_versions_uses_empty_when_source_missing() {
28072 let entry = MetaCacheEntry {
28073 name: "obj.txt".to_string(),
28074 metadata: Vec::new(),
28075 cached: None,
28076 reusable: false,
28077 };
28078
28079 let resolved = resolve_decommission_entry_exact_versions(Ok(None), &entry, "bucket-a", "file_info_versions")
28080 .expect("missing source metadata should be treated as empty");
28081
28082 assert_eq!(resolved.volume, "bucket-a");
28083 assert_eq!(resolved.name, "obj.txt");
28084 assert!(resolved.versions.is_empty());
28085 }
28086
28087 #[test]
28088 fn test_resolve_decommission_check_after_list_result_prefers_entry_error() {
28089 let err = resolve_decommission_check_after_list_result(Err(Error::OperationCanceled), Some(Error::SlowDown))
28090 .expect_err("entry error should win over cancellation");
28091 assert!(matches!(err, Error::SlowDown));
28092 }
28093
28094 #[test]
28095 fn test_resolve_decommission_partial_listing_entry_rejects_unresolved_metadata() {
28096 let generation = OffsetDateTime::now_utc();
28097 let unresolved_entry = resolve_decommission_partial_listing_entry(
28098 MetaCacheEntries(vec![None]),
28099 MetadataResolutionParams {
28100 dir_quorum: 2,
28101 obj_quorum: 2,
28102 bucket: "bucket-a".to_string(),
28103 ..Default::default()
28104 },
28105 "bucket-a",
28106 "prefix/",
28107 1,
28108 2,
28109 3,
28110 generation,
28111 )
28112 .expect_err("unresolved partial listing must fail closed");
28113
28114 assert_eq!(unresolved_entry.bucket, "bucket-a");
28115 assert_eq!(unresolved_entry.object, "prefix/");
28116 assert_eq!(unresolved_entry.pool_index, 2);
28117 assert_eq!(unresolved_entry.set_index, 3);
28118 assert_eq!(unresolved_entry.source_generation, generation);
28119 assert_eq!(unresolved_entry.candidate_count, 0);
28120 assert_eq!(unresolved_entry.disk_error_count, 1);
28121 assert_eq!(unresolved_entry.reason, "metadata_resolution_failed");
28122
28123 let message = decommission_unresolved_listing_error(&unresolved_entry).to_string();
28124 assert!(message.contains("decommission listing could not resolve metadata"));
28125 assert!(message.contains("bucket-a/prefix/"));
28126 assert!(message.contains("pool 2 set 3"));
28127 assert!(message.contains("1 disk error(s)"));
28128 }
28129
28130 #[test]
28131 fn unresolved_entry_ledger_requires_individual_verification_before_completion() {
28132 let generation = OffsetDateTime::now_utc();
28133 let mut pool_meta = PoolMeta {
28134 version: POOL_META_VERSION,
28135 pools: vec![PoolStatus {
28136 id: 0,
28137 cmd_line: "/data/pool".to_string(),
28138 last_update: generation,
28139 decommission: Some(PoolDecommissionInfo {
28140 start_time: Some(generation),
28141 ..Default::default()
28142 }),
28143 }],
28144 dont_save: true,
28145 };
28146 let unresolved_entry = DecommissionUnresolvedEntry {
28147 bucket: "bucket-a".to_string(),
28148 object: "object-a".to_string(),
28149 pool_index: 0,
28150 set_index: 0,
28151 source_generation: generation,
28152 candidate_count: 1,
28153 disk_error_count: 0,
28154 observed_at: generation,
28155 reason: "metadata_resolution_failed".to_string(),
28156 };
28157
28158 assert!(
28159 record_decommission_unresolved_entry(
28160 &mut pool_meta,
28161 0,
28162 generation,
28163 unresolved_entry.clone(),
28164 generation + Duration::nanoseconds(1),
28165 None,
28166 )
28167 .expect("active generation should accept unresolved entry")
28168 );
28169 let err = reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, None, None)
28170 .expect_err("unverified completion must retain unresolved entries");
28171 assert!(err.to_string().contains("1 unresolved listing entries remain"));
28172 assert_eq!(
28173 pool_meta.pools[0]
28174 .decommission
28175 .as_ref()
28176 .expect("decommission should exist")
28177 .unresolved_entries
28178 .len(),
28179 1
28180 );
28181
28182 let err = reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), None)
28183 .expect_err("a same-generation sweep without entry proof must retain the ledger");
28184 assert!(err.to_string().contains("not individually verified"));
28185
28186 let stale_verified = vec![unresolved_entry.clone()];
28187 let mut replacement = unresolved_entry;
28188 replacement.disk_error_count = 2;
28189 replacement.observed_at = generation + Duration::seconds(1);
28190 assert!(
28191 record_decommission_unresolved_entry(
28192 &mut pool_meta,
28193 0,
28194 generation,
28195 replacement.clone(),
28196 generation + Duration::nanoseconds(2),
28197 None,
28198 )
28199 .expect("a newer observation should replace the ledger entry")
28200 );
28201 let err =
28202 reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), Some(&stale_verified))
28203 .expect_err("stale entry verification must not clear a concurrent replacement");
28204 assert!(err.to_string().contains("not individually verified"));
28205
28206 let verified = vec![replacement];
28207 reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), Some(&verified))
28208 .expect("individually verified entries should reconcile");
28209 assert!(
28210 pool_meta.pools[0]
28211 .decommission
28212 .as_ref()
28213 .expect("decommission should exist")
28214 .unresolved_entries
28215 .is_empty()
28216 );
28217 }
28218
28219 #[tokio::test]
28220 async fn final_sweep_persists_unresolved_entry_from_real_listing() {
28221 let (dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
28222 let bucket = "decommission-final-sweep-unresolved";
28223 let object = "corrupt-object";
28224 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
28225 store
28226 .make_bucket(bucket, &MakeBucketOptions::default())
28227 .await
28228 .expect("test bucket should be created");
28229
28230 let generation = OffsetDateTime::now_utc();
28231 {
28232 let mut pool_meta = store.pool_meta.write().await;
28233 pool_meta.dont_save = false;
28234 pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
28235 start_time: Some(generation),
28236 ..Default::default()
28237 });
28238 }
28239 store
28240 .save_current_pool_meta_for_test(&[0])
28241 .await
28242 .expect("active decommission metadata should be persisted before the final sweep");
28243
28244 for (disk_index, dir) in dirs.iter().enumerate() {
28245 let object_dir = dir.path().join(bucket).join(object);
28246 tokio::fs::create_dir_all(&object_dir)
28247 .await
28248 .expect("corrupt object directory should be created");
28249 tokio::fs::write(object_dir.join(STORAGE_FORMAT_FILE), format!("corrupt-xl-meta-{disk_index}").into_bytes())
28250 .await
28251 .expect("divergent corrupt metadata should be written");
28252 }
28253
28254 let err = store
28255 .check_after_decommission(0, &CancellationToken::new(), generation)
28256 .await
28257 .expect_err("real final sweep must fail closed on unresolved listing metadata");
28258 assert!(
28259 err.to_string().contains("decommission listing could not resolve metadata"),
28260 "unexpected final sweep error: {err:?}"
28261 );
28262
28263 let in_memory_entries = store.pool_meta.read().await.pools[0]
28264 .decommission
28265 .as_ref()
28266 .expect("decommission should remain active")
28267 .unresolved_entries
28268 .clone();
28269 assert_eq!(in_memory_entries.len(), 1);
28270 let unresolved_entry = &in_memory_entries[0];
28271 assert_eq!(unresolved_entry.bucket, bucket);
28272 assert_eq!(unresolved_entry.object, object);
28273 assert_eq!(unresolved_entry.pool_index, 0);
28274 assert_eq!(unresolved_entry.set_index, 0);
28275 assert_eq!(unresolved_entry.source_generation, generation);
28276 assert_eq!(unresolved_entry.candidate_count, 4);
28277 assert_eq!(unresolved_entry.disk_error_count, 0);
28278 assert_eq!(unresolved_entry.reason, "metadata_resolution_failed");
28279
28280 let mut restored = PoolMeta::default();
28281 restored
28282 .load_no_lock_from_replicas(vec![store.pools[0].clone()])
28283 .await
28284 .expect("persisted pool metadata should reload after the final-sweep failure");
28285 assert_eq!(
28286 restored.pools[0]
28287 .decommission
28288 .as_ref()
28289 .expect("reloaded decommission should exist")
28290 .unresolved_entries,
28291 in_memory_entries
28292 );
28293 }
28294
28295 #[tokio::test]
28296 async fn decommission_metadata_saves_stay_monotonic_across_clock_rollback_before_terminal_restart() {
28297 let (_dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
28298 let persisted_floor = store.pool_meta.read().await.pools[0].last_update;
28299 let generation = persisted_floor
28300 .checked_add(Duration::seconds(1))
28301 .expect("test generation should advance the initialized pool metadata");
28302 let earlier_tick = generation - Duration::nanoseconds(10);
28303 {
28304 let mut pool_meta = store.pool_meta.write().await;
28305 pool_meta.dont_save = false;
28306 pool_meta.pools[0].last_update = generation;
28307 pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
28308 start_time: Some(generation),
28309 ..Default::default()
28310 });
28311 }
28312 store
28313 .save_current_pool_meta_for_test(&[0])
28314 .await
28315 .expect("active decommission metadata should persist before rollback checkpoints");
28316
28317 let unresolved_entry = DecommissionUnresolvedEntry {
28318 bucket: "bucket-a".to_string(),
28319 object: "object-a".to_string(),
28320 pool_index: 0,
28321 set_index: 0,
28322 source_generation: generation,
28323 candidate_count: 1,
28324 disk_error_count: 1,
28325 observed_at: earlier_tick,
28326 reason: "metadata_resolution_failed".to_string(),
28327 };
28328 {
28329 let mut pool_meta = store.pool_meta.write().await;
28330 assert!(
28331 record_decommission_unresolved_entry(&mut pool_meta, 0, generation, unresolved_entry, earlier_tick, None)
28332 .expect("unresolved entry should record under active generation")
28333 );
28334 assert_eq!(pool_meta.pools[0].last_update, generation + Duration::nanoseconds(1));
28335 }
28336 store
28337 .save_current_pool_meta_for_test(&[0])
28338 .await
28339 .expect("rollback unresolved-entry save should not stale-reject");
28340 store.pool_meta.write().await.pools[0]
28341 .decommission
28342 .as_mut()
28343 .expect("decommission metadata should exist")
28344 .items_decommissioned = DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD;
28345
28346 assert!(
28347 store
28348 .save_decommission_progress_checkpoint_at(0, generation, earlier_tick)
28349 .await
28350 .expect("rollback progress checkpoint should use a monotonic durable identity")
28351 );
28352 let progress_at = generation + Duration::nanoseconds(2);
28353 assert_eq!(store.pool_meta.read().await.pools[0].last_update, progress_at);
28354
28355 let terminal_at = generation + Duration::nanoseconds(3);
28356 {
28357 let mut pool_meta = store.pool_meta.write().await;
28358 assert!(pool_meta.decommission_failed_at_for_test(0, earlier_tick, None));
28359 assert_eq!(pool_meta.pools[0].last_update, terminal_at);
28360 }
28361 store
28362 .save_current_pool_meta_for_test(&[0])
28363 .await
28364 .expect("terminal failure after rollback checkpoints should persist");
28365
28366 let mut restored = PoolMeta::default();
28367 restored
28368 .load_no_lock_from_replicas(vec![store.pools[0].clone()])
28369 .await
28370 .expect("terminal metadata should reload after restart");
28371 assert_eq!(restored.pools[0].last_update, terminal_at);
28372 assert!(
28373 restored.pools[0]
28374 .decommission
28375 .as_ref()
28376 .is_some_and(|info| info.failed && !info.complete && !info.canceled)
28377 );
28378
28379 let restarted = decommission_worker_test_store(restored, Vec::new());
28380 let status = restarted.scanner_data_movement_pause_status().await;
28381 let expected_generation =
28382 u64::try_from(terminal_at.unix_timestamp_nanos()).expect("fixed positive timestamp should fit generation");
28383
28384 assert_eq!(status.movement_generation, expected_generation);
28385 assert_eq!(status.reasons, vec![crate::store::ScannerDataMovementPauseReason::DecommissionFailed]);
28386 assert_eq!(restarted.scanner_data_movement_generation(), expected_generation);
28387 }
28388
28389 #[tokio::test]
28390 async fn unresolved_entry_probe_verifies_absence_on_every_source_disk() {
28391 let (_dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
28392 let bucket = "decommission-final-sweep-absent-ledger";
28393 store
28394 .peer_sys
28395 .make_bucket(bucket, &MakeBucketOptions::default())
28396 .await
28397 .expect("test bucket should be created");
28398 metadata_sys::init_bucket_metadata_sys(store.clone(), vec![bucket.to_string()]).await;
28399
28400 let generation = OffsetDateTime::now_utc();
28401 let unresolved_entry = DecommissionUnresolvedEntry {
28402 bucket: bucket.to_string(),
28403 object: "absent-object".to_string(),
28404 pool_index: 0,
28405 set_index: 0,
28406 source_generation: generation,
28407 candidate_count: 1,
28408 disk_error_count: 1,
28409 observed_at: generation,
28410 reason: "metadata_resolution_failed".to_string(),
28411 };
28412 {
28413 let mut pool_meta = store.pool_meta.write().await;
28414 pool_meta.version = POOL_META_VERSION;
28415 pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
28416 start_time: Some(generation),
28417 unresolved_entries: vec![unresolved_entry.clone()],
28418 ..Default::default()
28419 });
28420 }
28421
28422 assert!(
28423 store.pools[0].disk_set[0]
28424 .decommission_unresolved_entry_absent_on_all_disks(0, &unresolved_entry)
28425 .await
28426 .expect("all source disks should be readable")
28427 );
28428 let verified = vec![unresolved_entry.clone()];
28429
28430 let mut pool_meta = store.pool_meta.write().await;
28431 reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), Some(&verified))
28432 .expect("the individually verified entry should reconcile");
28433 assert!(
28434 pool_meta.pools[0]
28435 .decommission
28436 .as_ref()
28437 .expect("decommission should remain present")
28438 .unresolved_entries
28439 .is_empty()
28440 );
28441 }
28442
28443 #[tokio::test]
28444 async fn test_record_decommission_entry_error_cancels_listing_and_preserves_first_error() {
28445 let entry_error = Arc::new(tokio::sync::Mutex::new(None));
28446 let rx = CancellationToken::new();
28447
28448 assert!(record_decommission_entry_error(&entry_error, &rx, Error::SlowDown).await);
28449 assert!(!record_decommission_entry_error(&entry_error, &rx, Error::OperationCanceled).await);
28450
28451 assert!(rx.is_cancelled());
28452 assert!(matches!(*entry_error.lock().await, Some(Error::SlowDown)));
28453 }
28454
28455 #[tokio::test]
28456 async fn test_record_decommission_entry_error_ignores_already_canceled_listing() {
28457 let entry_error = Arc::new(tokio::sync::Mutex::new(None));
28458 let rx = CancellationToken::new();
28459 rx.cancel();
28460
28461 assert!(!record_decommission_entry_error(&entry_error, &rx, Error::SlowDown).await);
28462
28463 assert!(entry_error.lock().await.is_none());
28464 }
28465
28466 #[test]
28467 fn unresolved_entry_verification_requires_every_source_disk() {
28468 assert!(ensure_decommission_unresolved_verification_disk_count(4, 4, 0, 1).is_ok());
28469
28470 let err = ensure_decommission_unresolved_verification_disk_count(4, 3, 0, 1)
28471 .expect_err("an offline source disk must block ledger reconciliation");
28472 assert!(err.to_string().contains("requires all 4 source disks, but only 3 are online"));
28473 }
28474
28475 #[test]
28476 fn test_resolve_decommission_listing_error_preserves_real_listing_failure() {
28477 let err = resolve_decommission_listing_error(Some(Error::SlowDown), Some(Error::OperationCanceled))
28478 .expect("listing failure should be returned");
28479 assert!(matches!(err, Error::SlowDown));
28480
28481 let err = resolve_decommission_listing_error(Some(Error::OperationCanceled), Some(Error::SlowDown))
28482 .expect("entry failure should be returned");
28483 assert!(matches!(err, Error::SlowDown));
28484 }
28485
28486 #[test]
28487 fn test_resolve_decommission_check_after_list_result_returns_list_result_without_entry_error() {
28488 let err = resolve_decommission_check_after_list_result(Err(Error::OperationCanceled), None)
28489 .expect_err("list result should be preserved without entry error");
28490 assert!(matches!(err, Error::OperationCanceled));
28491 }
28492
28493 #[test]
28494 fn test_resolve_decommission_pool_meta_reload_result_passthrough_ok() {
28495 assert!(resolve_decommission_pool_meta_reload_result(Ok(()), "start_decommission").is_ok());
28496 }
28497
28498 #[test]
28499 fn test_resolve_decommission_pool_meta_reload_result_wraps_error_context() {
28500 let err = resolve_decommission_pool_meta_reload_result(Err(Error::SlowDown), "decommission_failed for pool 3")
28501 .expect_err("reload failure should be wrapped with stage context");
28502 let message = err.to_string();
28503 assert!(message.contains("decommission pool meta reload failed during decommission_failed for pool 3"));
28504 assert!(message.contains(Error::SlowDown.to_string().as_str()));
28505 }
28506
28507 #[test]
28508 fn test_resolve_start_decommission_pool_meta_reload_result_returns_failure() {
28509 let err = resolve_start_decommission_pool_meta_reload_result(Err(Error::other(
28510 "reload_pool_meta encountered 1 failure(s): peer[0] reload_pool_meta failed",
28511 )))
28512 .expect_err("start_decommission must fail when peer pool meta reload fails");
28513 let message = err.to_string();
28514
28515 assert!(message.contains("decommission pool meta reload failed during start_decommission"));
28516 assert!(message.contains("reload_pool_meta encountered 1 failure(s)"));
28517 assert!(message.contains("peer[0]"));
28518 }
28519
28520 #[test]
28521 fn test_resolve_decommission_listing_worker_result_passthrough_ok() {
28522 assert!(resolve_decommission_listing_worker_result(2, Ok(Ok(()))).is_ok());
28523 }
28524
28525 #[test]
28526 fn test_resolve_decommission_listing_worker_result_passthrough_worker_error() {
28527 let err = resolve_decommission_listing_worker_result(2, Ok(Err(Error::SlowDown)))
28528 .expect_err("listing worker error should be returned");
28529
28530 assert!(matches!(err, Error::SlowDown));
28531 }
28532
28533 #[tokio::test]
28534 async fn test_resolve_decommission_listing_worker_result_wraps_join_error_context() {
28535 let join_error = tokio::spawn(async {
28536 panic!("listing worker panic");
28537 })
28538 .await
28539 .expect_err("panic task should return JoinError");
28540
28541 let err = resolve_decommission_listing_worker_result(4, Err(join_error))
28542 .expect_err("join error should be wrapped with context");
28543 let message = err.to_string();
28544 assert!(message.contains("decommission listing worker 4 task join error"));
28545 assert!(message.contains("panic"));
28546 }
28547
28548 #[test]
28549 fn test_should_retry_decommission_listing_respects_attempt_limit_and_bucket_missing() {
28550 assert!(should_retry_decommission_listing(&Error::SlowDown, 0, 2));
28551 assert!(!should_retry_decommission_listing(&Error::SlowDown, 1, 2));
28552 assert!(!should_retry_decommission_listing(
28553 &StorageError::BucketNotFound("bucket".to_string()),
28554 0,
28555 2
28556 ));
28557 }
28558
28559 #[tokio::test]
28560 async fn test_wait_decommission_retry_backoff_reports_canceled_without_sleeping() {
28561 let token = CancellationToken::new();
28562 token.cancel();
28563
28564 assert!(wait_decommission_retry_backoff(&token, StdDuration::from_secs(30)).await);
28565 }
28566
28567 #[test]
28568 fn test_decommission_retry_backoff_ceiling_grows_exponentially_and_caps() {
28569 let base = StdDuration::from_millis(100);
28570
28571 assert_eq!(decommission_retry_backoff_ceiling(base, 1), StdDuration::from_millis(100));
28572 assert_eq!(decommission_retry_backoff_ceiling(base, 2), StdDuration::from_millis(200));
28573 assert_eq!(decommission_retry_backoff_ceiling(base, 3), StdDuration::from_millis(400));
28574 assert_eq!(decommission_retry_backoff_ceiling(base, 4), StdDuration::from_millis(800));
28575 assert_eq!(decommission_retry_backoff_ceiling(base, 5), DECOMMISSION_RETRY_BACKOFF_CAP);
28576 assert_eq!(decommission_retry_backoff_ceiling(base, usize::MAX), DECOMMISSION_RETRY_BACKOFF_CAP);
28577 }
28578
28579 #[test]
28580 fn test_decommission_retry_backoff_delay_stays_within_jittered_ceiling() {
28581 let base = StdDuration::from_millis(100);
28582
28583 for attempt in 1..=6 {
28584 let ceiling = decommission_retry_backoff_ceiling(base, attempt);
28585 let delay = decommission_retry_backoff_delay(base, attempt);
28586 assert!(delay <= ceiling, "delay {delay:?} must not exceed ceiling {ceiling:?}");
28587 }
28588 assert_eq!(decommission_retry_backoff_delay(StdDuration::ZERO, 3), StdDuration::ZERO);
28589 }
28590
28591 #[test]
28592 fn test_source_changed_exhaustion_fails_only_after_pool_limit() {
28593 assert!(!should_fail_decommission_pool_after_exhausted_source_changed(
28594 DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT
28595 ));
28596 assert!(should_fail_decommission_pool_after_exhausted_source_changed(
28597 DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT + 1
28598 ));
28599 }
28600
28601 #[tokio::test(start_paused = true)]
28602 async fn test_run_decommission_listing_with_retry_stops_after_attempt_limit() {
28603 let attempts = Arc::new(AtomicUsize::new(0));
28604 let err = run_decommission_listing_with_retry(
28605 CancellationToken::new(),
28606 "bucket-a".to_string(),
28607 noop_decommission_list_callback(),
28608 1,
28609 2,
28610 3,
28611 {
28612 let attempts = attempts.clone();
28613 move |_| {
28614 let attempts = attempts.clone();
28615 async move {
28616 attempts.fetch_add(1, Ordering::SeqCst);
28617 Err(Error::SlowDown)
28618 }
28619 }
28620 },
28621 )
28622 .await
28623 .expect_err("permanent listing failure must not retry forever");
28624
28625 assert_eq!(attempts.load(Ordering::SeqCst), 3);
28626 assert!(err.to_string().contains("attempt 3/3"));
28627 }
28628
28629 #[tokio::test]
28630 async fn test_run_decommission_listing_with_retry_treats_bucket_missing_as_complete() {
28631 let attempts = Arc::new(AtomicUsize::new(0));
28632 run_decommission_listing_with_retry(
28633 CancellationToken::new(),
28634 "bucket-a".to_string(),
28635 noop_decommission_list_callback(),
28636 1,
28637 2,
28638 3,
28639 {
28640 let attempts = attempts.clone();
28641 move |_| {
28642 let attempts = attempts.clone();
28643 async move {
28644 attempts.fetch_add(1, Ordering::SeqCst);
28645 Err(StorageError::BucketNotFound("bucket-a".to_string()))
28646 }
28647 }
28648 },
28649 )
28650 .await
28651 .expect("missing bucket should keep previous decommission listing behavior");
28652
28653 assert_eq!(attempts.load(Ordering::SeqCst), 1);
28654 }
28655
28656 #[test]
28657 fn test_should_count_decommission_version_complete_for_cleanup_safe_ignored_result() {
28658 assert!(should_count_decommission_version_complete(true, true, false));
28659 }
28660
28661 #[test]
28662 fn test_should_count_decommission_version_complete_rejects_skip_only_ignored_result() {
28663 assert!(!should_count_decommission_version_complete(true, false, false));
28664 }
28665
28666 #[test]
28667 fn test_should_count_decommission_version_complete_for_completed_result() {
28668 assert!(should_count_decommission_version_complete(false, false, false));
28669 }
28670
28671 #[test]
28672 fn test_should_count_decommission_version_complete_rejects_failed_result() {
28673 assert!(!should_count_decommission_version_complete(false, false, true));
28674 }
28675
28676 #[test]
28677 fn test_should_cleanup_decommission_source_entry_accepts_all_versions_completed() {
28678 assert!(should_cleanup_decommission_source_entry(3, 3, 0));
28679 }
28680
28681 #[test]
28682 fn test_should_cleanup_decommission_source_entry_accepts_migrated_and_safely_expired_versions() {
28683 assert!(should_cleanup_decommission_source_entry(1, 2, 1));
28684 }
28685
28686 #[test]
28687 fn test_should_cleanup_decommission_source_entry_accepts_versions_only_safely_expired_by_lifecycle() {
28688 assert!(should_cleanup_decommission_source_entry(0, 2, 2));
28689 }
28690
28691 #[test]
28692 fn test_should_cleanup_decommission_source_entry_rejects_object_lock_retained_version() {
28693 assert!(!should_cleanup_decommission_source_entry(1, 2, 0));
28694 }
28695
28696 #[test]
28697 fn test_should_cleanup_decommission_source_entry_rejects_replication_pending_version() {
28698 assert!(!should_cleanup_decommission_source_entry(2, 3, 0));
28699 }
28700
28701 #[test]
28702 fn test_should_cleanup_decommission_source_entry_rejects_counter_overrun() {
28703 assert!(!should_cleanup_decommission_source_entry(2, 2, 1));
28704 }
28705
28706 #[test]
28707 fn test_pool_meta_update_after_rejects_out_of_range_index() {
28708 let mut meta = PoolMeta::default();
28709 let err = meta
28710 .update_after(1, Duration::seconds(1))
28711 .expect_err("out-of-range index should fail");
28712 assert!(err.to_string().contains("invalid decommission pool index 1 for 0 pools"));
28713 }
28714
28715 #[test]
28716 fn test_pool_meta_update_after_rejects_when_decommission_missing() {
28717 let mut meta = PoolMeta {
28718 pools: vec![PoolStatus {
28719 id: 0,
28720 cmd_line: "pool-0".to_string(),
28721 last_update: OffsetDateTime::UNIX_EPOCH,
28722 decommission: None,
28723 }],
28724 ..Default::default()
28725 };
28726
28727 let err = meta
28728 .update_after(0, Duration::seconds(1))
28729 .expect_err("pool without decommission should fail");
28730 assert!(
28731 err.to_string()
28732 .contains("failed to update decommission metadata timestamp: decommission metadata not initialized")
28733 );
28734 }
28735
28736 #[test]
28737 fn test_track_decommission_stage_does_not_advance_checkpoint_state() {
28738 let mut meta = PoolMeta {
28739 pools: vec![PoolStatus {
28740 id: 0,
28741 cmd_line: "pool-0".to_string(),
28742 last_update: OffsetDateTime::UNIX_EPOCH,
28743 decommission: Some(PoolDecommissionInfo {
28744 items_decommissioned: 3,
28745 items_decommission_failed: 2,
28746 ..Default::default()
28747 }),
28748 }],
28749 ..Default::default()
28750 };
28751
28752 track_decommission_current_object_stage(&mut meta, 0, "bucket", "object", "migrate_object")
28753 .expect("valid decommission progress should be tracked");
28754
28755 assert_eq!(meta.pools[0].last_update, OffsetDateTime::UNIX_EPOCH);
28756 let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
28757 assert_eq!(info.items_since_last_progress_save(), 5);
28758 assert_eq!(info.stage, "migrate_object");
28759 }
28760
28761 #[test]
28762 fn test_pool_meta_update_after_skips_before_time_and_item_thresholds() {
28763 let mut meta = PoolMeta {
28764 pools: vec![PoolStatus {
28765 id: 0,
28766 cmd_line: "pool-0".to_string(),
28767 last_update: OffsetDateTime::now_utc(),
28768 decommission: Some(PoolDecommissionInfo {
28769 items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD - 1,
28770 ..Default::default()
28771 }),
28772 }],
28773 ..Default::default()
28774 };
28775
28776 let saved = meta
28777 .update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
28778 .expect("valid decommission state should update");
28779
28780 assert!(!saved);
28781 let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
28782 assert_eq!(info.items_since_last_progress_save(), DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD - 1);
28783 }
28784
28785 #[test]
28786 fn test_pool_meta_update_after_requests_save_when_item_threshold_reached() {
28787 let mut meta = PoolMeta {
28788 pools: vec![PoolStatus {
28789 id: 0,
28790 cmd_line: "pool-0".to_string(),
28791 last_update: OffsetDateTime::now_utc(),
28792 decommission: Some(PoolDecommissionInfo {
28793 items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
28794 ..Default::default()
28795 }),
28796 }],
28797 ..Default::default()
28798 };
28799
28800 let saved = meta
28801 .update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
28802 .expect("item threshold should save progress");
28803
28804 assert!(saved);
28805 let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
28806 assert_eq!(info.items_since_last_progress_save(), DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD);
28807 }
28808
28809 #[test]
28810 fn test_pool_meta_update_after_requests_save_when_time_threshold_reached() {
28811 let mut meta = PoolMeta {
28812 pools: vec![PoolStatus {
28813 id: 0,
28814 cmd_line: "pool-0".to_string(),
28815 last_update: OffsetDateTime::now_utc() - DECOMMISSION_PROGRESS_SAVE_INTERVAL,
28816 decommission: Some(PoolDecommissionInfo {
28817 items_decommissioned: 1,
28818 ..Default::default()
28819 }),
28820 }],
28821 ..Default::default()
28822 };
28823
28824 let saved = meta
28825 .update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
28826 .expect("time threshold should save progress");
28827
28828 assert!(saved);
28829 let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
28830 assert_eq!(info.items_since_last_progress_save(), 1);
28831 }
28832
28833 #[test]
28834 fn test_pool_meta_update_after_does_not_advance_last_update_before_save() {
28835 let last_update = OffsetDateTime::UNIX_EPOCH;
28836 let mut meta = PoolMeta {
28837 pools: vec![PoolStatus {
28838 id: 0,
28839 cmd_line: "pool-0".to_string(),
28840 last_update,
28841 decommission: Some(PoolDecommissionInfo {
28842 start_time: Some(last_update),
28843 items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
28844 ..Default::default()
28845 }),
28846 }],
28847 ..Default::default()
28848 };
28849
28850 assert!(
28851 meta.update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
28852 .expect("item threshold should request a checkpoint")
28853 );
28854 assert_eq!(meta.pools[0].last_update, last_update);
28855 }
28856
28857 #[test]
28858 fn test_decommission_progress_checkpoint_commits_exact_snapshot_watermark() {
28859 let start_time = OffsetDateTime::UNIX_EPOCH;
28860 let checkpoint_at = start_time + Duration::seconds(30);
28861 let mut meta = PoolMeta {
28862 pools: vec![PoolStatus {
28863 id: 0,
28864 cmd_line: "pool-0".to_string(),
28865 last_update: start_time,
28866 decommission: Some(PoolDecommissionInfo {
28867 start_time: Some(start_time),
28868 items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
28869 ..Default::default()
28870 }),
28871 }],
28872 ..Default::default()
28873 };
28874
28875 let checkpoint = meta
28876 .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None)
28877 .expect("valid decommission state should produce a checkpoint")
28878 .expect("item threshold should produce a checkpoint");
28879 meta.count_item(0, 1, false);
28880
28881 assert!(meta.commit_decommission_progress_checkpoint(0, checkpoint));
28882 let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
28883 assert_eq!(info.progress_save_item_baseline, checkpoint.counted_items);
28884 assert_eq!(info.items_since_last_progress_save(), 1);
28885 assert_eq!(meta.pools[0].last_update, checkpoint_at);
28886 }
28887
28888 #[test]
28889 fn test_decommission_progress_checkpoint_backoff_does_not_advance_baseline() {
28890 let start_time = OffsetDateTime::UNIX_EPOCH;
28891 let checkpoint_at = start_time + Duration::seconds(30);
28892 let retry_after = checkpoint_at + DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
28893 let mut meta = PoolMeta {
28894 pools: vec![PoolStatus {
28895 id: 0,
28896 cmd_line: "pool-0".to_string(),
28897 last_update: start_time,
28898 decommission: Some(PoolDecommissionInfo {
28899 start_time: Some(start_time),
28900 items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
28901 ..Default::default()
28902 }),
28903 }],
28904 ..Default::default()
28905 };
28906
28907 let checkpoint = meta
28908 .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None)
28909 .expect("valid decommission state should produce a checkpoint")
28910 .expect("item threshold should produce a checkpoint");
28911 meta.defer_decommission_progress_checkpoint(0, checkpoint, retry_after);
28912
28913 assert!(
28914 meta.decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None)
28915 .expect("retry backoff check should succeed")
28916 .is_none()
28917 );
28918 assert_eq!(meta.pools[0].last_update, start_time);
28919 assert_eq!(
28920 meta.pools[0]
28921 .decommission
28922 .as_ref()
28923 .expect("decommission info should exist")
28924 .progress_save_item_baseline,
28925 0
28926 );
28927 }
28928
28929 #[test]
28930 fn test_decommission_progress_checkpoint_count_scales_with_threshold() {
28931 let start_time = OffsetDateTime::UNIX_EPOCH;
28932 let checkpoint_at = start_time;
28933 let mut meta = PoolMeta {
28934 pools: vec![PoolStatus {
28935 id: 0,
28936 cmd_line: "pool-0".to_string(),
28937 last_update: start_time,
28938 decommission: Some(PoolDecommissionInfo {
28939 start_time: Some(start_time),
28940 ..Default::default()
28941 }),
28942 }],
28943 ..Default::default()
28944 };
28945 let mut checkpoint_count = 0;
28946
28947 for _ in 0..(DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD * 10) {
28948 meta.count_item(0, 1, false);
28949 if let Some(checkpoint) = meta
28950 .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None)
28951 .expect("valid decommission state should produce a checkpoint")
28952 {
28953 checkpoint_count += 1;
28954 assert!(meta.commit_decommission_progress_checkpoint(0, checkpoint));
28955 }
28956 }
28957
28958 assert_eq!(checkpoint_count, 10);
28959 }
28960
28961 #[test]
28962 fn test_ensure_decommission_not_rebalancing_rejects_running_rebalance() {
28963 let err = ensure_decommission_not_rebalancing(true).expect_err("rebalance running should be rejected");
28964 assert!(matches!(err, Error::RebalanceAlreadyRunning));
28965 }
28966
28967 #[test]
28968 fn test_ensure_decommission_not_rebalancing_allows_idle() {
28969 assert!(ensure_decommission_not_rebalancing(false).is_ok());
28970 }
28971
28972 #[test]
28973 fn test_ensure_decommission_start_rebalance_meta_allowed_rejects_active_rebalance() {
28974 let meta = RebalanceMeta {
28975 pool_stats: vec![RebalanceStats {
28976 participating: true,
28977 info: RebalanceInfo {
28978 status: RebalStatus::Started,
28979 ..Default::default()
28980 },
28981 ..Default::default()
28982 }],
28983 ..Default::default()
28984 };
28985
28986 let err = ensure_decommission_start_rebalance_meta_allowed(Some(&meta))
28987 .expect_err("persisted active rebalance should block decommission start");
28988
28989 assert!(matches!(err, Error::RebalanceAlreadyRunning));
28990 }
28991
28992 #[test]
28993 fn test_ensure_decommission_start_rebalance_meta_allowed_rejects_stopping_rebalance() {
28994 let meta = RebalanceMeta {
28995 pool_stats: vec![RebalanceStats {
28996 info: RebalanceInfo {
28997 status: RebalStatus::Started,
28998 stopping: true,
28999 ..Default::default()
29000 },
29001 ..Default::default()
29002 }],
29003 ..Default::default()
29004 };
29005
29006 let err = ensure_decommission_start_rebalance_meta_allowed(Some(&meta))
29007 .expect_err("persisted stopping rebalance should block decommission start");
29008
29009 assert!(matches!(err, Error::RebalanceAlreadyRunning));
29010 }
29011
29012 #[test]
29013 fn test_ensure_decommission_start_rebalance_meta_allowed_allows_terminal_or_missing_rebalance() {
29014 let terminal_meta = RebalanceMeta {
29015 pool_stats: vec![RebalanceStats {
29016 participating: true,
29017 info: RebalanceInfo {
29018 status: RebalStatus::Completed,
29019 ..Default::default()
29020 },
29021 ..Default::default()
29022 }],
29023 ..Default::default()
29024 };
29025
29026 assert!(ensure_decommission_start_rebalance_meta_allowed(Some(&terminal_meta)).is_ok());
29027 assert!(ensure_decommission_start_rebalance_meta_allowed(None).is_ok());
29028 }
29029
29030 #[test]
29031 fn test_ensure_local_decommission_pool_leaders_allows_local_first_endpoint() {
29032 let endpoints = EndpointServerPools::from(vec![
29033 decommission_test_pool_endpoint(0, false),
29034 decommission_test_pool_endpoint(1, true),
29035 ]);
29036
29037 assert!(ensure_local_decommission_pool_leaders(&endpoints, &[1]).is_ok());
29038 }
29039
29040 #[test]
29041 fn test_ensure_local_decommission_pool_leaders_rejects_remote_first_endpoint() {
29042 let endpoints = EndpointServerPools::from(vec![decommission_test_pool_endpoint(0, false)]);
29043
29044 let err = ensure_local_decommission_pool_leaders(&endpoints, &[0])
29045 .expect_err("remote first endpoint should reject local decommission start");
29046
29047 assert!(err.to_string().contains("must run on the pool first endpoint"));
29048 }
29049
29050 #[test]
29051 fn test_ensure_local_decommission_pool_leaders_rejects_empty_endpoints() {
29052 let endpoints = EndpointServerPools::from(vec![PoolEndpoints {
29053 legacy: false,
29054 set_count: 1,
29055 drives_per_set: 1,
29056 endpoints: Endpoints::from(Vec::<Endpoint>::new()),
29057 cmd_line: "pool-0".to_string(),
29058 platform: String::new(),
29059 }]);
29060
29061 let err = ensure_local_decommission_pool_leaders(&endpoints, &[0])
29062 .expect_err("pool without endpoints should reject local decommission start");
29063
29064 assert!(err.to_string().contains("has no configured endpoints"));
29065 }
29066
29067 #[test]
29068 fn test_decommission_meta_bucket_options_are_idempotent() {
29069 let opts = decommission_meta_bucket_options();
29070
29071 assert!(opts.force_create);
29072 }
29073
29074 #[test]
29075 fn test_is_decommission_active_true_only_when_not_terminal() {
29076 assert!(is_decommission_active(false, false, false));
29077 assert!(!is_decommission_active(true, false, false));
29078 assert!(!is_decommission_active(false, true, false));
29079 assert!(!is_decommission_active(false, false, true));
29080 }
29081
29082 #[test]
29083 fn test_ensure_decommission_generation_rejects_stale_or_queued_workers() {
29084 let generation = OffsetDateTime::UNIX_EPOCH;
29085 let mut meta = PoolMeta {
29086 pools: vec![PoolStatus {
29087 id: 0,
29088 cmd_line: "pool-0".to_string(),
29089 last_update: generation,
29090 decommission: Some(PoolDecommissionInfo {
29091 start_time: Some(generation),
29092 ..Default::default()
29093 }),
29094 }],
29095 ..Default::default()
29096 };
29097
29098 assert!(ensure_decommission_generation(&meta, 0, generation).is_ok());
29099 assert!(ensure_decommission_generation(&meta, 0, generation + Duration::seconds(1)).is_err());
29100
29101 meta.pools[0]
29102 .decommission
29103 .as_mut()
29104 .expect("decommission metadata should exist")
29105 .queued = true;
29106 assert!(ensure_decommission_generation(&meta, 0, generation).is_err());
29107
29108 let replacement_generation = generation + Duration::seconds(2);
29109 let info = meta.pools[0]
29110 .decommission
29111 .as_mut()
29112 .expect("decommission metadata should exist");
29113 info.queued = false;
29114 info.start_time = Some(replacement_generation);
29115 assert!(ensure_decommission_generation(&meta, 0, generation).is_err());
29116 assert!(ensure_decommission_generation(&meta, 0, replacement_generation).is_ok());
29117 }
29118
29119 #[test]
29120 fn test_pool_meta_has_active_decommission_counts_running_and_queued_states() {
29121 let active_meta = PoolMeta {
29122 pools: vec![PoolStatus {
29123 id: 0,
29124 cmd_line: "pool-0".to_string(),
29125 last_update: OffsetDateTime::UNIX_EPOCH,
29126 decommission: Some(PoolDecommissionInfo {
29127 start_time: Some(OffsetDateTime::UNIX_EPOCH),
29128 ..Default::default()
29129 }),
29130 }],
29131 ..Default::default()
29132 };
29133 let queued_meta = PoolMeta {
29134 pools: vec![PoolStatus {
29135 id: 0,
29136 cmd_line: "pool-0".to_string(),
29137 last_update: OffsetDateTime::UNIX_EPOCH,
29138 decommission: Some(PoolDecommissionInfo {
29139 queued: true,
29140 ..Default::default()
29141 }),
29142 }],
29143 ..Default::default()
29144 };
29145
29146 assert!(pool_meta_has_active_decommission(&active_meta));
29147 assert!(pool_meta_has_active_decommission(&queued_meta));
29148 }
29149
29150 #[test]
29151 fn test_pool_meta_has_active_decommission_ignores_capacity_placeholder() {
29152 let meta = PoolMeta {
29153 pools: vec![PoolStatus {
29154 id: 0,
29155 cmd_line: "pool-0".to_string(),
29156 last_update: OffsetDateTime::UNIX_EPOCH,
29157 decommission: Some(PoolDecommissionInfo {
29158 total_size: 100,
29159 current_size: 75,
29160 ..Default::default()
29161 }),
29162 }],
29163 ..Default::default()
29164 };
29165
29166 assert!(!pool_meta_has_active_decommission(&meta));
29167 assert!(!meta.is_suspended(0));
29168 assert_eq!(decommission_start_pool_state(meta.pools.first()), DecommissionStartPoolState::Active);
29169 }
29170
29171 #[test]
29172 fn test_pool_meta_has_active_decommission_ignores_terminal_states() {
29173 let terminal_meta = PoolMeta {
29174 pools: vec![
29175 PoolStatus {
29176 id: 0,
29177 cmd_line: "pool-0".to_string(),
29178 last_update: OffsetDateTime::UNIX_EPOCH,
29179 decommission: Some(PoolDecommissionInfo {
29180 complete: true,
29181 ..Default::default()
29182 }),
29183 },
29184 PoolStatus {
29185 id: 1,
29186 cmd_line: "pool-1".to_string(),
29187 last_update: OffsetDateTime::UNIX_EPOCH,
29188 decommission: Some(PoolDecommissionInfo {
29189 failed: true,
29190 ..Default::default()
29191 }),
29192 },
29193 PoolStatus {
29194 id: 2,
29195 cmd_line: "pool-2".to_string(),
29196 last_update: OffsetDateTime::UNIX_EPOCH,
29197 decommission: Some(PoolDecommissionInfo {
29198 canceled: true,
29199 ..Default::default()
29200 }),
29201 },
29202 ],
29203 ..Default::default()
29204 };
29205
29206 assert!(!pool_meta_has_active_decommission(&terminal_meta));
29207 }
29208
29209 #[test]
29210 fn test_decommission_request_rejections_preserve_invalid_argument_type() {
29211 for result in [
29212 ensure_decommission_start_allowed(DecommissionStartPoolState::Missing),
29213 ensure_decommission_start_allowed(DecommissionStartPoolState::Decommissioned),
29214 ensure_decommission_start_allowed(DecommissionStartPoolState::Blocked),
29215 ensure_decommission_cancel_allowed(false, false, false),
29216 ensure_decommission_clear_allowed(false, false, false, false, false, 0),
29217 ensure_decommission_clear_allowed(true, true, false, true, false, 1),
29218 ensure_decommission_terminal_operation_supported(true, "cancel decommission"),
29219 validate_start_decommission_request(&[], false),
29220 validate_start_decommission_request(&[0], true),
29221 ensure_decommission_start_keeps_active_pool(&PoolMeta::default(), &[]),
29222 ] {
29223 let err = result.expect_err("invalid lifecycle requests must be rejected before mutation");
29224 assert!(matches!(&err, Error::InvalidArgument(_, _, reason) if !reason.is_empty()), "{err:?}");
29225 }
29226 }
29227
29228 #[test]
29229 fn test_ensure_decommission_start_allowed_rejects_missing_pool() {
29230 let err =
29231 ensure_decommission_start_allowed(DecommissionStartPoolState::Missing).expect_err("missing pool should be invalid");
29232 assert!(
29233 err.to_string()
29234 .contains("failed to start decommission: target pool was not found")
29235 );
29236 }
29237
29238 #[test]
29239 fn test_ensure_decommission_start_allowed_rejects_running_state() {
29240 let err = ensure_decommission_start_allowed(DecommissionStartPoolState::Decommissioning)
29241 .expect_err("active decommission should be rejected");
29242 assert!(matches!(err, Error::DecommissionAlreadyRunning));
29243 }
29244
29245 #[test]
29246 fn test_ensure_decommission_start_allowed_rejects_completed_state() {
29247 let err = ensure_decommission_start_allowed(DecommissionStartPoolState::Decommissioned)
29248 .expect_err("completed decommission should be rejected");
29249 assert!(err.to_string().contains("target pool is already decommissioned"));
29250 }
29251
29252 #[test]
29253 fn test_ensure_decommission_start_allowed_rejects_blocked_state() {
29254 let err = ensure_decommission_start_allowed(DecommissionStartPoolState::Blocked)
29255 .expect_err("blocked decommission should be rejected");
29256 assert!(err.to_string().contains("target pool decommission is blocked"));
29257 }
29258
29259 #[test]
29260 fn test_ensure_decommission_start_allowed_allows_active_state() {
29261 assert!(ensure_decommission_start_allowed(DecommissionStartPoolState::Active).is_ok());
29262 }
29263
29264 #[test]
29265 fn test_ensure_decommission_start_allowed_allows_retryable_state() {
29266 assert!(ensure_decommission_start_allowed(DecommissionStartPoolState::Retryable).is_ok());
29267 }
29268
29269 #[test]
29270 fn test_decommission_start_pool_state_reports_missing_pool() {
29271 assert_eq!(decommission_start_pool_state(None), DecommissionStartPoolState::Missing);
29272 }
29273
29274 #[test]
29275 fn test_decommission_start_pool_state_reports_idle_pool_without_decommission_info() {
29276 let pool = PoolStatus {
29277 id: 0,
29278 cmd_line: "pool-0".to_string(),
29279 last_update: OffsetDateTime::UNIX_EPOCH,
29280 decommission: None,
29281 };
29282
29283 assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Active);
29284 }
29285
29286 #[test]
29287 fn test_decommission_start_pool_state_reports_decommissioning_pool_when_not_terminal() {
29288 let pool = PoolStatus {
29289 id: 0,
29290 cmd_line: "pool-0".to_string(),
29291 last_update: OffsetDateTime::UNIX_EPOCH,
29292 decommission: Some(PoolDecommissionInfo {
29293 start_time: Some(OffsetDateTime::UNIX_EPOCH),
29294 complete: false,
29295 failed: false,
29296 canceled: false,
29297 ..Default::default()
29298 }),
29299 };
29300
29301 assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Decommissioning);
29302 }
29303
29304 #[test]
29305 fn test_decommission_start_pool_state_reports_canceled_pool_as_blocked() {
29306 let pool = PoolStatus {
29307 id: 0,
29308 cmd_line: "pool-0".to_string(),
29309 last_update: OffsetDateTime::UNIX_EPOCH,
29310 decommission: Some(PoolDecommissionInfo {
29311 complete: false,
29312 failed: false,
29313 canceled: true,
29314 ..Default::default()
29315 }),
29316 };
29317
29318 assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Blocked);
29319 }
29320
29321 #[test]
29322 fn test_decommission_start_pool_state_reports_failed_pool_as_blocked() {
29323 let pool = PoolStatus {
29324 id: 0,
29325 cmd_line: "pool-0".to_string(),
29326 last_update: OffsetDateTime::UNIX_EPOCH,
29327 decommission: Some(PoolDecommissionInfo {
29328 failed: true,
29329 ..Default::default()
29330 }),
29331 };
29332
29333 assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Blocked);
29334 }
29335
29336 #[test]
29337 fn test_decommission_start_pool_state_reports_terminal_pool_with_unresolved_entries_as_retryable() {
29338 let pool = PoolStatus {
29339 id: 0,
29340 cmd_line: "pool-0".to_string(),
29341 last_update: OffsetDateTime::UNIX_EPOCH,
29342 decommission: Some(PoolDecommissionInfo {
29343 failed: true,
29344 unresolved_entries: vec![DecommissionUnresolvedEntry {
29345 bucket: "bucket-a".to_string(),
29346 object: "object-a".to_string(),
29347 pool_index: 0,
29348 set_index: 0,
29349 source_generation: OffsetDateTime::UNIX_EPOCH,
29350 candidate_count: 1,
29351 disk_error_count: 1,
29352 observed_at: OffsetDateTime::UNIX_EPOCH,
29353 reason: "metadata_resolution_failed".to_string(),
29354 }],
29355 ..Default::default()
29356 }),
29357 };
29358
29359 assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Retryable);
29360 }
29361
29362 #[test]
29363 fn test_decommission_start_pool_state_reports_completed_pool() {
29364 let pool = PoolStatus {
29365 id: 0,
29366 cmd_line: "pool-0".to_string(),
29367 last_update: OffsetDateTime::UNIX_EPOCH,
29368 decommission: Some(PoolDecommissionInfo {
29369 complete: true,
29370 ..Default::default()
29371 }),
29372 };
29373
29374 assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Decommissioned);
29375 }
29376
29377 #[test]
29378 fn test_ensure_decommission_start_keeps_active_pool_rejects_last_active_pool() {
29379 let meta = PoolMeta {
29380 pools: vec![PoolStatus {
29381 id: 0,
29382 cmd_line: "pool-0".to_string(),
29383 last_update: OffsetDateTime::UNIX_EPOCH,
29384 decommission: None,
29385 }],
29386 ..Default::default()
29387 };
29388
29389 let err = ensure_decommission_start_keeps_active_pool(&meta, &[0]).expect_err("last active pool should be rejected");
29390
29391 assert!(err.to_string().contains("at least one active pool must remain"));
29392 }
29393
29394 #[test]
29395 fn decommission_capacity_model_selection_uses_v2_only_with_a_live_fleet_proof() {
29396 let meta = PoolMeta::default();
29397
29398 assert_eq!(
29399 select_decommission_capacity_model(&meta, false).expect("a proofless empty cohort should remain compatible"),
29400 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION
29401 );
29402 assert_eq!(
29403 select_decommission_capacity_model(&meta, true).expect("an all-v4 proof should authorize the target fence"),
29404 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
29405 );
29406 }
29407
29408 #[test]
29409 fn decommission_capacity_model_selection_keeps_an_active_v1_cohort_sticky() {
29410 let meta = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION]);
29411
29412 assert_eq!(
29413 select_decommission_capacity_model(&meta, true).expect("a fleet upgrade must not change an active v1 lock model"),
29414 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION
29415 );
29416 }
29417
29418 #[test]
29419 fn decommission_capacity_model_selection_never_downgrades_an_active_v2_cohort() {
29420 let meta = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION]);
29421
29422 assert_eq!(
29423 select_decommission_capacity_model(&meta, true).expect("a live proof should admit another v2 reservation"),
29424 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
29425 );
29426 let err = select_decommission_capacity_model(&meta, false)
29427 .expect_err("proof loss must block new admission instead of falling back to the global lock model");
29428 assert!(err.to_string().contains("active per-target capacity cohort"));
29429 }
29430
29431 #[test]
29432 fn decommission_capacity_model_selection_rejects_a_mixed_active_cohort() {
29433 let meta = decommission_test_active_model_meta(&[
29434 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION,
29435 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
29436 ]);
29437
29438 let err = select_decommission_capacity_model(&meta, true)
29439 .expect_err("mixed global and per-target lock models cannot be made safe by a fleet proof");
29440 assert!(err.to_string().contains("mixed lock models"));
29441 }
29442
29443 #[test]
29444 fn test_ensure_decommission_start_target_capacity_allows_sufficient_free_space() {
29445 let meta = PoolMeta {
29446 version: POOL_META_VERSION,
29447 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
29448 ..Default::default()
29449 };
29450 let capacity_infos = vec![
29451 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 100, 700, 600),
29452 DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 1_600, 2_000, 400),
29453 ];
29454
29455 assert!(ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos, true).is_ok());
29456 }
29457
29458 #[test]
29459 fn test_ensure_decommission_start_target_capacity_rejects_insufficient_free_space() {
29460 let meta = PoolMeta {
29461 version: POOL_META_VERSION,
29462 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
29463 ..Default::default()
29464 };
29465 let capacity_infos = vec![
29466 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 100, 700, 600),
29467 DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 1_599, 2_000, 401),
29468 ];
29469
29470 let err = ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos, true)
29471 .expect_err("target physical free capacity below the modeled peak should be rejected");
29472
29473 assert!(err.to_string().contains("insufficient reserved physical target capacity"));
29474 assert!(err.to_string().contains("requires 1600 bytes, but 1599 bytes are available"));
29475 }
29476
29477 #[test]
29478 fn test_ensure_decommission_start_target_capacity_ignores_non_active_target_pool() {
29479 let meta = PoolMeta {
29480 version: POOL_META_VERSION,
29481 pools: vec![
29482 decommission_test_pool_status(0, None),
29483 decommission_test_pool_status(
29484 1,
29485 Some(PoolDecommissionInfo {
29486 complete: true,
29487 ..Default::default()
29488 }),
29489 ),
29490 decommission_test_pool_status(2, None),
29491 ],
29492 ..Default::default()
29493 };
29494 let capacity_infos = vec![
29495 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 100, 700, 600),
29496 DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 10_000, 10_000, 0),
29497 DecommissionPoolCapacityInfo::for_test(2, DecommissionErasureLayout { data: 4, parity: 4 }, 1_599, 2_000, 401),
29498 ];
29499
29500 let err = ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos, true)
29501 .expect_err("completed pools must not contribute target free capacity");
29502
29503 assert!(err.to_string().contains("requires 1600 bytes, but 1599 bytes are available"));
29504 }
29505
29506 #[test]
29507 fn decommission_capacity_model_converts_different_source_and_target_parity() {
29508 let now = OffsetDateTime::UNIX_EPOCH;
29509 let high_target_parity = build_decommission_capacity_reservation(
29510 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 0, 600, 600),
29511 DecommissionErasureLayout { data: 4, parity: 4 },
29512 uuid::Uuid::new_v4(),
29513 1,
29514 now,
29515 )
29516 .expect("the source and target layouts should be valid");
29517 assert_eq!(high_target_parity.source_data_equivalent_bytes, 400);
29518 assert_eq!(high_target_parity.predicted_physical_bytes, 800);
29519 assert_eq!(high_target_parity.temporary_physical_bytes, 800);
29520 assert_eq!(high_target_parity.peak_physical_bytes, 1_600);
29521
29522 let low_target_parity = build_decommission_capacity_reservation(
29523 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 4 }, 0, 800, 800),
29524 DecommissionErasureLayout { data: 4, parity: 2 },
29525 uuid::Uuid::new_v4(),
29526 2,
29527 now,
29528 )
29529 .expect("the inverse source and target layouts should be valid");
29530 assert_eq!(low_target_parity.source_data_equivalent_bytes, 400);
29531 assert_eq!(low_target_parity.predicted_physical_bytes, 600);
29532 assert_eq!(low_target_parity.peak_physical_bytes, 1_200);
29533 }
29534
29535 #[test]
29536 fn decommission_batch_persists_one_replayable_operation_identity_and_generation() {
29537 let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(1);
29538 let operation_id = uuid::Uuid::new_v4();
29539 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
29540 let capacity_infos = vec![
29541 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
29542 DecommissionPoolCapacityInfo::for_test(1, layout, 0, 20, 20),
29543 DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
29544 ];
29545 let mut meta = PoolMeta {
29546 version: POOL_META_VERSION,
29547 pools: vec![
29548 decommission_test_pool_status(0, None),
29549 decommission_test_pool_status(1, None),
29550 decommission_test_pool_status(2, None),
29551 ],
29552 ..Default::default()
29553 };
29554 meta.decommission(0, capacity_infos[0].space).unwrap();
29555 meta.queue_decommission(1, capacity_infos[1].space).unwrap();
29556
29557 reserve_decommission_start_target_capacity(
29558 &mut meta,
29559 &[0, 1],
29560 &capacity_infos,
29561 operation_id,
29562 17,
29563 now,
29564 DECOMMISSION_CAPACITY_MODEL_VERSION,
29565 )
29566 .expect("the batch reservation should fit exactly");
29567 for idx in [0, 1] {
29568 let reservation = meta.pools[idx]
29569 .decommission
29570 .as_ref()
29571 .and_then(|info| info.capacity_reservation.as_ref())
29572 .expect("each source pool should carry the batch reservation identity");
29573 assert_eq!(reservation.operation_id, operation_id);
29574 assert_eq!(reservation.generation, 17);
29575 }
29576 }
29577
29578 #[test]
29579 fn decommission_capacity_reservation_grows_after_stale_empty_snapshot() {
29580 let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(1);
29581 let layout = DecommissionErasureLayout { data: 3, parity: 1 };
29582 let capacity_infos = vec![
29583 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 0, 0),
29584 DecommissionPoolCapacityInfo::for_test(1, layout, 64, 64, 0),
29585 ];
29586 let mut meta = PoolMeta {
29587 version: POOL_META_VERSION,
29588 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
29589 ..Default::default()
29590 };
29591 meta.decommission(0, capacity_infos[0].space).unwrap();
29592 reserve_decommission_start_target_capacity(
29593 &mut meta,
29594 &[0],
29595 &capacity_infos,
29596 uuid::Uuid::new_v4(),
29597 1,
29598 now,
29599 DECOMMISSION_CAPACITY_MODEL_VERSION,
29600 )
29601 .expect("an empty scanner snapshot should still start decommission");
29602 let grew = grow_decommission_target_reservation(&mut meta, &capacity_infos, 0, 1, 1, uuid::Uuid::new_v4(), now)
29603 .expect("the reservation should grow against current target capacity");
29604 assert!(grew);
29605 let reservation = meta.pools[0]
29606 .decommission
29607 .as_ref()
29608 .and_then(|info| info.capacity_reservation.as_ref())
29609 .unwrap();
29610 assert_eq!(reservation.source_data_equivalent_bytes, 2);
29611 assert_eq!(reservation.predicted_physical_bytes, 3);
29612 assert_eq!(reservation.targets.len(), 1);
29613 assert_eq!(reservation.targets[0].reserved_physical_bytes, 6);
29614 }
29615
29616 #[test]
29617 fn decommission_capacity_model_reserves_versions_delete_markers_and_temporary_copies_from_physical_usage() {
29618 let reservation = build_decommission_capacity_reservation(
29619 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 2, parity: 2 }, 0, 1_024, 1_024),
29620 DecommissionErasureLayout { data: 2, parity: 2 },
29621 uuid::Uuid::new_v4(),
29622 1,
29623 OffsetDateTime::UNIX_EPOCH,
29624 )
29625 .expect("physical source usage should produce a reservation");
29626
29627 assert_eq!(reservation.source_physical_bytes, 1_024);
29628 assert_eq!(reservation.predicted_physical_bytes, 1_024);
29629 assert_eq!(reservation.temporary_copies, 1);
29630 assert_eq!(reservation.peak_physical_bytes, 2_048);
29631 }
29632
29633 #[test]
29634 fn decommission_runtime_capacity_tracks_own_target_consumption_but_rejects_external_drop() {
29635 let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
29636 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
29637 let initial = vec![
29638 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
29639 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
29640 ];
29641 let mut meta = PoolMeta {
29642 version: POOL_META_VERSION,
29643 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
29644 ..Default::default()
29645 };
29646 meta.decommission(0, initial[0].space).unwrap();
29647 reserve_decommission_start_target_capacity(
29648 &mut meta,
29649 &[0],
29650 &initial,
29651 uuid::Uuid::new_v4(),
29652 1,
29653 now,
29654 DECOMMISSION_CAPACITY_MODEL_VERSION,
29655 )
29656 .expect("the initial target capacity should fit exactly");
29657
29658 let mutation_id = uuid::Uuid::from_u128(1);
29659 reserve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id, now + Duration::seconds(1))
29660 .expect("the target intent should be durable before its write");
29661 let own_consumption = vec![initial[0], DecommissionPoolCapacityInfo::for_test(1, layout, 50, 60, 10)];
29662 ensure_decommission_capacity_reservations_available(&meta, &own_consumption, "restart")
29663 .expect("a persisted target intent must recognize its own write after restart");
29664 resolve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id)
29665 .expect("the persisted target intent should resolve at commit");
29666 record_decommission_target_consumption(
29667 &mut meta,
29668 0,
29669 1,
29670 DecommissionTargetConsumption {
29671 committed_data_bytes: 10,
29672 target_physical_bytes: 10,
29673 observed_physical_bytes: 10,
29674 },
29675 mutation_id,
29676 now + Duration::seconds(1),
29677 )
29678 .expect("the operation's own target consumption should be persisted");
29679 ensure_decommission_capacity_reservations_available(&meta, &own_consumption, "migration")
29680 .expect("the operation's own committed target bytes must not look like external capacity loss");
29681
29682 let dropped = vec![initial[0], DecommissionPoolCapacityInfo::for_test(1, layout, 39, 60, 21)];
29683 let err = ensure_decommission_capacity_reservations_available(&meta, &dropped, "migration")
29684 .expect_err("runtime migration must stop after a sudden competing write consumes reserved capacity");
29685 assert!(
29686 err.to_string()
29687 .contains("requires 40 physical bytes, but only 39 bytes remain")
29688 );
29689 assert!(is_decommission_capacity_blocked_error(&err));
29690 }
29691
29692 #[test]
29693 fn decommission_target_capacity_reuses_only_matching_pending_intent() {
29694 let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
29695 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
29696 let initial = vec![
29697 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
29698 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
29699 ];
29700 let mut meta = PoolMeta {
29701 version: POOL_META_VERSION,
29702 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
29703 ..Default::default()
29704 };
29705 meta.decommission(0, initial[0].space).unwrap();
29706 reserve_decommission_start_target_capacity(
29707 &mut meta,
29708 &[0],
29709 &initial,
29710 uuid::Uuid::new_v4(),
29711 1,
29712 now,
29713 DECOMMISSION_CAPACITY_MODEL_VERSION,
29714 )
29715 .expect("the initial target capacity should fit exactly");
29716
29717 let first_mutation_id = uuid::Uuid::from_u128(1);
29718 let second_mutation_id = uuid::Uuid::from_u128(2);
29719 reserve_decommission_target_pending(&mut meta, 0, 1, 10, first_mutation_id, now + Duration::seconds(1))
29720 .expect("the first mutation should persist its target intent");
29721 assert_eq!(
29722 reserve_decommission_target_pending(&mut meta, 0, 1, 10, first_mutation_id, now + Duration::seconds(2))
29723 .expect("the same mutation should reuse its persisted target intent"),
29724 0
29725 );
29726 let err = reserve_decommission_target_pending(&mut meta, 0, 1, 10, second_mutation_id, now + Duration::seconds(2))
29727 .expect_err("a second mutation must not reuse the first mutation's pending intent");
29728 assert!(is_decommission_capacity_blocked_error(&err));
29729 assert!(err.to_string().contains("unresolved target capacity intent"));
29730
29731 resolve_decommission_target_pending(&mut meta, 0, 1, 10, first_mutation_id)
29732 .expect("the first mutation should finalize its intent");
29733 record_decommission_target_consumption(
29734 &mut meta,
29735 0,
29736 1,
29737 DecommissionTargetConsumption {
29738 committed_data_bytes: 10,
29739 target_physical_bytes: 10,
29740 observed_physical_bytes: 10,
29741 },
29742 first_mutation_id,
29743 now + Duration::seconds(2),
29744 )
29745 .expect("the first mutation's consumption should be durable");
29746 reserve_decommission_target_pending(&mut meta, 0, 1, 10, second_mutation_id, now + Duration::seconds(3))
29747 .expect("the second mutation should retry only after the first intent is finalized");
29748 assert_eq!(
29749 meta.pools[0]
29750 .decommission
29751 .as_ref()
29752 .and_then(|info| info.capacity_reservation.as_ref())
29753 .expect("the reservation should remain present")
29754 .pending_target_physical_bytes,
29755 10
29756 );
29757 }
29758
29759 #[test]
29760 fn decommission_target_capacity_allows_same_mutation_to_grow_its_pending_stage() {
29761 let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
29762 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
29763 let initial = vec![
29764 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
29765 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
29766 ];
29767 let mut meta = PoolMeta {
29768 version: POOL_META_VERSION,
29769 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
29770 ..Default::default()
29771 };
29772 meta.decommission(0, initial[0].space).unwrap();
29773 reserve_decommission_start_target_capacity(
29774 &mut meta,
29775 &[0],
29776 &initial,
29777 uuid::Uuid::new_v4(),
29778 1,
29779 now,
29780 DECOMMISSION_CAPACITY_MODEL_VERSION,
29781 )
29782 .expect("the initial target capacity should fit exactly");
29783
29784 let mutation_id = uuid::Uuid::from_u128(1);
29785 assert_eq!(
29786 reserve_decommission_target_pending(&mut meta, 0, 1, 1, mutation_id, now + Duration::seconds(1))
29787 .expect("the initial multipart stage should persist its one-byte intent"),
29788 1
29789 );
29790 assert_eq!(
29791 reserve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id, now + Duration::seconds(2))
29792 .expect("the same mutation should grow its intent for the commit stage"),
29793 9
29794 );
29795 assert_eq!(
29796 meta.pools[0]
29797 .decommission
29798 .as_ref()
29799 .and_then(|info| info.capacity_reservation.as_ref())
29800 .expect("the reservation should remain present")
29801 .targets[0]
29802 .pending_physical_bytes,
29803 10
29804 );
29805 assert_eq!(
29806 reserve_decommission_target_pending(&mut meta, 0, 1, 1, mutation_id, now + Duration::seconds(3))
29807 .expect("a retry of an earlier stage must reuse the larger same-mutation intent"),
29808 0
29809 );
29810 }
29811
29812 #[test]
29813 fn decommission_capacity_mutation_identity_survives_lease_nonce_rotation() {
29814 let owner = DecommissionCapacityOwner {
29815 source_pool_index: 0,
29816 operation_id: uuid::Uuid::from_u128(1),
29817 generation: 2,
29818 owner_nonce: uuid::Uuid::from_u128(3),
29819 mutation_id: None,
29820 };
29821 let recovered_owner = DecommissionCapacityOwner {
29822 owner_nonce: uuid::Uuid::from_u128(4),
29823 ..owner
29824 };
29825 let first = decommission_capacity_mutation_id(owner, "bucket", "object", Some("version"), false, None);
29826 let recovered = decommission_capacity_mutation_id(recovered_owner, "bucket", "object", Some("version"), false, None);
29827 assert_eq!(first, recovered, "a lease nonce rotation must not change the mutation identity");
29828 }
29829
29830 #[test]
29831 fn exact_delete_capacity_plan_requires_identity_and_exact_size() {
29832 let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
29833 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
29834 let capacity_infos = vec![
29835 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
29836 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
29837 ];
29838 let mut meta = PoolMeta {
29839 version: POOL_META_VERSION,
29840 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
29841 ..Default::default()
29842 };
29843 meta.decommission(0, capacity_infos[0].space).unwrap();
29844 reserve_decommission_start_target_capacity(
29845 &mut meta,
29846 &[0],
29847 &capacity_infos,
29848 uuid::Uuid::new_v4(),
29849 1,
29850 now,
29851 DECOMMISSION_CAPACITY_MODEL_VERSION,
29852 )
29853 .expect("the exact-delete test reservation should fit");
29854 let exact = ObjectInfo {
29855 bucket: "bucket".to_string(),
29856 name: "object".to_string(),
29857 version_id: Some(uuid::Uuid::from_u128(7)),
29858 mod_time: Some(now),
29859 size: 10,
29860 ..Default::default()
29861 };
29862 let owner = {
29863 let reservation = meta.pools[0]
29864 .decommission
29865 .as_ref()
29866 .and_then(|info| info.capacity_reservation.as_ref())
29867 .expect("the exact-delete test reservation should exist");
29868 DecommissionCapacityOwner {
29869 source_pool_index: 0,
29870 operation_id: reservation.operation_id,
29871 generation: reservation.generation,
29872 owner_nonce: reservation.owner_nonce,
29873 mutation_id: None,
29874 }
29875 };
29876 let version_id = exact.version_id.map(|version_id| version_id.to_string());
29877 let mutation_id = decommission_capacity_mutation_id(
29878 owner,
29879 &exact.bucket,
29880 &exact.name,
29881 version_id.as_deref(),
29882 exact.delete_marker,
29883 exact.mod_time,
29884 );
29885 reserve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id, now + Duration::seconds(1))
29886 .expect("the exact-delete test intent should be reserved");
29887
29888 let plan = plan_exact_delete_capacity_reconciliations(&meta, &exact.name, &exact)
29889 .expect("the exact identity should match the pending intent");
29890 assert_eq!(plan.len(), 1);
29891 assert_eq!(plan[0].source_pool_index, 0);
29892 assert_eq!(plan[0].target_pool_index, 1);
29893 assert_eq!(plan[0].expected_data_bytes, 10);
29894 assert_eq!(plan[0].expected_target_physical_bytes, 10);
29895
29896 let mismatched_size = ObjectInfo {
29897 size: 9,
29898 ..exact.clone()
29899 };
29900 let mismatched_size = plan_exact_delete_capacity_reconciliations(&meta, &mismatched_size.name, &mismatched_size)
29901 .expect_err("a different exact size must not consume the pending intent");
29902 assert!(mismatched_size.to_string().contains("does not match the exact object size"));
29903
29904 let directory_exact = ObjectInfo {
29905 name: "directory/".to_string(),
29906 ..exact.clone()
29907 };
29908 let internal_directory = rustfs_utils::path::encode_dir_object(&directory_exact.name);
29909 let internal_directory_mutation_id = decommission_capacity_mutation_id(
29910 owner,
29911 &directory_exact.bucket,
29912 &internal_directory,
29913 version_id.as_deref(),
29914 directory_exact.delete_marker,
29915 directory_exact.mod_time,
29916 );
29917 meta.pools[0]
29918 .decommission
29919 .as_mut()
29920 .and_then(|info| info.capacity_reservation.as_mut())
29921 .expect("the exact-delete test reservation should exist")
29922 .targets[0]
29923 .pending_mutation_id = Some(internal_directory_mutation_id);
29924 let directory_plan = plan_exact_delete_capacity_reconciliations(&meta, &internal_directory, &directory_exact)
29925 .expect("an internally encoded directory intent should match its logical exact object");
29926 assert_eq!(directory_plan[0].mutation_id, internal_directory_mutation_id);
29927
29928 let logical_directory_mutation_id = decommission_capacity_mutation_id(
29929 owner,
29930 &directory_exact.bucket,
29931 &directory_exact.name,
29932 version_id.as_deref(),
29933 directory_exact.delete_marker,
29934 directory_exact.mod_time,
29935 );
29936 meta.pools[0]
29937 .decommission
29938 .as_mut()
29939 .and_then(|info| info.capacity_reservation.as_mut())
29940 .expect("the exact-delete test reservation should exist")
29941 .targets[0]
29942 .pending_mutation_id = Some(logical_directory_mutation_id);
29943 let directory_plan = plan_exact_delete_capacity_reconciliations(&meta, &internal_directory, &directory_exact)
29944 .expect("a logical directory intent should match its internally encoded delete path");
29945 assert_eq!(directory_plan[0].mutation_id, logical_directory_mutation_id);
29946
29947 meta.pools[0]
29948 .decommission
29949 .as_mut()
29950 .and_then(|info| info.capacity_reservation.as_mut())
29951 .expect("the exact-delete test reservation should exist")
29952 .targets[0]
29953 .pending_mutation_id = None;
29954 let unidentified = plan_exact_delete_capacity_reconciliations(&meta, &exact.name, &exact)
29955 .expect_err("an unidentified pending intent must fail closed");
29956 assert!(unidentified.to_string().contains("without an object identity"));
29957
29958 let mut opts = ObjectOptions::default();
29959 let unfenced = ensure_exact_delete_capacity_namespace_fences(&opts, &exact.bucket, &exact.name)
29960 .expect_err("capacity reconciliation must reject a missing object namespace fence");
29961 assert!(unfenced.to_string().contains("requires an object namespace fence"));
29962 opts.ensure_namespace_lock_fence();
29963 ensure_exact_delete_capacity_namespace_fences(&opts, &exact.bucket, &exact.name)
29964 .expect("a live object namespace fence should admit capacity reconciliation");
29965 }
29966
29967 #[test]
29968 fn ordinary_write_admission_shares_a_reserved_target_without_becoming_its_owner() {
29969 let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
29970 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
29971 let capacity_infos = vec![
29972 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
29973 DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
29974 ];
29975 let mut meta = PoolMeta {
29976 version: POOL_META_VERSION,
29977 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
29978 ..Default::default()
29979 };
29980 meta.decommission(0, capacity_infos[0].space).unwrap();
29981 reserve_decommission_start_target_capacity(
29982 &mut meta,
29983 &[0],
29984 &capacity_infos,
29985 uuid::Uuid::new_v4(),
29986 1,
29987 now,
29988 DECOMMISSION_CAPACITY_MODEL_VERSION,
29989 )
29990 .expect("the decommission reservation should fit");
29991
29992 for admission in [
29993 DecommissionCapacityAdmission::Mutation,
29994 DecommissionCapacityAdmission::BatchDelete,
29995 DecommissionCapacityAdmission::ScannerBacklog,
29996 ] {
29997 ensure_external_decommission_target_admission(&meta, 1, admission)
29998 .expect("a healthy target must remain writable while sharing capacity with migration");
29999 }
30000 assert!(matches!(
30001 ensure_external_decommission_target_admission(&meta, 1, DecommissionCapacityAdmission::Heal),
30002 Err(Error::SlowDown)
30003 ));
30004 let rebalance_opts = ObjectOptions {
30005 data_movement: true,
30006 src_pool_idx: 0,
30007 ..Default::default()
30008 };
30009 assert!(
30010 DecommissionCapacityOwner::from_options(&rebalance_opts).is_none(),
30011 "rebalance data movement must remain an external capacity consumer"
30012 );
30013 let reservation = meta.pools[0]
30014 .decommission
30015 .as_ref()
30016 .and_then(|info| info.capacity_reservation.as_ref())
30017 .expect("the active reservation should remain available");
30018 let expected_owner = DecommissionCapacityOwner {
30019 source_pool_index: 0,
30020 operation_id: reservation.operation_id,
30021 generation: reservation.generation,
30022 owner_nonce: reservation.owner_nonce,
30023 mutation_id: None,
30024 };
30025 ensure_decommission_target_owner_admission(&meta, expected_owner, 1, 10, now)
30026 .expect("the reservation owner should consume its own allocation under the shared boundary");
30027 let mut decommission_opts = rebalance_opts;
30028 expected_owner.apply_to(&mut decommission_opts);
30029 assert_eq!(DecommissionCapacityOwner::from_options(&decommission_opts), Some(expected_owner));
30030
30031 meta.pools[0]
30032 .decommission
30033 .as_mut()
30034 .expect("active source")
30035 .capacity_reservation = None;
30036 for admission in [
30037 DecommissionCapacityAdmission::Mutation,
30038 DecommissionCapacityAdmission::BatchDelete,
30039 DecommissionCapacityAdmission::ScannerBacklog,
30040 ] {
30041 assert!(
30042 matches!(ensure_external_decommission_target_admission(&meta, 1, admission), Err(Error::SlowDown)),
30043 "shared capacity must not bypass an active source's missing durable ledger"
30044 );
30045 }
30046 }
30047
30048 #[test]
30049 fn external_decommission_admission_fences_suspended_sources_but_preserves_repair() {
30050 let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
30051 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
30052 let capacity_infos = vec![
30053 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
30054 DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
30055 ];
30056 let mut active = PoolMeta {
30057 version: POOL_META_VERSION,
30058 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
30059 ..Default::default()
30060 };
30061 active
30062 .decommission(0, capacity_infos[0].space)
30063 .expect("start the source admission fixture");
30064 reserve_decommission_start_target_capacity(
30065 &mut active,
30066 &[0],
30067 &capacity_infos,
30068 uuid::Uuid::new_v4(),
30069 1,
30070 now,
30071 DECOMMISSION_CAPACITY_MODEL_VERSION,
30072 )
30073 .expect("the active source must have a valid reservation to isolate its write fence");
30074
30075 for (state, queued, failed, canceled, complete) in [
30076 ("running", false, false, false, false),
30077 ("queued", true, false, false, false),
30078 ("failed", false, true, false, false),
30079 ("canceled", false, false, true, false),
30080 ("completed", false, false, false, true),
30081 ] {
30082 let mut meta = active.clone();
30083 let info = meta.pools[0].decommission.as_mut().expect("the source fixture must exist");
30084 info.queued = queued;
30085 info.failed = failed;
30086 info.canceled = canceled;
30087 info.complete = complete;
30088 if queued || failed || canceled || complete {
30089 info.start_time = None;
30090 }
30091 for admission in [
30092 DecommissionCapacityAdmission::Mutation,
30093 DecommissionCapacityAdmission::BatchDelete,
30094 ] {
30095 assert!(
30096 matches!(ensure_external_decommission_target_admission(&meta, 0, admission), Err(Error::SlowDown)),
30097 "{state} source must reject new publication until its decommission metadata is cleared"
30098 );
30099 }
30100 let existing_multipart =
30101 ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::ExistingMultipart);
30102 if active_decommission_source_indices(&meta).contains(&0) {
30103 existing_multipart
30104 .unwrap_or_else(|err| panic!("{state} source must allow an existing multipart upload to drain: {err}"));
30105 } else {
30106 assert!(
30107 matches!(existing_multipart, Err(Error::SlowDown)),
30108 "{state} terminal source must reject an existing multipart publication"
30109 );
30110 }
30111 ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::Heal)
30112 .unwrap_or_else(|err| panic!("{state} source repair must retain its capacity-only admission: {err}"));
30113 let scanner_result =
30114 ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::ScannerBacklog);
30115 assert_eq!(
30116 meta.scanner_pause_backlog_pool_writable(0),
30117 failed || canceled,
30118 "{state} scanner selection"
30119 );
30120 if failed || canceled {
30121 scanner_result.unwrap_or_else(|err| panic!("{state} scanner membership repair must remain writable: {err}"));
30122 } else {
30123 assert!(
30124 matches!(scanner_result, Err(Error::SlowDown)),
30125 "{state} scanner publication must reject its source"
30126 );
30127 }
30128 meta.pools[0].decommission = None;
30129 ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::Mutation)
30130 .unwrap_or_else(|err| panic!("cleared {state} source must become writable again: {err}"));
30131 ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::ScannerBacklog)
30132 .unwrap_or_else(|err| panic!("cleared {state} scanner source must rejoin membership: {err}"));
30133 }
30134 assert!(!active.scanner_pause_backlog_pool_writable(active.pools.len()));
30135 assert!(matches!(
30136 ensure_external_decommission_target_admission(
30137 &active,
30138 active.pools.len(),
30139 DecommissionCapacityAdmission::ScannerBacklog
30140 ),
30141 Err(Error::SlowDown)
30142 ));
30143 }
30144
30145 #[test]
30146 fn decommission_physical_capacity_uses_per_set_bottleneck_and_widest_usage() {
30147 let disks = [10_u64, 20, 30, 40]
30148 .into_iter()
30149 .enumerate()
30150 .map(|(disk_index, available_space)| rustfs_madmin::Disk {
30151 endpoint: format!("http://node-{disk_index}"),
30152 drive_path: format!("/disk-{disk_index}"),
30153 state: "ok".to_string(),
30154 total_space: 100,
30155 used_space: 100 - available_space,
30156 available_space,
30157 pool_index: 0,
30158 set_index: 0,
30159 disk_index: disk_index as i32,
30160 ..Default::default()
30161 })
30162 .collect::<Vec<_>>();
30163 let capacity = decommission_physical_pool_capacity(
30164 &disks,
30165 0,
30166 DecommissionErasureLayout { data: 2, parity: 2 },
30167 PoolSpaceInfo {
30168 free: 0,
30169 total: 0,
30170 used: 0,
30171 },
30172 None,
30173 );
30174
30175 assert_eq!(capacity, (400, 40, 360));
30176 }
30177
30178 #[test]
30179 fn decommission_physical_capacity_prefers_rustfs_owned_usage_over_filesystem_usage() {
30180 let disks = [10_u64, 20, 30, 40]
30181 .into_iter()
30182 .enumerate()
30183 .map(|(disk_index, available_space)| rustfs_madmin::Disk {
30184 endpoint: format!("http://node-{disk_index}"),
30185 drive_path: format!("/disk-{disk_index}"),
30186 state: "ok".to_string(),
30187 total_space: 100,
30188 used_space: 100 - available_space,
30189 available_space,
30190 pool_index: 0,
30191 set_index: 0,
30192 disk_index: disk_index as i32,
30193 ..Default::default()
30194 })
30195 .collect::<Vec<_>>();
30196 let owned = HashMap::from([(0, 12)]);
30197 let capacity = decommission_physical_pool_capacity(
30198 &disks,
30199 0,
30200 DecommissionErasureLayout { data: 2, parity: 2 },
30201 PoolSpaceInfo {
30202 free: 0,
30203 total: 0,
30204 used: 0,
30205 },
30206 Some(&owned),
30207 );
30208
30209 assert_eq!(capacity, (400, 40, 12));
30210 }
30211
30212 #[test]
30213 fn decommission_terminal_transitions_release_capacity_reservations() {
30214 let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(1);
30215 let reservation = build_decommission_capacity_reservation(
30216 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 1, parity: 0 }, 0, 50, 50),
30217 DecommissionErasureLayout { data: 1, parity: 0 },
30218 uuid::Uuid::new_v4(),
30219 7,
30220 now,
30221 )
30222 .expect("test reservation should be valid");
30223 let active = PoolMeta {
30224 version: POOL_META_VERSION,
30225 pools: vec![decommission_test_pool_status(
30226 0,
30227 Some(PoolDecommissionInfo {
30228 start_time: Some(now),
30229 capacity_reservation: Some(reservation),
30230 ..Default::default()
30231 }),
30232 )],
30233 ..Default::default()
30234 };
30235
30236 for (reason, transition) in [
30237 (
30238 DECOMMISSION_CAPACITY_RELEASE_CANCELED,
30239 PoolMeta::decommission_cancel as fn(&mut PoolMeta, usize) -> bool,
30240 ),
30241 (DECOMMISSION_CAPACITY_RELEASE_FAILED, PoolMeta::decommission_failed),
30242 (DECOMMISSION_CAPACITY_RELEASE_COMPLETED, PoolMeta::decommission_complete),
30243 ] {
30244 let mut meta = active.clone();
30245 assert!(transition(&mut meta, 0));
30246 let released = meta.pools[0]
30247 .decommission
30248 .as_ref()
30249 .and_then(|info| info.capacity_reservation.as_ref())
30250 .expect("terminal metadata should retain reservation observability");
30251 assert!(released.released_at.is_some());
30252 assert_eq!(released.release_reason.as_deref(), Some(reason));
30253 assert_eq!(
30254 released.operation_id,
30255 active.pools[0]
30256 .decommission
30257 .as_ref()
30258 .unwrap()
30259 .capacity_reservation
30260 .as_ref()
30261 .unwrap()
30262 .operation_id
30263 );
30264 }
30265 }
30266
30267 #[test]
30268 fn decommission_capacity_target_round_trip_preserves_temporary_mutation_without_pending_intent() {
30269 let mutation_id = uuid::Uuid::new_v4();
30270 let target = DecommissionCapacityTarget {
30271 pool_index: 1,
30272 layout: DecommissionErasureLayout { data: 2, parity: 2 },
30273 physical_total_at_reservation: 200,
30274 physical_free_at_reservation: 200,
30275 reserved_physical_bytes: 200,
30276 consumed_physical_bytes: 0,
30277 observed_physical_bytes: 1,
30278 inflight_physical_bytes: 1,
30279 pending_physical_bytes: 0,
30280 pending_mutation_id: None,
30281 temporary_mutations: vec![DecommissionCapacityTemporaryMutation {
30282 mutation_id,
30283 physical_bytes: 1,
30284 }],
30285 };
30286 let mut encoded = Vec::new();
30287 target
30288 .serialize(&mut Serializer::new(&mut encoded))
30289 .expect("capacity target should serialize");
30290
30291 let restored: DecommissionCapacityTarget =
30292 rmp_serde::from_slice(&encoded).expect("capacity target with a released pending intent should deserialize");
30293
30294 assert_eq!(restored, target);
30295 }
30296
30297 #[test]
30298 fn temporary_cleanup_releases_only_its_exact_scoped_mutation_and_is_idempotent() {
30299 let mutation_id = uuid::Uuid::new_v4();
30300 let foreign_id = uuid::Uuid::new_v4();
30301 let mut meta = decommission_test_cleanup_meta(
30302 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
30303 vec![
30304 DecommissionCapacityTemporaryMutation {
30305 mutation_id,
30306 physical_bytes: 10,
30307 },
30308 DecommissionCapacityTemporaryMutation {
30309 mutation_id: foreign_id,
30310 physical_bytes: 20,
30311 },
30312 ],
30313 30,
30314 None,
30315 );
30316
30317 assert!(
30318 release_decommission_target_inflight(
30319 &mut meta,
30320 0,
30321 1,
30322 0,
30323 mutation_id,
30324 DecommissionCapacityReleaseProof::confirmed_absence(true),
30325 OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
30326 )
30327 .expect("confirmed absence should release the exact mutation")
30328 );
30329 let reservation = meta.pools[0]
30330 .decommission
30331 .as_ref()
30332 .and_then(|info| info.capacity_reservation.as_ref())
30333 .expect("cleanup reservation should remain present");
30334 assert_eq!(reservation.inflight_target_physical_bytes, 20);
30335 assert_eq!(
30336 reservation.targets[0].temporary_mutations,
30337 vec![DecommissionCapacityTemporaryMutation {
30338 mutation_id: foreign_id,
30339 physical_bytes: 20,
30340 }]
30341 );
30342
30343 let before_replay = reservation.clone();
30344 assert!(
30345 !release_decommission_target_inflight(
30346 &mut meta,
30347 0,
30348 1,
30349 0,
30350 mutation_id,
30351 DecommissionCapacityReleaseProof::confirmed_absence(true),
30352 OffsetDateTime::UNIX_EPOCH + Duration::seconds(2),
30353 )
30354 .expect("repeated confirmed absence should be a metadata no-op")
30355 );
30356 let after_replay = meta.pools[0]
30357 .decommission
30358 .as_ref()
30359 .and_then(|info| info.capacity_reservation.as_ref())
30360 .expect("cleanup reservation should remain present");
30361 assert_eq!(after_replay, &before_replay);
30362 }
30363
30364 #[test]
30365 fn temporary_cleanup_uses_aggregate_fallback_only_for_legacy_unscoped_state() {
30366 let mutation_id = uuid::Uuid::new_v4();
30367 let mut meta = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, Vec::new(), 30, None);
30368
30369 assert!(
30370 release_decommission_target_inflight(
30371 &mut meta,
30372 0,
30373 1,
30374 12,
30375 mutation_id,
30376 DecommissionCapacityReleaseProof::confirmed_absence(true),
30377 OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
30378 )
30379 .expect("legacy unscoped cleanup should use its observed release delta")
30380 );
30381 let reservation = meta.pools[0]
30382 .decommission
30383 .as_ref()
30384 .and_then(|info| info.capacity_reservation.as_ref())
30385 .expect("legacy cleanup reservation should remain present");
30386 assert_eq!(reservation.inflight_target_physical_bytes, 18);
30387 assert_eq!(reservation.targets[0].inflight_physical_bytes, 18);
30388 }
30389
30390 #[test]
30391 fn temporary_cleanup_clears_only_a_matching_pending_identity_after_confirmed_absence() {
30392 let mutation_id = uuid::Uuid::new_v4();
30393 let mut meta = decommission_test_cleanup_meta(
30394 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
30395 Vec::new(),
30396 0,
30397 Some((mutation_id, 15)),
30398 );
30399
30400 assert!(
30401 release_decommission_target_inflight(
30402 &mut meta,
30403 0,
30404 1,
30405 7,
30406 mutation_id,
30407 DecommissionCapacityReleaseProof::confirmed_absence(true),
30408 OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
30409 )
30410 .expect("a confirmed absent upload should clear its matching pending identity")
30411 );
30412 let reservation = meta.pools[0]
30413 .decommission
30414 .as_ref()
30415 .and_then(|info| info.capacity_reservation.as_ref())
30416 .expect("cleanup reservation should remain present");
30417 assert_eq!(reservation.pending_target_physical_bytes, 0);
30418 assert_eq!(reservation.targets[0].pending_mutation_id, None);
30419 assert_eq!(reservation.targets[0].pending_physical_bytes, 0);
30420 }
30421
30422 #[test]
30423 fn temporary_cleanup_preserves_pending_for_an_already_published_target() {
30424 let mutation_id = uuid::Uuid::new_v4();
30425 let mut meta = decommission_test_cleanup_meta(
30426 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
30427 vec![DecommissionCapacityTemporaryMutation {
30428 mutation_id,
30429 physical_bytes: 5,
30430 }],
30431 5,
30432 Some((mutation_id, 15)),
30433 );
30434
30435 assert!(
30436 release_decommission_target_inflight(
30437 &mut meta,
30438 0,
30439 1,
30440 0,
30441 mutation_id,
30442 DecommissionCapacityReleaseProof::confirmed_absence(false),
30443 OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
30444 )
30445 .expect("published-target cleanup should release only its temporary staging state")
30446 );
30447 let reservation = meta.pools[0]
30448 .decommission
30449 .as_ref()
30450 .and_then(|info| info.capacity_reservation.as_ref())
30451 .expect("cleanup reservation should remain present");
30452 assert_eq!(reservation.inflight_target_physical_bytes, 0);
30453 assert_eq!(reservation.pending_target_physical_bytes, 15);
30454 assert_eq!(reservation.targets[0].pending_mutation_id, Some(mutation_id));
30455 assert_eq!(reservation.targets[0].pending_physical_bytes, 15);
30456 assert!(reservation.targets[0].temporary_mutations.is_empty());
30457 }
30458
30459 #[test]
30460 fn published_pending_ignores_a_delayed_release_observation() {
30461 let mutation_id = uuid::Uuid::new_v4();
30462 let mut meta = decommission_test_cleanup_meta(
30463 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
30464 Vec::new(),
30465 0,
30466 Some((mutation_id, 15)),
30467 );
30468 let before = meta.pools[0]
30469 .decommission
30470 .as_ref()
30471 .and_then(|info| info.capacity_reservation.as_ref())
30472 .expect("published-target reservation should remain present")
30473 .clone();
30474
30475 assert!(
30476 !release_decommission_target_inflight(
30477 &mut meta,
30478 0,
30479 1,
30480 7,
30481 mutation_id,
30482 DecommissionCapacityReleaseProof::confirmed_absence(false),
30483 OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
30484 )
30485 .expect("a delayed statfs release must not block published-target reconciliation")
30486 );
30487 let after = meta.pools[0]
30488 .decommission
30489 .as_ref()
30490 .and_then(|info| info.capacity_reservation.as_ref())
30491 .expect("published-target reservation should remain present");
30492 assert_eq!(after, &before);
30493 }
30494
30495 #[test]
30496 fn temporary_cleanup_fails_closed_on_a_foreign_scoped_release_delta() {
30497 let foreign_id = uuid::Uuid::new_v4();
30498 let mut meta = decommission_test_cleanup_meta(
30499 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
30500 vec![DecommissionCapacityTemporaryMutation {
30501 mutation_id: foreign_id,
30502 physical_bytes: 20,
30503 }],
30504 20,
30505 None,
30506 );
30507 let before = meta.pools[0]
30508 .decommission
30509 .as_ref()
30510 .and_then(|info| info.capacity_reservation.as_ref())
30511 .expect("cleanup reservation should remain present")
30512 .clone();
30513
30514 let err = release_decommission_target_inflight(
30515 &mut meta,
30516 0,
30517 1,
30518 5,
30519 uuid::Uuid::new_v4(),
30520 DecommissionCapacityReleaseProof::confirmed_absence(true),
30521 OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
30522 )
30523 .expect_err("an observed release cannot be charged to a foreign scoped mutation");
30524 assert!(err.to_string().contains("cannot be attributed"));
30525 let after = meta.pools[0]
30526 .decommission
30527 .as_ref()
30528 .and_then(|info| info.capacity_reservation.as_ref())
30529 .expect("cleanup reservation should remain present");
30530 assert_eq!(after, &before);
30531 }
30532
30533 #[test]
30534 fn temporary_cleanup_missing_identity_and_nonzero_delta_is_a_metadata_noop() {
30535 let mut meta = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, Vec::new(), 0, None);
30536 let before = meta.pools[0]
30537 .decommission
30538 .as_ref()
30539 .and_then(|info| info.capacity_reservation.as_ref())
30540 .expect("cleanup reservation should remain present")
30541 .clone();
30542
30543 assert!(
30544 !release_decommission_target_inflight(
30545 &mut meta,
30546 0,
30547 1,
30548 7,
30549 uuid::Uuid::new_v4(),
30550 DecommissionCapacityReleaseProof::confirmed_absence(true),
30551 OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
30552 )
30553 .expect("confirmed absence without tracked state should ignore a delayed capacity observation")
30554 );
30555 let after = meta.pools[0]
30556 .decommission
30557 .as_ref()
30558 .and_then(|info| info.capacity_reservation.as_ref())
30559 .expect("cleanup reservation should remain present");
30560 assert_eq!(after, &before);
30561 }
30562
30563 #[test]
30564 fn temporary_cleanup_missing_identity_fails_closed_on_foreign_pending_state() {
30565 let foreign_id = uuid::Uuid::new_v4();
30566 let mut meta = decommission_test_cleanup_meta(
30567 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
30568 Vec::new(),
30569 0,
30570 Some((foreign_id, 11)),
30571 );
30572 let before = meta.pools[0]
30573 .decommission
30574 .as_ref()
30575 .and_then(|info| info.capacity_reservation.as_ref())
30576 .expect("cleanup reservation should remain present")
30577 .clone();
30578
30579 let err = release_decommission_target_inflight(
30580 &mut meta,
30581 0,
30582 1,
30583 7,
30584 uuid::Uuid::new_v4(),
30585 DecommissionCapacityReleaseProof::confirmed_absence(true),
30586 OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
30587 )
30588 .expect_err("a delayed release observation must not clear a foreign pending mutation");
30589 assert!(err.to_string().contains("cannot be attributed"));
30590 let after = meta.pools[0]
30591 .decommission
30592 .as_ref()
30593 .and_then(|info| info.capacity_reservation.as_ref())
30594 .expect("cleanup reservation should remain present");
30595 assert_eq!(after, &before);
30596 }
30597
30598 #[test]
30599 fn pool_meta_v2_round_trip_preserves_each_supported_capacity_lock_model() {
30600 for model_version in [
30601 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION,
30602 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
30603 ] {
30604 let meta = decommission_test_cleanup_meta(model_version, Vec::new(), 0, None);
30605 let encoded = meta
30606 .encode_config_data_for_test()
30607 .expect("supported capacity lock model should encode in pool metadata V2");
30608 let mut restored = PoolMeta::default();
30609 restored
30610 .load_from_config_data(encoded)
30611 .expect("supported capacity lock model should decode from pool metadata V2");
30612 assert_eq!(
30613 restored.pools[0]
30614 .decommission
30615 .as_ref()
30616 .and_then(|info| info.capacity_reservation.as_ref())
30617 .expect("round-tripped reservation should remain present")
30618 .model_version,
30619 model_version
30620 );
30621 }
30622 }
30623
30624 #[test]
30625 fn pool_meta_v2_decode_rejects_mixed_active_capacity_lock_models() {
30626 let mut mixed = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, Vec::new(), 0, None);
30627 let mut target_fence_source =
30628 decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, Vec::new(), 0, None)
30629 .pools
30630 .remove(0);
30631 target_fence_source.id = 2;
30632 target_fence_source.cmd_line = "pool-2".to_string();
30633 target_fence_source
30634 .decommission
30635 .as_mut()
30636 .and_then(|info| info.capacity_reservation.as_mut())
30637 .expect("second active reservation should remain present")
30638 .source_pool_index = 2;
30639 mixed.pools.push(target_fence_source);
30640
30641 let encoded = mixed
30642 .encode_config_data_for_test()
30643 .expect("the decode test must be able to construct a mixed persisted payload");
30644 let mut restored = PoolMeta::default();
30645 let err = restored
30646 .load_from_config_data(encoded)
30647 .expect_err("mixed active lock models must fail closed while loading pool metadata");
30648 assert!(err.to_string().contains("mixed lock models"));
30649 }
30650
30651 #[test]
30652 fn decommission_capacity_reservation_recovers_expired_lease_after_restart_round_trip() {
30653 let created_at = OffsetDateTime::UNIX_EPOCH + Duration::hours(1);
30654 let recovered_at = created_at + DECOMMISSION_CAPACITY_RESERVATION_TTL + Duration::seconds(1);
30655 let mut reservation = build_decommission_capacity_reservation(
30656 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 2, parity: 2 }, 0, 100, 100),
30657 DecommissionErasureLayout { data: 2, parity: 2 },
30658 uuid::Uuid::new_v4(),
30659 41,
30660 created_at,
30661 )
30662 .expect("test reservation should be valid");
30663 reservation.targets.push(DecommissionCapacityTarget {
30664 pool_index: 1,
30665 layout: DecommissionErasureLayout { data: 2, parity: 2 },
30666 physical_total_at_reservation: 200,
30667 physical_free_at_reservation: 200,
30668 reserved_physical_bytes: 200,
30669 consumed_physical_bytes: 0,
30670 observed_physical_bytes: 0,
30671 inflight_physical_bytes: 0,
30672 pending_physical_bytes: 0,
30673 pending_mutation_id: None,
30674 temporary_mutations: Vec::new(),
30675 });
30676 let operation_id = reservation.operation_id;
30677 let owner_nonce = reservation.owner_nonce;
30678 let mut meta = PoolMeta {
30679 version: POOL_META_VERSION,
30680 pools: vec![decommission_test_pool_status(
30681 0,
30682 Some(PoolDecommissionInfo {
30683 start_time: Some(created_at),
30684 capacity_reservation: Some(reservation),
30685 ..Default::default()
30686 }),
30687 )],
30688 ..Default::default()
30689 };
30690
30691 let encoded = meta
30692 .encode_config_data_for_test()
30693 .expect("capacity reservation should persist in V2");
30694 let mut restored = PoolMeta::default();
30695 restored
30696 .load_from_config_data(encoded)
30697 .expect("capacity reservation should survive a restart round trip");
30698 let info = restored.pools[0]
30699 .decommission
30700 .as_mut()
30701 .expect("active decommission should restore");
30702 let recovered = info.capacity_reservation.as_mut().expect("reservation should remain present");
30703 assert!(renew_decommission_capacity_reservation(recovered, recovered_at, true));
30704 assert_eq!(recovered.operation_id, operation_id);
30705 assert_eq!(recovered.generation, 41);
30706 assert_ne!(recovered.owner_nonce, owner_nonce);
30707 assert_eq!(recovered.recovered_at, Some(recovered_at));
30708 assert_eq!(recovered.expires_at, recovered_at + DECOMMISSION_CAPACITY_RESERVATION_TTL);
30709 assert_eq!(next_decommission_capacity_generation(&restored).unwrap(), 42);
30710
30711 meta.decommission_failed(0);
30712 let terminal = meta.pools[0]
30713 .decommission
30714 .as_ref()
30715 .and_then(|info| info.capacity_reservation.as_ref())
30716 .expect("failed metadata should retain the released reservation");
30717 assert!(!terminal.active());
30718 }
30719
30720 #[test]
30721 fn decommission_restart_reconstructs_a_missing_capacity_reservation_fail_closed() {
30722 let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(2);
30723 let mut meta = PoolMeta {
30724 version: POOL_META_VERSION,
30725 pools: vec![
30726 decommission_test_pool_status(
30727 0,
30728 Some(PoolDecommissionInfo {
30729 start_time: Some(now - Duration::minutes(1)),
30730 ..Default::default()
30731 }),
30732 ),
30733 decommission_test_pool_status(1, None),
30734 ],
30735 ..Default::default()
30736 };
30737 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
30738 let capacity_infos = vec![
30739 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
30740 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
30741 ];
30742
30743 let recovered_indices = recover_decommission_capacity_reservations(&mut meta, &capacity_infos, now, true)
30744 .expect("restart recovery should rebuild the missing reservation while capacity still fits");
30745 assert_eq!(recovered_indices, vec![0]);
30746 let reservation = meta.pools[0]
30747 .decommission
30748 .as_ref()
30749 .and_then(|info| info.capacity_reservation.as_ref())
30750 .expect("restart recovery should persist a replacement reservation");
30751 assert_eq!(reservation.peak_physical_bytes, 60);
30752 assert_eq!(reservation.recovered_at, Some(now));
30753 assert_eq!(reservation.generation, 1);
30754 }
30755
30756 #[test]
30757 fn test_ensure_decommission_start_pool_states_rejects_blocked_pool() {
30758 let meta = PoolMeta {
30759 pools: vec![
30760 PoolStatus {
30761 id: 0,
30762 cmd_line: "pool-0".to_string(),
30763 last_update: OffsetDateTime::UNIX_EPOCH,
30764 decommission: Some(PoolDecommissionInfo {
30765 failed: true,
30766 ..Default::default()
30767 }),
30768 },
30769 PoolStatus {
30770 id: 1,
30771 cmd_line: "pool-1".to_string(),
30772 last_update: OffsetDateTime::UNIX_EPOCH,
30773 decommission: None,
30774 },
30775 ],
30776 ..Default::default()
30777 };
30778
30779 let err = ensure_decommission_start_pool_states(&meta, &[0]).expect_err("blocked pool should be rejected");
30780
30781 assert!(err.to_string().contains("target pool decommission is blocked"));
30782 }
30783
30784 #[test]
30785 fn test_ensure_decommission_start_pool_states_allows_active_pool_with_remaining_active_pool() {
30786 let meta = PoolMeta {
30787 pools: vec![
30788 PoolStatus {
30789 id: 0,
30790 cmd_line: "pool-0".to_string(),
30791 last_update: OffsetDateTime::UNIX_EPOCH,
30792 decommission: None,
30793 },
30794 PoolStatus {
30795 id: 1,
30796 cmd_line: "pool-1".to_string(),
30797 last_update: OffsetDateTime::UNIX_EPOCH,
30798 decommission: None,
30799 },
30800 ],
30801 ..Default::default()
30802 };
30803
30804 assert!(ensure_decommission_start_pool_states(&meta, &[0]).is_ok());
30805 }
30806
30807 #[test]
30808 fn test_ensure_decommission_start_pool_states_allows_retryable_pool_with_active_peer() {
30809 let meta = PoolMeta {
30810 pools: vec![
30811 PoolStatus {
30812 id: 0,
30813 cmd_line: "pool-0".to_string(),
30814 last_update: OffsetDateTime::UNIX_EPOCH,
30815 decommission: Some(PoolDecommissionInfo {
30816 failed: true,
30817 unresolved_entries: vec![DecommissionUnresolvedEntry {
30818 bucket: "bucket-a".to_string(),
30819 object: "object-a".to_string(),
30820 pool_index: 0,
30821 set_index: 0,
30822 source_generation: OffsetDateTime::UNIX_EPOCH,
30823 candidate_count: 1,
30824 disk_error_count: 1,
30825 observed_at: OffsetDateTime::UNIX_EPOCH,
30826 reason: "metadata_resolution_failed".to_string(),
30827 }],
30828 ..Default::default()
30829 }),
30830 },
30831 PoolStatus {
30832 id: 1,
30833 cmd_line: "pool-1".to_string(),
30834 last_update: OffsetDateTime::UNIX_EPOCH,
30835 decommission: None,
30836 },
30837 ],
30838 ..Default::default()
30839 };
30840
30841 assert!(ensure_decommission_start_pool_states(&meta, &[0]).is_ok());
30842 }
30843
30844 #[test]
30845 fn test_ensure_valid_decommission_pool_index_accepts_in_range_index() {
30846 assert!(ensure_valid_decommission_pool_index(4, 3).is_ok());
30847 }
30848
30849 #[test]
30850 fn test_ensure_valid_decommission_pool_index_rejects_out_of_range_index() {
30851 let err = ensure_valid_decommission_pool_index(2, 2).expect_err("out-of-range index should fail");
30852 assert!(err.to_string().contains("invalid decommission pool index 2 for 2 pools"));
30853 }
30854
30855 #[test]
30856 fn test_ensure_valid_decommission_pool_index_rejects_when_pool_count_zero() {
30857 let err = ensure_valid_decommission_pool_index(0, 0).expect_err("empty pool list should reject all indices");
30858 assert!(err.to_string().contains("invalid decommission pool index 0 for 0 pools"));
30859 }
30860
30861 #[test]
30862 fn test_classify_decommission_terminal_state_completed_when_no_failures() {
30863 assert_eq!(classify_decommission_terminal_state(false), DecommissionTerminalState::Completed);
30864 }
30865
30866 #[test]
30867 fn test_classify_decommission_terminal_state_failed_when_failures_present() {
30868 assert_eq!(classify_decommission_terminal_state(true), DecommissionTerminalState::Failed);
30869 }
30870
30871 #[test]
30872 fn test_should_preserve_decommission_canceled_state_when_meta_canceled() {
30873 assert!(should_preserve_decommission_canceled_state(true, false));
30874 }
30875
30876 #[test]
30877 fn test_should_preserve_decommission_canceled_state_when_signal_canceled() {
30878 assert!(!should_preserve_decommission_canceled_state(false, true));
30879 }
30880
30881 #[test]
30882 fn test_should_preserve_decommission_canceled_state_when_not_canceled() {
30883 assert!(!should_preserve_decommission_canceled_state(false, false));
30884 }
30885
30886 #[test]
30887 fn test_should_continue_decommission_queue_requires_clean_completion() {
30888 let meta = PoolMeta {
30889 pools: vec![
30890 PoolStatus {
30891 id: 0,
30892 cmd_line: "pool-0".to_string(),
30893 last_update: OffsetDateTime::UNIX_EPOCH,
30894 decommission: Some(PoolDecommissionInfo {
30895 complete: true,
30896 ..Default::default()
30897 }),
30898 },
30899 PoolStatus {
30900 id: 1,
30901 cmd_line: "pool-1".to_string(),
30902 last_update: OffsetDateTime::UNIX_EPOCH,
30903 decommission: Some(PoolDecommissionInfo::default()),
30904 },
30905 PoolStatus {
30906 id: 2,
30907 cmd_line: "pool-2".to_string(),
30908 last_update: OffsetDateTime::UNIX_EPOCH,
30909 decommission: Some(PoolDecommissionInfo {
30910 failed: true,
30911 ..Default::default()
30912 }),
30913 },
30914 PoolStatus {
30915 id: 3,
30916 cmd_line: "pool-3".to_string(),
30917 last_update: OffsetDateTime::UNIX_EPOCH,
30918 decommission: Some(PoolDecommissionInfo {
30919 canceled: true,
30920 ..Default::default()
30921 }),
30922 },
30923 PoolStatus {
30924 id: 4,
30925 cmd_line: "pool-4".to_string(),
30926 last_update: OffsetDateTime::UNIX_EPOCH,
30927 decommission: None,
30928 },
30929 ],
30930 ..Default::default()
30931 };
30932
30933 assert!(should_continue_decommission_queue(&meta, 0));
30934 assert!(!should_continue_decommission_queue(&meta, 1));
30935 assert!(!should_continue_decommission_queue(&meta, 2));
30936 assert!(!should_continue_decommission_queue(&meta, 3));
30937 assert!(!should_continue_decommission_queue(&meta, 4));
30938 assert!(!should_continue_decommission_queue(&meta, 5));
30939 }
30940
30941 #[test]
30942 fn test_decommission_cancel_signal_result_returns_err_when_canceled() {
30943 let err = decommission_cancel_signal_result(true).expect_err("canceled signal should return operation-canceled");
30944 assert!(matches!(err, Error::OperationCanceled));
30945 }
30946
30947 #[test]
30948 fn test_decommission_cancel_signal_result_returns_ok_when_not_canceled() {
30949 assert!(decommission_cancel_signal_result(false).is_ok());
30950 }
30951
30952 #[test]
30953 fn test_is_decommission_cancel_requested_accepts_signal_or_metadata() {
30954 let pool = PoolStatus {
30955 id: 0,
30956 cmd_line: "pool-0".to_string(),
30957 last_update: OffsetDateTime::UNIX_EPOCH,
30958 decommission: Some(PoolDecommissionInfo {
30959 canceled: true,
30960 ..Default::default()
30961 }),
30962 };
30963
30964 assert!(is_decommission_cancel_requested(false, Some(&pool)));
30965 assert!(is_decommission_cancel_requested(true, None));
30966 }
30967
30968 #[test]
30969 fn test_is_decommission_cancel_requested_rejects_active_without_signal() {
30970 let pool = PoolStatus {
30971 id: 0,
30972 cmd_line: "pool-0".to_string(),
30973 last_update: OffsetDateTime::UNIX_EPOCH,
30974 decommission: Some(PoolDecommissionInfo::default()),
30975 };
30976
30977 assert!(!is_decommission_cancel_requested(false, Some(&pool)));
30978 assert!(!is_decommission_cancel_requested(false, None));
30979 }
30980
30981 #[test]
30982 fn test_skip_canceled_decommission_routine_only_for_terminal_canceled_state() {
30983 let canceled = PoolStatus {
30984 id: 0,
30985 cmd_line: "pool-0".to_string(),
30986 last_update: OffsetDateTime::UNIX_EPOCH,
30987 decommission: Some(PoolDecommissionInfo {
30988 canceled: true,
30989 ..Default::default()
30990 }),
30991 };
30992 let active = PoolStatus {
30993 id: 1,
30994 cmd_line: "pool-1".to_string(),
30995 last_update: OffsetDateTime::UNIX_EPOCH,
30996 decommission: Some(PoolDecommissionInfo::default()),
30997 };
30998
30999 assert!(should_skip_canceled_decommission_routine(true, Some(&canceled)));
31000 assert!(!should_skip_canceled_decommission_routine(false, Some(&canceled)));
31001 assert!(!should_skip_canceled_decommission_routine(true, Some(&active)));
31002 assert!(!should_skip_canceled_decommission_routine(true, None));
31003 }
31004
31005 #[test]
31006 fn test_ensure_decommission_cancel_allowed_rejects_missing_pool() {
31007 let err = ensure_decommission_cancel_allowed(false, false, false).expect_err("missing pool should be invalid");
31008 assert!(
31009 err.to_string()
31010 .contains("failed to cancel decommission: target pool was not found")
31011 );
31012 }
31013
31014 #[test]
31015 fn test_should_reject_decommission_cancel_as_terminal_true_when_completed() {
31016 assert!(should_reject_decommission_cancel_as_terminal(true, false));
31017 }
31018
31019 #[test]
31020 fn test_should_reject_decommission_cancel_as_terminal_true_when_failed() {
31021 assert!(should_reject_decommission_cancel_as_terminal(false, true));
31022 }
31023
31024 #[test]
31025 fn test_should_reject_decommission_cancel_as_terminal_false_when_active_or_canceled() {
31026 assert!(!should_reject_decommission_cancel_as_terminal(false, false));
31027 }
31028
31029 #[test]
31030 fn test_should_retry_decommission_cancel_reload_when_changed_or_already_canceled() {
31031 assert!(should_retry_decommission_cancel_reload(true, false));
31032 assert!(should_retry_decommission_cancel_reload(false, true));
31033 assert!(!should_retry_decommission_cancel_reload(false, false));
31034 }
31035
31036 #[test]
31037 fn test_ensure_decommission_cancel_allowed_rejects_not_started() {
31038 let err =
31039 ensure_decommission_cancel_allowed(true, false, false).expect_err("not-started decommission should be rejected");
31040 assert!(matches!(err, Error::DecommissionNotStarted));
31041 }
31042
31043 #[test]
31044 fn test_ensure_decommission_cancel_allowed_rejects_terminal() {
31045 let err = ensure_decommission_cancel_allowed(true, true, true).expect_err("terminal decommission should be rejected");
31046 assert!(matches!(err, Error::DecommissionNotStarted));
31047 }
31048
31049 #[test]
31050 fn test_ensure_decommission_cancel_allowed_allows_active() {
31051 assert!(ensure_decommission_cancel_allowed(true, true, false).is_ok());
31052 }
31053
31054 #[test]
31055 fn test_ensure_decommission_clear_allowed_allows_failed_or_canceled() {
31056 assert!(ensure_decommission_clear_allowed(true, true, false, true, false, 0).is_ok());
31057 assert!(ensure_decommission_clear_allowed(true, true, false, false, true, 0).is_ok());
31058 }
31059
31060 #[test]
31061 fn test_ensure_decommission_clear_allowed_rejects_active_or_completed() {
31062 let active = ensure_decommission_clear_allowed(true, true, false, false, false, 0)
31063 .expect_err("active decommission should not be clearable");
31064 assert!(matches!(active, Error::DecommissionAlreadyRunning));
31065
31066 let complete = ensure_decommission_clear_allowed(true, true, true, false, false, 0)
31067 .expect_err("completed decommission should not be clearable");
31068 assert!(matches!(complete, Error::DecommissionNotStarted));
31069 }
31070
31071 #[test]
31072 fn test_ensure_decommission_clear_allowed_rejects_unresolved_entries() {
31073 let err = ensure_decommission_clear_allowed(true, true, false, true, false, 1)
31074 .expect_err("unresolved entries must survive until a retry reconciles them");
31075
31076 assert!(err.to_string().contains("must be reconciled by retrying decommission"));
31077 }
31078
31079 #[test]
31080 fn test_pool_meta_clear_decommission_restores_failed_or_canceled_pool() {
31081 for decommission in [
31082 PoolDecommissionInfo {
31083 failed: true,
31084 ..Default::default()
31085 },
31086 PoolDecommissionInfo {
31087 canceled: true,
31088 ..Default::default()
31089 },
31090 ] {
31091 let mut meta = PoolMeta {
31092 pools: vec![PoolStatus {
31093 id: 0,
31094 cmd_line: "pool-0".to_string(),
31095 last_update: OffsetDateTime::UNIX_EPOCH,
31096 decommission: Some(decommission),
31097 }],
31098 ..Default::default()
31099 };
31100
31101 assert!(meta.is_suspended(0));
31102 assert!(meta.clear_decommission(0).expect("terminal decommission should clear"));
31103 assert!(
31104 meta.pools[0]
31105 .decommission
31106 .as_ref()
31107 .is_some_and(|info| !info.has_decommission_state())
31108 );
31109 assert!(!meta.is_suspended(0));
31110 }
31111 }
31112
31113 #[test]
31114 fn test_pool_meta_clear_decommission_preserves_unresolved_entries_for_retry() {
31115 let generation = OffsetDateTime::UNIX_EPOCH;
31116 let unresolved_entry = DecommissionUnresolvedEntry {
31117 bucket: "bucket-a".to_string(),
31118 object: "object-a".to_string(),
31119 pool_index: 0,
31120 set_index: 0,
31121 source_generation: generation,
31122 candidate_count: 1,
31123 disk_error_count: 1,
31124 observed_at: generation,
31125 reason: "metadata_resolution_failed".to_string(),
31126 };
31127 let mut meta = PoolMeta {
31128 pools: vec![PoolStatus {
31129 id: 0,
31130 cmd_line: "pool-0".to_string(),
31131 last_update: generation,
31132 decommission: Some(PoolDecommissionInfo {
31133 failed: true,
31134 unresolved_entries: vec![unresolved_entry.clone()],
31135 ..Default::default()
31136 }),
31137 }],
31138 ..Default::default()
31139 };
31140
31141 let err = meta
31142 .clear_decommission(0)
31143 .expect_err("clear must not discard the unresolved-entry recovery ledger");
31144
31145 assert!(err.to_string().contains("must be reconciled by retrying decommission"));
31146 assert_eq!(
31147 meta.pools[0]
31148 .decommission
31149 .as_ref()
31150 .expect("failed state should remain retryable")
31151 .unresolved_entries,
31152 vec![unresolved_entry]
31153 );
31154 }
31155
31156 #[test]
31157 fn test_pool_meta_clear_decommission_rejects_active_or_completed_pool() {
31158 for decommission in [
31159 PoolDecommissionInfo::default(),
31160 PoolDecommissionInfo {
31161 complete: true,
31162 ..Default::default()
31163 },
31164 ] {
31165 let mut meta = PoolMeta {
31166 pools: vec![PoolStatus {
31167 id: 0,
31168 cmd_line: "pool-0".to_string(),
31169 last_update: OffsetDateTime::UNIX_EPOCH,
31170 decommission: Some(decommission),
31171 }],
31172 ..Default::default()
31173 };
31174
31175 assert!(meta.clear_decommission(0).is_err());
31176 assert!(meta.pools[0].decommission.is_some());
31177 }
31178 }
31179
31180 #[test]
31181 fn test_contextualized_decommission_terminal_operation_supported_rejects_single_pool() {
31182 let err = ensure_decommission_terminal_operation_supported(true, "complete decommission")
31183 .expect_err("single-pool decommission terminal operations should be rejected");
31184 assert!(
31185 err.to_string()
31186 .contains("failed to complete decommission: single pool deployments do not support decommission")
31187 );
31188 }
31189
31190 #[test]
31191 fn test_contextualized_decommission_terminal_operation_supported_allows_multi_pool() {
31192 assert!(ensure_decommission_terminal_operation_supported(false, "mark decommission failed").is_ok());
31193 }
31194
31195 #[test]
31196 fn test_contextualized_decommission_start_request_rejects_empty_indices() {
31197 let err = validate_start_decommission_request(&[], false).expect_err("empty decommission target list should be rejected");
31198 assert!(
31199 err.to_string()
31200 .contains("failed to start decommission: no target pools were provided")
31201 );
31202 }
31203
31204 #[test]
31205 fn test_contextualized_decommission_start_request_rejects_single_pool() {
31206 let err = validate_start_decommission_request(&[0], true)
31207 .expect_err("single-pool deployments should reject decommission start");
31208 assert!(
31209 err.to_string()
31210 .contains("failed to start decommission: single pool deployments do not support decommission")
31211 );
31212 }
31213
31214 #[test]
31215 fn test_contextualized_decommission_start_request_allows_multiple_target_pools() {
31216 assert!(validate_start_decommission_request(&[0, 1], false).is_ok());
31217 }
31218
31219 #[test]
31220 fn test_contextualized_decommission_start_request_allows_one_target_pool() {
31221 assert!(validate_start_decommission_request(&[0], false).is_ok());
31222 }
31223
31224 #[test]
31225 fn test_decommission_retry_preserves_and_rebinds_unresolved_entries() {
31226 let previous_generation = OffsetDateTime::UNIX_EPOCH;
31227 let unresolved_entry = DecommissionUnresolvedEntry {
31228 bucket: "bucket-a".to_string(),
31229 object: "object-a".to_string(),
31230 pool_index: 0,
31231 set_index: 0,
31232 source_generation: previous_generation,
31233 candidate_count: 1,
31234 disk_error_count: 1,
31235 observed_at: previous_generation,
31236 reason: "metadata_resolution_failed".to_string(),
31237 };
31238 let mut meta = PoolMeta {
31239 pools: vec![PoolStatus {
31240 id: 0,
31241 cmd_line: "pool-0".to_string(),
31242 last_update: previous_generation,
31243 decommission: Some(PoolDecommissionInfo {
31244 failed: true,
31245 decommissioned_buckets: vec!["bucket-a".to_string()],
31246 items_decommissioned: 7,
31247 bytes_done: 1024,
31248 unresolved_entries: vec![unresolved_entry],
31249 ..Default::default()
31250 }),
31251 }],
31252 ..Default::default()
31253 };
31254
31255 meta.decommission(
31256 0,
31257 PoolSpaceInfo {
31258 total: 200,
31259 free: 50,
31260 used: 150,
31261 },
31262 )
31263 .expect("failed decommission with unresolved entries should retry atomically");
31264
31265 let info = meta.pools[0]
31266 .decommission
31267 .as_ref()
31268 .expect("retried decommission metadata should exist");
31269 let next_generation = info.start_time.expect("retry should assign a new generation");
31270 assert!(next_generation > previous_generation);
31271 assert_eq!(info.decommissioned_buckets, vec!["bucket-a".to_string()]);
31272 assert_eq!(info.items_decommissioned, 7);
31273 assert_eq!(info.bytes_done, 1024);
31274 assert_eq!(info.unresolved_entries.len(), 1);
31275 assert_eq!(info.unresolved_entries[0].source_generation, next_generation);
31276 }
31277
31278 #[test]
31279 fn test_queued_decommission_retry_rebinds_unresolved_entries_when_promoted() {
31280 let previous_generation = OffsetDateTime::UNIX_EPOCH;
31281 let mut meta = PoolMeta {
31282 pools: vec![PoolStatus {
31283 id: 0,
31284 cmd_line: "pool-0".to_string(),
31285 last_update: previous_generation,
31286 decommission: Some(PoolDecommissionInfo {
31287 canceled: true,
31288 unresolved_entries: vec![DecommissionUnresolvedEntry {
31289 bucket: "bucket-a".to_string(),
31290 object: "object-a".to_string(),
31291 pool_index: 0,
31292 set_index: 0,
31293 source_generation: previous_generation,
31294 candidate_count: 1,
31295 disk_error_count: 1,
31296 observed_at: previous_generation,
31297 reason: "metadata_resolution_failed".to_string(),
31298 }],
31299 ..Default::default()
31300 }),
31301 }],
31302 ..Default::default()
31303 };
31304
31305 meta.queue_decommission(
31306 0,
31307 PoolSpaceInfo {
31308 total: 100,
31309 free: 25,
31310 used: 75,
31311 },
31312 )
31313 .expect("canceled decommission with unresolved entries should queue an atomic retry");
31314
31315 assert!(meta.is_suspended(0));
31316 assert!(meta.promote_queued_decommission(0));
31317 let promoted = meta.pools[0]
31318 .decommission
31319 .as_ref()
31320 .expect("promoted decommission metadata should exist");
31321 let generation = promoted.start_time.expect("promotion should assign a generation");
31322 assert_eq!(promoted.unresolved_entries.len(), 1);
31323 assert_eq!(promoted.unresolved_entries[0].source_generation, generation);
31324 }
31325
31326 #[test]
31327 fn test_queued_decommission_promotion_advances_generation_after_clock_rollback() {
31328 let queued_at = OffsetDateTime::from_unix_timestamp(1_260).expect("fixed timestamp should be valid");
31329 let earlier_tick = queued_at - Duration::nanoseconds(10);
31330 let rebalance_floor = queued_at + Duration::nanoseconds(5);
31331 let rebalance = RebalanceMeta {
31332 stopped_at: Some(rebalance_floor),
31333 id: "completed-rebalance".to_string(),
31334 ..Default::default()
31335 };
31336 let mut meta = PoolMeta {
31337 pools: vec![PoolStatus {
31338 id: 0,
31339 cmd_line: "pool-0".to_string(),
31340 last_update: queued_at,
31341 decommission: Some(PoolDecommissionInfo {
31342 queued: true,
31343 ..Default::default()
31344 }),
31345 }],
31346 ..Default::default()
31347 };
31348
31349 assert!(meta.promote_queued_decommission_at_for_test(0, earlier_tick, Some(&rebalance)));
31350 let expected_generation = rebalance_floor + Duration::nanoseconds(1);
31351 let promoted = meta.pools[0]
31352 .decommission
31353 .as_ref()
31354 .expect("promoted decommission metadata should exist");
31355 assert_eq!(meta.pools[0].last_update, expected_generation);
31356 assert_eq!(promoted.start_time, Some(expected_generation));
31357 assert!(!promoted.queued);
31358 }
31359
31360 #[test]
31361 fn test_pool_meta_queued_decommission_is_suspended_to_preserve_reserved_capacity() {
31362 let mut meta = PoolMeta {
31363 pools: vec![PoolStatus {
31364 id: 0,
31365 cmd_line: "pool-0".to_string(),
31366 last_update: OffsetDateTime::UNIX_EPOCH,
31367 decommission: None,
31368 }],
31369 ..Default::default()
31370 };
31371
31372 meta.queue_decommission(
31373 0,
31374 PoolSpaceInfo {
31375 total: 100,
31376 free: 10,
31377 used: 90,
31378 },
31379 )
31380 .expect("queued decommission should be stored");
31381
31382 assert!(meta.is_suspended(0));
31383 assert!(meta.promote_queued_decommission(0));
31384 assert!(meta.is_suspended(0));
31385 }
31386
31387 #[test]
31388 fn test_pool_meta_promoted_queued_decommission_can_be_canceled() {
31389 let mut meta = PoolMeta {
31390 pools: vec![PoolStatus {
31391 id: 0,
31392 cmd_line: "pool-0".to_string(),
31393 last_update: OffsetDateTime::UNIX_EPOCH,
31394 decommission: None,
31395 }],
31396 ..Default::default()
31397 };
31398
31399 meta.queue_decommission(
31400 0,
31401 PoolSpaceInfo {
31402 total: 100,
31403 free: 10,
31404 used: 90,
31405 },
31406 )
31407 .expect("queued decommission should be stored");
31408
31409 assert!(pool_meta_has_active_decommission(&meta));
31410 assert!(meta.promote_queued_decommission(0));
31411 assert!(meta.decommission_cancel(0));
31412
31413 let info = meta.pools[0]
31414 .decommission
31415 .as_ref()
31416 .expect("canceled decommission state should be kept for clear");
31417 assert!(info.canceled);
31418 assert!(!info.queued);
31419 assert!(!info.failed);
31420 assert!(!info.complete);
31421 assert!(!pool_meta_has_active_decommission(&meta));
31422 }
31423
31424 #[test]
31425 fn test_pool_meta_failed_decommission_requires_clear_before_restart() {
31426 let mut meta = PoolMeta {
31427 pools: vec![PoolStatus {
31428 id: 0,
31429 cmd_line: "pool-0".to_string(),
31430 last_update: OffsetDateTime::UNIX_EPOCH,
31431 decommission: Some(PoolDecommissionInfo {
31432 failed: true,
31433 decommissioned_buckets: vec!["bucket-done".to_string()],
31434 queued_buckets: vec!["bucket-pending".to_string()],
31435 bucket: "bucket-pending".to_string(),
31436 prefix: "prefix".to_string(),
31437 object: "object.txt".to_string(),
31438 items_decommissioned: 7,
31439 items_decommission_failed: 3,
31440 bytes_done: 1024,
31441 bytes_failed: 256,
31442 progress_save_item_baseline: 10,
31443 ..Default::default()
31444 }),
31445 }],
31446 ..Default::default()
31447 };
31448
31449 let err = meta
31450 .decommission(
31451 0,
31452 PoolSpaceInfo {
31453 total: 200,
31454 free: 50,
31455 used: 150,
31456 },
31457 )
31458 .expect_err("failed decommission should be blocked until cleared");
31459 assert!(err.to_string().contains("target pool decommission is blocked"));
31460 let blocked = meta.pools[0]
31461 .decommission
31462 .as_ref()
31463 .expect("blocked metadata should remain until clear");
31464 assert!(blocked.failed);
31465 assert_eq!(blocked.decommissioned_buckets, vec!["bucket-done".to_string()]);
31466 assert_eq!(blocked.items_decommissioned, 7);
31467 assert_eq!(blocked.bytes_done, 1024);
31468
31469 assert!(meta.clear_decommission(0).expect("failed decommission should clear"));
31470 assert!(
31471 meta.pools[0]
31472 .decommission
31473 .as_ref()
31474 .is_some_and(|info| !info.has_decommission_state())
31475 );
31476
31477 meta.decommission(
31478 0,
31479 PoolSpaceInfo {
31480 total: 200,
31481 free: 50,
31482 used: 150,
31483 },
31484 )
31485 .expect("cleared decommission should be restartable");
31486 meta.queue_buckets(
31487 0,
31488 vec![
31489 DecomBucketInfo {
31490 name: "bucket-done".to_string(),
31491 prefix: String::new(),
31492 },
31493 DecomBucketInfo {
31494 name: "bucket-pending".to_string(),
31495 prefix: String::new(),
31496 },
31497 ],
31498 );
31499
31500 let info = meta.pools[0]
31501 .decommission
31502 .as_ref()
31503 .expect("decommission info should be rebuilt");
31504 assert!(!info.failed);
31505 assert!(!info.canceled);
31506 assert!(!info.complete);
31507 assert!(info.decommissioned_buckets.is_empty());
31508 assert_eq!(info.queued_buckets, vec!["bucket-done".to_string(), "bucket-pending".to_string()]);
31509 assert_eq!(info.items_decommissioned, 0);
31510 assert_eq!(info.items_decommission_failed, 0);
31511 assert_eq!(info.bytes_done, 0);
31512 assert_eq!(info.bytes_failed, 0);
31513 assert_eq!(info.items_since_last_progress_save(), 0);
31514 assert_eq!(info.start_size, 50);
31515 assert_eq!(info.total_size, 200);
31516 assert_eq!(info.current_size, 50);
31517 assert_eq!(info.bucket, "bucket-pending");
31518 assert!(info.prefix.is_empty());
31519 assert!(info.object.is_empty());
31520 assert!(info.start_time.is_some());
31521 }
31522
31523 #[test]
31524 fn test_pool_meta_canceled_queued_decommission_requires_clear_before_restart() {
31525 let mut meta = PoolMeta {
31526 pools: vec![PoolStatus {
31527 id: 0,
31528 cmd_line: "pool-0".to_string(),
31529 last_update: OffsetDateTime::UNIX_EPOCH,
31530 decommission: Some(PoolDecommissionInfo {
31531 canceled: true,
31532 decommissioned_buckets: vec!["bucket-done".to_string()],
31533 items_decommissioned: 5,
31534 bytes_done: 512,
31535 ..Default::default()
31536 }),
31537 }],
31538 ..Default::default()
31539 };
31540
31541 let err = meta
31542 .queue_decommission(
31543 0,
31544 PoolSpaceInfo {
31545 total: 100,
31546 free: 25,
31547 used: 75,
31548 },
31549 )
31550 .expect_err("canceled queued decommission should be blocked until cleared");
31551 assert!(err.to_string().contains("target pool decommission is blocked"));
31552 let blocked = meta.pools[0]
31553 .decommission
31554 .as_ref()
31555 .expect("blocked metadata should remain until clear");
31556 assert!(blocked.canceled);
31557 assert_eq!(blocked.decommissioned_buckets, vec!["bucket-done".to_string()]);
31558 assert_eq!(blocked.items_decommissioned, 5);
31559 assert_eq!(blocked.bytes_done, 512);
31560
31561 assert!(meta.clear_decommission(0).expect("canceled decommission should clear"));
31562 assert!(
31563 meta.pools[0]
31564 .decommission
31565 .as_ref()
31566 .is_some_and(|info| !info.has_decommission_state())
31567 );
31568
31569 meta.queue_decommission(
31570 0,
31571 PoolSpaceInfo {
31572 total: 100,
31573 free: 25,
31574 used: 75,
31575 },
31576 )
31577 .expect("cleared queued decommission should be restartable");
31578
31579 let info = meta.pools[0]
31580 .decommission
31581 .as_ref()
31582 .expect("decommission info should be rebuilt");
31583 assert!(info.queued);
31584 assert!(info.start_time.is_none());
31585 assert!(info.decommissioned_buckets.is_empty());
31586 assert_eq!(info.items_decommissioned, 0);
31587 assert_eq!(info.bytes_done, 0);
31588 }
31589
31590 #[test]
31591 fn test_contextualized_decommission_listing_disks_available_rejects_empty_set() {
31592 let err = ensure_decommission_listing_disks_available(false, "bucket-a")
31593 .expect_err("missing online disks should be reported with bucket context");
31594 assert!(
31595 err.to_string()
31596 .contains("failed to list objects to decommission for bucket bucket-a: no disks available")
31597 );
31598 }
31599
31600 #[test]
31601 fn test_contextualized_decommission_listing_disks_available_allows_online_disks() {
31602 assert!(ensure_decommission_listing_disks_available(true, "bucket-a").is_ok());
31603 }
31604
31605 #[test]
31606 fn test_require_decommission_store_returns_value_when_present() {
31607 let store = require_decommission_store(Some(7_u8), "start decommission").expect("present store should be returned");
31608 assert_eq!(store, 7);
31609 }
31610
31611 #[test]
31612 fn test_require_decommission_store_returns_error_when_missing() {
31613 let err = require_decommission_store::<u8>(None, "start decommission").expect_err("missing store should return error");
31614 assert!(
31615 err.to_string()
31616 .contains("failed to start decommission: store not initialized")
31617 );
31618 }
31619
31620 #[test]
31621 fn test_bind_decommission_cancelers_binds_existing_slots_only() {
31622 let parent = CancellationToken::new();
31623 let mut cancelers = vec![None, None];
31624
31625 let bound = bind_decommission_cancelers(&[0, 3, 1], &parent, cancelers.as_mut_slice());
31626
31627 assert_eq!(bound.len(), 2);
31628 assert_eq!(bound[0].0, 0);
31629 assert_eq!(bound[1].0, 1);
31630 assert!(cancelers[0].is_some());
31631 assert!(cancelers[1].is_some());
31632 }
31633
31634 #[test]
31635 fn test_bind_decommission_cancelers_child_tokens_follow_parent_cancel() {
31636 let parent = CancellationToken::new();
31637 let mut cancelers = vec![None];
31638
31639 let bound = bind_decommission_cancelers(&[0], &parent, cancelers.as_mut_slice());
31640 assert_eq!(bound.len(), 1);
31641 assert!(!bound[0].1.is_cancelled());
31642
31643 parent.cancel();
31644 assert!(bound[0].1.is_cancelled());
31645 }
31646
31647 #[test]
31648 fn test_bind_decommission_cancelers_replaces_existing_slot() {
31649 let parent = CancellationToken::new();
31650 let existing = DecommissionCanceler::new(CancellationToken::new());
31651 let mut cancelers = vec![Some(existing.clone())];
31652
31653 let bound = bind_decommission_cancelers(&[0], &parent, cancelers.as_mut_slice());
31654
31655 assert_eq!(bound.len(), 1);
31656 assert_eq!(bound[0].0, 0);
31657 assert!(existing.is_cancelled());
31658 let replacement = cancelers[0].as_ref().expect("replacement token should be stored");
31659 assert!(!replacement.is_cancelled());
31660 parent.cancel();
31661 assert!(replacement.is_cancelled());
31662 }
31663
31664 #[test]
31665 fn test_bind_missing_decommission_cancelers_stops_at_existing_slot() {
31666 let parent = CancellationToken::new();
31667 let existing = DecommissionCanceler::new(CancellationToken::new());
31668 let mut cancelers = vec![None, Some(existing.clone()), None];
31669
31670 let bound = bind_missing_decommission_cancelers(&[0, 1, 2], &parent, cancelers.as_mut_slice());
31671
31672 assert_eq!(bound.len(), 1);
31673 assert_eq!(bound[0].0, 0);
31674 assert!(cancelers[0].is_some());
31675 assert!(cancelers[1].is_some());
31676 assert!(cancelers[2].is_none());
31677 assert!(!existing.is_cancelled());
31678 }
31679
31680 #[test]
31681 fn test_serialized_decommission_double_start_preserves_first_operation() {
31682 let mut pool_meta = PoolMeta {
31683 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
31684 ..Default::default()
31685 };
31686 let first_parent = CancellationToken::new();
31687 let second_parent = CancellationToken::new();
31688 let mut cancelers = vec![None, None];
31689
31690 let first = reserve_decommission_start_cancelers(&pool_meta, &[0], &[0], &first_parent, cancelers.as_mut_slice())
31691 .expect("first start should reserve its worker");
31692 pool_meta
31693 .decommission(
31694 0,
31695 PoolSpaceInfo {
31696 total: 100,
31697 free: 40,
31698 used: 60,
31699 },
31700 )
31701 .expect("first start should install active metadata");
31702
31703 let second = reserve_decommission_start_cancelers(&pool_meta, &[0], &[0], &second_parent, cancelers.as_mut_slice());
31704
31705 assert!(matches!(second, Err(Error::DecommissionAlreadyRunning)));
31706 let current = cancelers[0].as_ref().expect("first operation should retain the slot");
31707 assert!(current.owns_same_operation(first[0].1.canceler()));
31708 assert!(current.is_active());
31709 assert!(!first_parent.is_cancelled());
31710 }
31711
31712 #[test]
31713 fn test_local_decommission_queue_prefix_stops_at_remote_leader() {
31714 let endpoints = EndpointServerPools::from(vec![
31715 decommission_test_pool_endpoint(0, true),
31716 decommission_test_pool_endpoint(1, true),
31717 decommission_test_pool_endpoint(2, false),
31718 decommission_test_pool_endpoint(3, true),
31719 ]);
31720
31721 let local = local_decommission_queue_prefix(&endpoints, &[0, 1, 2, 3]).expect("prefix should resolve");
31722
31723 assert_eq!(local, vec![0, 1]);
31724 }
31725
31726 #[test]
31727 fn test_local_decommission_queue_prefix_empty_when_first_leader_remote() {
31728 let endpoints = EndpointServerPools::from(vec![
31729 decommission_test_pool_endpoint(0, false),
31730 decommission_test_pool_endpoint(1, true),
31731 ]);
31732
31733 let local = local_decommission_queue_prefix(&endpoints, &[0, 1]).expect("prefix should resolve");
31734
31735 assert!(local.is_empty());
31736 }
31737
31738 #[test]
31739 fn test_decommission_start_local_leader_allows_remote_queued_pool() {
31740 let endpoints = EndpointServerPools::from(vec![
31741 decommission_test_pool_endpoint(0, true),
31742 decommission_test_pool_endpoint(1, false),
31743 ]);
31744
31745 assert!(ensure_decommission_start_local_leader(&endpoints, &[0, 1]).is_ok());
31746 }
31747
31748 #[test]
31749 fn test_decommission_start_local_leader_rejects_remote_active_pool() {
31750 let endpoints = EndpointServerPools::from(vec![decommission_test_pool_endpoint(0, false)]);
31751
31752 let err = ensure_decommission_start_local_leader(&endpoints, &[0]).expect_err("remote active pool should be rejected");
31753
31754 assert!(
31755 err.to_string()
31756 .contains("decommission for pool 0 must run on the pool first endpoint")
31757 );
31758 }
31759
31760 #[test]
31761 fn test_missing_decommission_worker_prefix_stops_at_active_worker() {
31762 let cancelers = vec![None, Some(DecommissionCanceler::new(CancellationToken::new())), None];
31763
31764 let missing = missing_decommission_worker_prefix(&[0, 1, 2], cancelers.as_slice());
31765
31766 assert_eq!(missing, vec![0]);
31767 }
31768
31769 #[test]
31770 fn test_resumable_decommission_queue_indices_skip_terminal_predecessors() {
31771 let meta = PoolMeta {
31772 pools: vec![
31773 decommission_test_pool_status(
31774 0,
31775 Some(PoolDecommissionInfo {
31776 complete: true,
31777 ..Default::default()
31778 }),
31779 ),
31780 decommission_test_pool_status(
31781 1,
31782 Some(PoolDecommissionInfo {
31783 canceled: true,
31784 ..Default::default()
31785 }),
31786 ),
31787 decommission_test_pool_status(
31788 2,
31789 Some(PoolDecommissionInfo {
31790 failed: true,
31791 ..Default::default()
31792 }),
31793 ),
31794 decommission_test_pool_status(
31795 3,
31796 Some(PoolDecommissionInfo {
31797 queued: true,
31798 ..Default::default()
31799 }),
31800 ),
31801 decommission_test_pool_status(4, None),
31802 ],
31803 ..Default::default()
31804 };
31805
31806 assert_eq!(resumable_decommission_queue_indices(&meta), vec![3]);
31807 }
31808
31809 #[test]
31810 fn test_resumable_decommission_queue_indices_preserve_active_predecessor_order() {
31811 let meta = PoolMeta {
31812 pools: vec![
31813 decommission_test_pool_status(
31814 0,
31815 Some(PoolDecommissionInfo {
31816 complete: true,
31817 ..Default::default()
31818 }),
31819 ),
31820 decommission_test_pool_status(
31821 1,
31822 Some(PoolDecommissionInfo {
31823 start_time: Some(OffsetDateTime::UNIX_EPOCH),
31824 ..Default::default()
31825 }),
31826 ),
31827 decommission_test_pool_status(
31828 2,
31829 Some(PoolDecommissionInfo {
31830 queued: true,
31831 ..Default::default()
31832 }),
31833 ),
31834 ],
31835 ..Default::default()
31836 };
31837
31838 assert_eq!(resumable_decommission_queue_indices(&meta), vec![1, 2]);
31839 }
31840
31841 #[tokio::test]
31842 async fn test_runtime_recovery_reserves_the_startup_resumable_queue() {
31843 let meta = PoolMeta {
31844 version: super::POOL_META_VERSION,
31845 pools: vec![
31846 decommission_test_pool_status(
31847 0,
31848 Some(PoolDecommissionInfo {
31849 failed: true,
31850 ..Default::default()
31851 }),
31852 ),
31853 decommission_test_pool_status(
31854 1,
31855 Some(PoolDecommissionInfo {
31856 canceled: true,
31857 ..Default::default()
31858 }),
31859 ),
31860 decommission_test_pool_status(
31861 2,
31862 Some(PoolDecommissionInfo {
31863 start_time: Some(OffsetDateTime::UNIX_EPOCH),
31864 ..Default::default()
31865 }),
31866 ),
31867 decommission_test_pool_status(
31868 3,
31869 Some(PoolDecommissionInfo {
31870 queued: true,
31871 ..Default::default()
31872 }),
31873 ),
31874 ],
31875 ..Default::default()
31876 };
31877 let startup_ids = meta
31878 .return_resumable_pools()
31879 .into_iter()
31880 .map(|pool| pool.id)
31881 .collect::<Vec<_>>();
31882 let store = decommission_worker_test_store(meta, vec![None, None, None, None]);
31883 let endpoints = EndpointServerPools::from(vec![
31884 decommission_test_pool_endpoint(0, true),
31885 decommission_test_pool_endpoint(1, true),
31886 decommission_test_pool_endpoint(2, true),
31887 decommission_test_pool_endpoint(3, true),
31888 ]);
31889 let reserved = store
31890 .reserve_missing_local_decommission_routines(&CancellationToken::new(), &endpoints)
31891 .await
31892 .expect("runtime recovery reservation should succeed");
31893 let runtime_indices = reserved.iter().map(|(idx, _)| *idx).collect::<Vec<_>>();
31894
31895 assert_eq!(runtime_indices, vec![2, 3]);
31896 assert_eq!(startup_ids, vec![2, 3]);
31897 }
31898
31899 #[tokio::test]
31900 async fn test_runtime_recovery_does_not_reserve_behind_active_predecessor() {
31901 let meta = PoolMeta {
31902 version: super::POOL_META_VERSION,
31903 pools: vec![
31904 decommission_test_pool_status(
31905 0,
31906 Some(PoolDecommissionInfo {
31907 failed: true,
31908 ..Default::default()
31909 }),
31910 ),
31911 decommission_test_pool_status(
31912 1,
31913 Some(PoolDecommissionInfo {
31914 canceled: true,
31915 ..Default::default()
31916 }),
31917 ),
31918 decommission_test_pool_status(
31919 2,
31920 Some(PoolDecommissionInfo {
31921 start_time: Some(OffsetDateTime::UNIX_EPOCH),
31922 ..Default::default()
31923 }),
31924 ),
31925 decommission_test_pool_status(
31926 3,
31927 Some(PoolDecommissionInfo {
31928 queued: true,
31929 ..Default::default()
31930 }),
31931 ),
31932 ],
31933 ..Default::default()
31934 };
31935 let active = DecommissionCanceler::new(CancellationToken::new());
31936 let store = decommission_worker_test_store(meta, vec![None, None, Some(active.clone()), None]);
31937 let endpoints = EndpointServerPools::from(vec![
31938 decommission_test_pool_endpoint(0, true),
31939 decommission_test_pool_endpoint(1, true),
31940 decommission_test_pool_endpoint(2, true),
31941 decommission_test_pool_endpoint(3, true),
31942 ]);
31943
31944 let reserved = store
31945 .reserve_missing_local_decommission_routines(&CancellationToken::new(), &endpoints)
31946 .await
31947 .expect("runtime recovery reservation should succeed");
31948
31949 assert!(reserved.is_empty());
31950 assert!(active.is_active());
31951 }
31952
31953 #[test]
31954 fn test_take_decommission_canceler_takes_and_clears_slot() {
31955 let canceler = DecommissionCanceler::new(CancellationToken::new());
31956 let mut cancelers = vec![Some(canceler)];
31957
31958 let taken = take_decommission_canceler(cancelers.as_mut_slice(), 0);
31959 assert!(taken.is_some());
31960 assert!(cancelers[0].is_none());
31961 }
31962
31963 #[test]
31964 fn test_take_decommission_canceler_returns_none_for_missing_slot() {
31965 let mut cancelers: Vec<Option<DecommissionCanceler>> = Vec::new();
31966 assert!(take_decommission_canceler(cancelers.as_mut_slice(), 0).is_none());
31967 }
31968
31969 #[test]
31970 fn test_has_active_decommission_canceler_true_when_any_slot_present() {
31971 let cancelers = vec![None, Some(DecommissionCanceler::new(CancellationToken::new()))];
31972 assert!(has_active_decommission_canceler(cancelers.as_slice()));
31973 }
31974
31975 #[test]
31976 fn test_has_active_decommission_canceler_false_when_all_empty() {
31977 let cancelers = vec![None, None];
31978 assert!(!has_active_decommission_canceler(cancelers.as_slice()));
31979 }
31980
31981 #[test]
31982 fn test_cancel_decommission_canceler_cancels_when_present() {
31983 let canceler = DecommissionCanceler::new(CancellationToken::new());
31984 let canceled = cancel_decommission_canceler(Some(canceler.clone()));
31985
31986 assert!(canceled);
31987 assert!(canceler.is_cancelled());
31988 assert!(!canceler.is_active());
31989 }
31990
31991 #[test]
31992 fn test_cancel_decommission_canceler_returns_false_when_missing() {
31993 assert!(!cancel_decommission_canceler(None));
31994 }
31995
31996 #[test]
31997 fn test_take_and_cancel_decommission_canceler_clears_slot() {
31998 let canceler = DecommissionCanceler::new(CancellationToken::new());
31999 let mut cancelers = vec![Some(canceler.clone())];
32000
32001 assert!(take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), 0));
32002 assert!(cancelers[0].is_none());
32003 assert!(canceler.is_cancelled());
32004 assert!(!canceler.is_active());
32005 }
32006
32007 #[test]
32008 fn test_take_and_cancel_decommission_canceler_missing_slot_is_false() {
32009 let mut cancelers = vec![None];
32010
32011 assert!(!take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), 0));
32012 assert!(cancelers[0].is_none());
32013 }
32014
32015 #[test]
32016 fn test_guarded_decommission_future_releases_without_first_poll() {
32017 let canceler = DecommissionCanceler::new(CancellationToken::new());
32018 let cancelers = vec![Some(canceler.clone())];
32019 let guards = guard_decommission_cancelers(vec![(0, canceler.clone())]);
32020 let unpolled = async move {
32021 let _guards = guards;
32022 std::future::pending::<()>().await;
32023 };
32024
32025 drop(unpolled);
32026
32027 assert!(canceler.is_cancelled());
32028 assert!(!has_active_decommission_canceler(cancelers.as_slice()));
32029 }
32030
32031 #[test]
32032 fn test_partial_decommission_spawn_reservation_releases_bound_slot() {
32033 let parent = CancellationToken::new();
32034 let mut cancelers = vec![None];
32035 let bound = bind_decommission_cancelers(&[0, 1], &parent, cancelers.as_mut_slice());
32036 let guards = guard_decommission_cancelers(bound);
32037
32038 let result = super::ensure_decommission_routines_scheduled(guards.len(), 2);
32039 drop(guards);
32040
32041 assert!(result.is_err());
32042 assert!(!has_active_decommission_canceler(cancelers.as_slice()));
32043 }
32044
32045 #[tokio::test]
32046 async fn test_decommission_supervisor_observes_worker_abort() {
32047 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
32048 let worker = tokio::spawn(async move {
32049 started_tx.send(()).expect("worker start should be observed");
32050 std::future::pending::<()>().await;
32051 #[allow(unreachable_code)]
32052 Ok(())
32053 });
32054
32055 started_rx.await.expect("worker start should be observed");
32056 worker.abort();
32057 let err = await_decommission_worker(3, worker)
32058 .await
32059 .expect_err("supervisor should observe aborted worker");
32060
32061 assert!(err.to_string().contains("decommission worker 3 task join error"));
32062 }
32063
32064 #[tokio::test]
32065 async fn test_decommission_supervisor_observes_worker_panic() {
32066 let worker = tokio::spawn(async move {
32067 panic!("injected decommission worker panic");
32068 });
32069
32070 let err = await_decommission_worker(4, worker)
32071 .await
32072 .expect_err("supervisor should observe panicked worker");
32073
32074 assert!(err.to_string().contains("decommission worker 4 task join error"));
32075 }
32076
32077 #[tokio::test]
32078 async fn test_decommission_worker_metadata_missing_releases_owned_slot() {
32079 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
32080 let canceler = DecommissionCanceler::new(CancellationToken::new());
32081 *store.pool_meta.write().await = PoolMeta::default();
32082 store.decommission_cancelers.write().await[0] = Some(canceler.clone());
32083
32084 let err = store
32085 .do_decommission_in_routine(canceler.clone(), 0, Arc::new(Semaphore::new(1)))
32086 .await
32087 .expect_err("missing worker metadata should fail the routine");
32088
32089 assert!(err.to_string().contains("target pool was not found"));
32090 assert!(!canceler.is_active());
32091 assert!(store.decommission_cancelers.read().await[0].is_none());
32092 }
32093
32094 #[tokio::test]
32095 async fn test_decommission_supervisor_failure_cancels_queued_successor() {
32096 let first = DecommissionCanceler::new(CancellationToken::new());
32097 let queued = DecommissionCanceler::new(CancellationToken::new());
32098 let store = decommission_worker_test_store(PoolMeta::default(), vec![Some(first.clone()), Some(queued.clone())]);
32099 let guards = guard_decommission_cancelers(vec![(0, first.clone()), (1, queued.clone())]);
32100
32101 spawn_decommission_index_cancelers(store.clone(), CancellationToken::new(), guards, Arc::new(Semaphore::new(1)))
32102 .await
32103 .expect("decommission supervisor should finish after queued cleanup");
32104
32105 assert!(!first.is_active());
32106 assert!(!queued.is_active());
32107 assert!(queued.is_cancelled());
32108 assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
32109 }
32110
32111 #[tokio::test]
32112 async fn test_decommission_supervisor_releases_slot_without_terminal_retry_when_pool_meta_is_blocked() {
32113 let generation = OffsetDateTime::UNIX_EPOCH;
32114 let canceler = DecommissionCanceler::new(CancellationToken::new());
32115 let pool_meta = PoolMeta {
32116 version: super::POOL_META_VERSION,
32117 pools: vec![decommission_test_pool_status(
32118 0,
32119 Some(PoolDecommissionInfo {
32120 start_time: Some(generation),
32121 ..Default::default()
32122 }),
32123 )],
32124 ..Default::default()
32125 };
32126 let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
32127 store
32128 .pool_meta_save_gate
32129 .lock()
32130 .await
32131 .observe_replicas(super::PoolMetaReplicaState {
32132 needs_repair: true,
32133 repair_write_safe: false,
32134 });
32135 let guards = guard_decommission_cancelers(vec![(0, canceler.clone())]);
32136
32137 tokio::time::timeout(
32138 StdDuration::from_secs(1),
32139 spawn_decommission_index_cancelers(store.clone(), CancellationToken::new(), guards, Arc::new(Semaphore::new(1))),
32140 )
32141 .await
32142 .expect("blocked supervisor must not enter terminal retry")
32143 .expect("blocked supervisor task should not panic");
32144
32145 assert!(store.decommission_cancelers.read().await[0].is_none());
32146 assert!(!canceler.is_active());
32147 let pool_meta = store.pool_meta.read().await;
32148 let info = pool_meta.pools[0]
32149 .decommission
32150 .as_ref()
32151 .expect("blocked decommission metadata should remain active");
32152 assert!(!info.failed);
32153 assert!(!info.canceled);
32154 assert!(!info.complete);
32155 assert_eq!(info.start_time, Some(generation));
32156 }
32157
32158 #[tokio::test]
32159 async fn test_decommission_promotion_rechecks_sticky_gate_after_start_wait() {
32160 let pool_meta = PoolMeta {
32161 pools: vec![decommission_test_pool_status(
32162 0,
32163 Some(PoolDecommissionInfo {
32164 queued: true,
32165 ..Default::default()
32166 }),
32167 )],
32168 ..Default::default()
32169 };
32170 let store = decommission_worker_test_store(pool_meta, vec![None]);
32171 let start_guard = store.start_gate.lock().await;
32172 let mut promotion = tokio::spawn({
32173 let store = store.clone();
32174 async move { store.promote_queued_decommission_for_test(0).await }
32175 });
32176 tokio::task::yield_now().await;
32177 assert!(!promotion.is_finished(), "promotion should be waiting for the start gate");
32178
32179 store
32180 .pool_meta_save_gate
32181 .lock()
32182 .await
32183 .observe_replicas(super::PoolMetaReplicaState {
32184 needs_repair: true,
32185 repair_write_safe: false,
32186 });
32187 drop(start_guard);
32188
32189 let err = tokio::time::timeout(StdDuration::from_secs(1), &mut promotion)
32190 .await
32191 .expect("blocked promotion should finish")
32192 .expect("promotion task should not panic")
32193 .expect_err("promotion must recheck the sticky gate after waiting");
32194 assert!(
32195 err.to_string()
32196 .contains("restart after all replicas are readable and consistent")
32197 );
32198
32199 let pool_meta = store.pool_meta.read().await;
32200 let info = pool_meta.pools[0]
32201 .decommission
32202 .as_ref()
32203 .expect("queued metadata should remain present");
32204 assert!(info.queued);
32205 assert!(info.start_time.is_none());
32206 }
32207
32208 #[tokio::test]
32209 #[serial_test::serial(pool_meta_version_env)]
32210 async fn test_decommission_promotion_persists_all_recovered_capacity_sources() {
32211 let (_temp_dirs, store, _other_store) =
32212 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
32213 persist_v3_pool_meta_for_test(&store).await;
32214 let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(3);
32215 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
32216 let mut queued_reservation = build_decommission_capacity_reservation(
32217 DecommissionPoolCapacityInfo::for_test(1, layout, 90, 100, 10),
32218 layout,
32219 uuid::Uuid::new_v4(),
32220 7,
32221 now,
32222 )
32223 .expect("queued source reservation should be valid");
32224 queued_reservation.targets.push(DecommissionCapacityTarget {
32225 pool_index: 2,
32226 layout,
32227 physical_total_at_reservation: 200,
32228 physical_free_at_reservation: 100,
32229 reserved_physical_bytes: queued_reservation.peak_physical_bytes,
32230 consumed_physical_bytes: 0,
32231 observed_physical_bytes: 0,
32232 inflight_physical_bytes: 0,
32233 pending_physical_bytes: 0,
32234 pending_mutation_id: None,
32235 temporary_mutations: Vec::new(),
32236 });
32237 {
32238 let mut pool_meta = store.pool_meta.write().await;
32239 pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
32240 start_time: Some(now),
32241 ..Default::default()
32242 });
32243 pool_meta.pools[1].decommission = Some(PoolDecommissionInfo {
32244 queued: true,
32245 start_time: Some(now),
32246 capacity_reservation: Some(queued_reservation),
32247 ..Default::default()
32248 });
32249 }
32250 let mut loaded_v3 = PoolMeta::default();
32251 loaded_v3
32252 .load_no_lock_from_replicas(store.pools.clone())
32253 .await
32254 .expect("the startup fixture must provide a durable V3 pool metadata replica");
32255 assert_eq!(loaded_v3.version, POOL_META_GENERATION_VERSION);
32256 store
32257 .save_current_pool_meta_for_test(&[0, 1, 2])
32258 .await
32259 .expect("active and queued source metadata should be persisted before promotion");
32260 set_decommission_capacity_info_overrides_for_test(
32261 store.id,
32262 vec![vec![
32263 DecommissionPoolCapacityInfo::for_test(0, layout, 90, 100, 10),
32264 DecommissionPoolCapacityInfo::for_test(1, layout, 90, 100, 10),
32265 DecommissionPoolCapacityInfo::for_test(2, layout, 100, 200, 100),
32266 ]],
32267 );
32268
32269 store
32270 .promote_queued_decommission_for_test(1)
32271 .await
32272 .expect("queued promotion should recover all active source reservations");
32273
32274 let mut persisted = PoolMeta::default();
32275 persisted
32276 .load_no_lock_from_replicas(store.pools.clone())
32277 .await
32278 .expect("promoted active and recovered source metadata should reload");
32279 assert_eq!(persisted.version, POOL_META_GENERATION_VERSION);
32280 assert!(
32281 persisted.pools[0]
32282 .decommission
32283 .as_ref()
32284 .and_then(|info| info.capacity_reservation.as_ref())
32285 .is_some_and(DecommissionCapacityReservation::active)
32286 );
32287 let promoted = persisted.pools[1]
32288 .decommission
32289 .as_ref()
32290 .expect("queued source should remain after promotion");
32291 assert!(!promoted.queued, "queued source should be promoted durably");
32292 assert!(
32293 promoted
32294 .capacity_reservation
32295 .as_ref()
32296 .is_some_and(DecommissionCapacityReservation::active)
32297 );
32298 }
32299
32300 #[tokio::test]
32301 async fn test_decommission_bucket_done_rechecks_sticky_gate_before_mutation() {
32302 let bucket = DecomBucketInfo {
32303 name: "bucket-a".to_string(),
32304 prefix: String::new(),
32305 };
32306 let pool_meta = PoolMeta {
32307 pools: vec![decommission_test_pool_status(
32308 0,
32309 Some(PoolDecommissionInfo {
32310 queued_buckets: vec![bucket.to_string()],
32311 ..Default::default()
32312 }),
32313 )],
32314 ..Default::default()
32315 };
32316 let store = decommission_worker_test_store(pool_meta, vec![None]);
32317 store
32318 .pool_meta_save_gate
32319 .lock()
32320 .await
32321 .observe_replicas(super::PoolMetaReplicaState {
32322 needs_repair: true,
32323 repair_write_safe: false,
32324 });
32325
32326 let err = store
32327 .mark_decommission_bucket_done_and_save(0, OffsetDateTime::UNIX_EPOCH, &bucket)
32328 .await
32329 .expect_err("bucket completion must stop before mutating sticky pool metadata");
32330 assert!(
32331 err.to_string()
32332 .contains("restart after all replicas are readable and consistent")
32333 );
32334
32335 let pool_meta = store.pool_meta.read().await;
32336 let info = pool_meta.pools[0]
32337 .decommission
32338 .as_ref()
32339 .expect("decommission metadata should remain present");
32340 assert_eq!(info.queued_buckets, vec![bucket.to_string()]);
32341 assert!(info.decommissioned_buckets.is_empty());
32342 }
32343
32344 #[tokio::test]
32345 async fn test_decommission_cancel_save_failure_preserves_generation_and_blocks_retry() {
32346 let generation = OffsetDateTime::UNIX_EPOCH;
32347 let canceler = DecommissionCanceler::new(CancellationToken::new());
32348 let pool_meta = PoolMeta {
32349 version: super::POOL_META_VERSION,
32350 pools: vec![decommission_test_pool_status(
32351 0,
32352 Some(PoolDecommissionInfo {
32353 start_time: Some(generation),
32354 ..Default::default()
32355 }),
32356 )],
32357 ..Default::default()
32358 };
32359 let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
32360
32361 let err = store
32362 .decommission_cancel_with_owner_and_save(0, Some(&canceler), |_, _| async { Err(Error::Timeout) })
32363 .await
32364 .expect_err("injected pool metadata timeout should fail cancel");
32365 assert!(matches!(err, Error::Timeout));
32366
32367 {
32368 let pool_meta = store.pool_meta.read().await;
32369 let info = pool_meta.pools[0]
32370 .decommission
32371 .as_ref()
32372 .expect("failed cancel must retain decommission metadata");
32373 assert_eq!(info.start_time, Some(generation));
32374 assert!(!info.canceled);
32375 assert!(!info.complete);
32376 assert!(!info.failed);
32377 assert!(ensure_decommission_generation(&pool_meta, 0, generation).is_ok());
32378 }
32379 {
32380 let cancelers = store.decommission_cancelers.read().await;
32381 let current = cancelers[0].as_ref().expect("failed cancel must retain the worker owner");
32382 assert!(current.owns_same_operation(&canceler));
32383 assert!(current.is_active());
32384 }
32385 assert!(!canceler.is_cancelled());
32386 assert!(store.decommission_terminal_retryable_for_operation(0, &canceler).await);
32387
32388 let retry_save_called = Arc::new(AtomicBool::new(false));
32389 let retry_save_called_by_closure = retry_save_called.clone();
32390 let retry_err = store
32391 .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |_, _| async move {
32392 retry_save_called_by_closure.store(true, Ordering::SeqCst);
32393 Ok(())
32394 })
32395 .await
32396 .expect_err("an ambiguous save failure must block same-process retry");
32397 assert!(
32398 retry_err
32399 .to_string()
32400 .contains("restart after all replicas are readable and consistent")
32401 );
32402 assert!(!retry_save_called.load(Ordering::SeqCst));
32403
32404 {
32405 let pool_meta = store.pool_meta.read().await;
32406 let info = pool_meta.pools[0]
32407 .decommission
32408 .as_ref()
32409 .expect("blocked retry must retain decommission metadata");
32410 assert_eq!(info.start_time, Some(generation));
32411 assert!(!info.canceled);
32412 assert!(!info.complete);
32413 assert!(!info.failed);
32414 }
32415 assert!(store.decommission_cancelers.read().await[0].is_some());
32416 assert!(canceler.is_active());
32417 assert!(!canceler.is_cancelled());
32418 }
32419
32420 #[tokio::test]
32421 async fn test_decommission_cancel_stays_blocked_after_unreadable_pool_meta_replica() {
32422 let generation = OffsetDateTime::UNIX_EPOCH;
32423 let canceler = DecommissionCanceler::new(CancellationToken::new());
32424 let pool_meta = PoolMeta {
32425 version: super::POOL_META_VERSION,
32426 pools: vec![decommission_test_pool_status(
32427 0,
32428 Some(PoolDecommissionInfo {
32429 start_time: Some(generation),
32430 ..Default::default()
32431 }),
32432 )],
32433 ..Default::default()
32434 };
32435 let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
32436 store
32437 .pool_meta_save_gate
32438 .lock()
32439 .await
32440 .observe_replicas(super::PoolMetaReplicaState {
32441 needs_repair: true,
32442 repair_write_safe: false,
32443 });
32444 let save_called = Arc::new(AtomicBool::new(false));
32445 let save_called_by_closure = save_called.clone();
32446
32447 let err = store
32448 .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |_, _| async move {
32449 save_called_by_closure.store(true, Ordering::SeqCst);
32450 Ok(())
32451 })
32452 .await
32453 .expect_err("cancel must remain blocked until restart after an unreadable replica");
32454
32455 assert!(
32456 err.to_string()
32457 .contains("restart after all replicas are readable and consistent")
32458 );
32459 assert!(!save_called.load(Ordering::SeqCst));
32460 let pool_meta = store.pool_meta.read().await;
32461 let info = pool_meta.pools[0]
32462 .decommission
32463 .as_ref()
32464 .expect("blocked cancel must preserve decommission metadata");
32465 assert_eq!(info.start_time, Some(generation));
32466 assert!(!info.canceled);
32467 drop(pool_meta);
32468 assert!(canceler.is_active());
32469 assert!(!canceler.is_cancelled());
32470 let cancelers = store.decommission_cancelers.read().await;
32471 assert!(
32472 cancelers[0]
32473 .as_ref()
32474 .is_some_and(|current| current.owns_same_operation(&canceler))
32475 );
32476 }
32477
32478 #[tokio::test]
32479 async fn test_decommission_cancel_serializes_reload_until_local_commit() {
32480 let generation = OffsetDateTime::UNIX_EPOCH;
32481 let canceler = DecommissionCanceler::new(CancellationToken::new());
32482 let pool_meta = PoolMeta {
32483 version: super::POOL_META_VERSION,
32484 pools: vec![decommission_test_pool_status(
32485 0,
32486 Some(PoolDecommissionInfo {
32487 start_time: Some(generation),
32488 stage: "migrate_object".to_string(),
32489 ..Default::default()
32490 }),
32491 )],
32492 ..Default::default()
32493 };
32494 let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
32495 let (persisted_tx, persisted_rx) = tokio::sync::oneshot::channel();
32496 let save_release = Arc::new(tokio::sync::Notify::new());
32497
32498 let cancel = tokio::spawn({
32499 let store = store.clone();
32500 let canceler = canceler.clone();
32501 let save_release = save_release.clone();
32502 async move {
32503 store
32504 .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move {
32505 persisted_tx
32506 .send(snapshot.encode_config_data()?)
32507 .map_err(|_| Error::other("failed to expose saved cancel snapshot"))?;
32508 save_release.notified().await;
32509 Ok(())
32510 })
32511 .await
32512 }
32513 });
32514 let persisted = persisted_rx.await.expect("save should expose the canceled snapshot");
32515 let reload_started = Arc::new(tokio::sync::Notify::new());
32516 let reload = tokio::spawn({
32517 let store = store.clone();
32518 let reload_started = reload_started.clone();
32519 async move {
32520 let mut reloaded = PoolMeta::default();
32521 reloaded.load_from_config_data(persisted)?;
32522 reload_started.notify_one();
32523 *store.pool_meta.write().await = reloaded;
32524 Ok::<(), Error>(())
32525 }
32526 });
32527 reload_started.notified().await;
32528 assert!(!reload.is_finished(), "peer reload must wait for cancel publication");
32529
32530 save_release.notify_one();
32531 cancel
32532 .await
32533 .expect("cancel task should not panic")
32534 .expect("cancel should commit before releasing the reload");
32535 reload
32536 .await
32537 .expect("reload task should not panic")
32538 .expect("reload should install the saved cancel after publication");
32539
32540 let pool_meta = store.pool_meta.read().await;
32541 let info = pool_meta.pools[0]
32542 .decommission
32543 .as_ref()
32544 .expect("reloaded cancel metadata should remain present");
32545 assert!(info.canceled);
32546 assert!(!info.complete);
32547 assert!(!info.failed);
32548 assert!(info.start_time.is_none());
32549 assert!(info.stage.is_empty(), "the persisted snapshot should have been decoded before commit");
32550 drop(pool_meta);
32551 assert!(store.decommission_cancelers.read().await[0].is_none());
32552 assert!(!canceler.is_active());
32553 assert!(canceler.is_cancelled());
32554 }
32555
32556 #[tokio::test]
32557 async fn test_decommission_cancel_transaction_survives_caller_abort_after_durable_save() {
32558 let generation = OffsetDateTime::UNIX_EPOCH;
32559 let canceler = DecommissionCanceler::new(CancellationToken::new());
32560 let pool_meta = PoolMeta {
32561 version: super::POOL_META_VERSION,
32562 pools: vec![decommission_test_pool_status(
32563 0,
32564 Some(PoolDecommissionInfo {
32565 start_time: Some(generation),
32566 ..Default::default()
32567 }),
32568 )],
32569 ..Default::default()
32570 };
32571 let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
32572 let persisted = Arc::new(std::sync::Mutex::new(None));
32573 let (durable_tx, durable_rx) = tokio::sync::oneshot::channel();
32574 let save_release = Arc::new(tokio::sync::Notify::new());
32575 let cancel = tokio::spawn({
32576 let store = store.clone();
32577 let canceler = canceler.clone();
32578 let persisted = persisted.clone();
32579 let save_release = save_release.clone();
32580 async move {
32581 store
32582 .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move {
32583 assert!(
32584 snapshot.pools[0]
32585 .decommission
32586 .as_ref()
32587 .is_some_and(|info| info.canceled && info.start_time.is_none())
32588 );
32589 *persisted.lock().expect("persisted cancel lock should not be poisoned") =
32590 Some(snapshot.encode_config_data()?);
32591 durable_tx
32592 .send(())
32593 .map_err(|_| Error::other("failed to report durable cancel snapshot"))?;
32594 save_release.notified().await;
32595 Ok(())
32596 })
32597 .await
32598 }
32599 });
32600 durable_rx.await.expect("save hook should report the durable cancel snapshot");
32601 cancel.abort();
32602 let join_err = cancel.await.expect_err("caller cancel future should be aborted");
32603 assert!(join_err.is_cancelled());
32604 assert!(
32605 store.pool_meta.try_read().is_err(),
32606 "the detached transaction must retain its state guard"
32607 );
32608 assert!(
32609 store.decommission_cancelers.try_read().is_err(),
32610 "the detached transaction must retain its owner guard"
32611 );
32612
32613 save_release.notify_one();
32614 tokio::time::timeout(StdDuration::from_secs(1), canceler.token().cancelled())
32615 .await
32616 .expect("detached cancel transaction should terminate the old token");
32617
32618 let persisted = persisted
32619 .lock()
32620 .expect("persisted cancel lock should not be poisoned")
32621 .take()
32622 .expect("save should capture the durable cancel snapshot");
32623 let mut durable = PoolMeta::default();
32624 durable
32625 .load_from_config_data(persisted)
32626 .expect("durable cancel snapshot should decode");
32627 assert!(
32628 durable.pools[0]
32629 .decommission
32630 .as_ref()
32631 .is_some_and(|info| info.canceled && info.start_time.is_none())
32632 );
32633 assert!(store.decommission_cancelers.read().await[0].is_none());
32634 assert!(!canceler.is_active());
32635 assert!(canceler.is_cancelled());
32636
32637 let side_effect_ran = Arc::new(AtomicBool::new(false));
32638 let operation_gate = store.ctx.data_movement_operation_gate();
32639 let result = run_decommission_side_effect(canceler.token(), &operation_gate, {
32640 let side_effect_ran = side_effect_ran.clone();
32641 move || async move {
32642 side_effect_ran.store(true, Ordering::SeqCst);
32643 Ok(())
32644 }
32645 })
32646 .await;
32647 assert!(matches!(result, Err(Error::OperationCanceled)));
32648 assert!(!side_effect_ran.load(Ordering::SeqCst));
32649 assert!(
32650 store.pool_meta.read().await.pools[0]
32651 .decommission
32652 .as_ref()
32653 .is_some_and(|info| info.canceled && !info.failed)
32654 );
32655 }
32656
32657 #[tokio::test]
32658 async fn test_decommission_cancel_persists_before_signaling_and_quiesces_before_return() {
32659 let generation = OffsetDateTime::UNIX_EPOCH;
32660 let canceler = DecommissionCanceler::new(CancellationToken::new());
32661 let pool_meta = PoolMeta {
32662 pools: vec![decommission_test_pool_status(
32663 0,
32664 Some(PoolDecommissionInfo {
32665 start_time: Some(generation),
32666 ..Default::default()
32667 }),
32668 )],
32669 ..Default::default()
32670 };
32671 let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
32672 let operation_gate = store.ctx.data_movement_operation_gate();
32673 let side_effect = operation_gate.read().await;
32674 let save_started = Arc::new(tokio::sync::Notify::new());
32675 let save_release = Arc::new(tokio::sync::Notify::new());
32676 let save_entered = Arc::new(AtomicBool::new(false));
32677 let persisted = Arc::new(std::sync::Mutex::new(None));
32678
32679 let mut cancel = tokio::spawn({
32680 let store = store.clone();
32681 let canceler = canceler.clone();
32682 let save_started = save_started.clone();
32683 let save_release = save_release.clone();
32684 let save_entered = save_entered.clone();
32685 let persisted = persisted.clone();
32686 async move {
32687 store
32688 .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move {
32689 *persisted.lock().expect("persisted cancel lock should not be poisoned") =
32690 Some(snapshot.encode_config_data()?);
32691 save_entered.store(true, Ordering::SeqCst);
32692 save_started.notify_one();
32693 save_release.notified().await;
32694 Ok(())
32695 })
32696 .await
32697 }
32698 });
32699
32700 tokio::time::timeout(StdDuration::from_secs(1), save_started.notified())
32701 .await
32702 .expect("cancel should persist while the side effect is still in flight");
32703 assert!(save_entered.load(Ordering::SeqCst));
32704 assert!(!cancel.is_finished(), "cancel must wait for the injected save");
32705 assert!(
32706 store.pool_meta_save_gate.try_lock().is_err(),
32707 "the cancel save must exclude stale full-document saves until publication"
32708 );
32709 assert!(
32710 store.pool_meta.try_read().is_err(),
32711 "the active generation must stay write-locked through persistence"
32712 );
32713 assert!(!canceler.is_cancelled(), "the token must remain live until persistence commits");
32714
32715 let mut fail = tokio::spawn({
32716 let store = store.clone();
32717 async move { store.decommission_failed(0).await }
32718 });
32719 tokio::task::yield_now().await;
32720 assert!(!fail.is_finished(), "fail must serialize behind the pending cancel");
32721
32722 save_release.notify_one();
32723 tokio::time::timeout(StdDuration::from_secs(1), canceler.token().cancelled())
32724 .await
32725 .expect("the durable cancel must signal the active worker");
32726 assert!(!cancel.is_finished(), "cancel must wait for in-flight movement after the durable signal");
32727 let canceled_at = {
32728 let pool_meta = store.pool_meta.read().await;
32729 let info = pool_meta.pools[0]
32730 .decommission
32731 .as_ref()
32732 .expect("the durable cancel must be published before quiescence");
32733 assert!(info.canceled);
32734 assert!(!info.complete);
32735 assert!(!info.failed);
32736 assert!(info.start_time.is_none());
32737 pool_meta.pools[0].last_update
32738 };
32739 let expected_generation = crate::store::scanner_data_movement_timestamp_generation(canceled_at);
32740 let scanner_status = store.scanner_data_movement_pause_snapshot_for_test().await;
32741 assert_eq!(scanner_status.movement_generation, expected_generation);
32742
32743 drop(side_effect);
32744 tokio::time::timeout(StdDuration::from_secs(1), &mut cancel)
32745 .await
32746 .expect("cancel should finish after in-flight movement quiesces")
32747 .expect("cancel task should not panic")
32748 .expect("cancel should commit");
32749 tokio::time::timeout(StdDuration::from_secs(1), &mut fail)
32750 .await
32751 .expect("fail should finish after cancel commits")
32752 .expect("fail task should not panic")
32753 .expect("stale fail should be a no-op");
32754
32755 let pool_meta = store.pool_meta.read().await;
32756 let info = pool_meta.pools[0]
32757 .decommission
32758 .as_ref()
32759 .expect("cancel metadata should remain present");
32760 assert!(info.canceled);
32761 assert!(!info.complete);
32762 assert!(!info.failed);
32763 assert!(info.start_time.is_none());
32764 drop(pool_meta);
32765 assert!(canceler.is_cancelled());
32766 assert!(!canceler.is_active());
32767 assert!(store.decommission_cancelers.read().await[0].is_none());
32768 assert_eq!(store.scanner_data_movement_generation(), expected_generation);
32769
32770 let persisted = persisted
32771 .lock()
32772 .expect("persisted cancel lock should not be poisoned")
32773 .take()
32774 .expect("cancel should persist a durable snapshot");
32775 let mut durable = PoolMeta::default();
32776 durable
32777 .load_from_config_data(persisted)
32778 .expect("durable cancel snapshot should decode after restart");
32779 let restarted = decommission_worker_test_store(durable.clone(), Vec::new());
32780 let restarted_status = restarted.scanner_data_movement_pause_status().await;
32781 assert_eq!(restarted_status.movement_generation, expected_generation);
32782
32783 assert!(
32784 durable
32785 .clear_decommission_at_for_test(0, canceled_at, None)
32786 .expect("same-tick clear should succeed")
32787 );
32788 assert!(durable.pools[0].last_update > canceled_at);
32789 }
32790
32791 #[test]
32792 fn test_decommission_cancel_commit_rejects_a_replaced_generation() {
32793 let old_generation = OffsetDateTime::UNIX_EPOCH;
32794 let new_generation = old_generation + Duration::seconds(1);
32795 let mut canceled = PoolMeta {
32796 pools: vec![decommission_test_pool_status(
32797 0,
32798 Some(PoolDecommissionInfo {
32799 start_time: Some(old_generation),
32800 ..Default::default()
32801 }),
32802 )],
32803 ..Default::default()
32804 };
32805 let previous_last_update = canceled.pools[0].last_update;
32806 assert!(canceled.decommission_cancel(0));
32807 let canceled_pool = canceled.pools.remove(0);
32808 let commit = super::DecommissionCancelCommit {
32809 previous_start_time: Some(old_generation),
32810 previous_queued: false,
32811 previous_last_update,
32812 canceled_pool,
32813 };
32814 let mut current = PoolMeta {
32815 pools: vec![decommission_test_pool_status(
32816 0,
32817 Some(PoolDecommissionInfo {
32818 start_time: Some(new_generation),
32819 ..Default::default()
32820 }),
32821 )],
32822 ..Default::default()
32823 };
32824
32825 let err = super::commit_decommission_cancel(&mut current, 0, commit)
32826 .expect_err("an old cancel must not publish over a replacement generation");
32827 assert!(err.to_string().contains("operation generation changed"));
32828 let info = current.pools[0]
32829 .decommission
32830 .as_ref()
32831 .expect("replacement generation should remain present");
32832 assert_eq!(info.start_time, Some(new_generation));
32833 assert!(!info.canceled);
32834 }
32835
32836 #[tokio::test]
32837 async fn test_decommission_cancel_rejects_stale_retry_after_queued_replacement() {
32838 let old_generation = OffsetDateTime::UNIX_EPOCH;
32839 let canceler = DecommissionCanceler::new(CancellationToken::new());
32840 let pool_meta = PoolMeta {
32841 pools: vec![decommission_test_pool_status(
32842 0,
32843 Some(PoolDecommissionInfo {
32844 start_time: Some(old_generation),
32845 ..Default::default()
32846 }),
32847 )],
32848 ..Default::default()
32849 };
32850 let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
32851 let queued_replacement = Arc::new(std::sync::Mutex::new(None));
32852 let queued_replacement_for_save = queued_replacement.clone();
32853
32854 store
32855 .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move {
32856 let saved_cancel = snapshot.pools[0]
32857 .decommission
32858 .as_ref()
32859 .expect("saved snapshot should contain decommission metadata");
32860 assert!(saved_cancel.canceled);
32861 assert!(saved_cancel.start_time.is_none());
32862 let mut queued_replacement = snapshot.clone();
32863 let replacement = queued_replacement
32864 .pools
32865 .get_mut(0)
32866 .expect("cancel snapshot should contain the pool");
32867 replacement.last_update += Duration::seconds(1);
32868 let info = replacement
32869 .decommission
32870 .as_mut()
32871 .expect("cancel snapshot should contain decommission metadata");
32872 info.canceled = false;
32873 info.queued = true;
32874 *queued_replacement_for_save
32875 .lock()
32876 .expect("queued replacement lock should not be poisoned") = Some(queued_replacement);
32877 Ok(())
32878 })
32879 .await
32880 .expect("cancel should commit before a queued replacement is installed");
32881 assert!(store.decommission_cancelers.read().await[0].is_none());
32882 assert!(!canceler.is_active());
32883 assert!(canceler.is_cancelled());
32884
32885 let queued_replacement = queued_replacement
32886 .lock()
32887 .expect("queued replacement lock should not be poisoned")
32888 .take()
32889 .expect("save should prepare the queued replacement");
32890 *store.pool_meta.write().await = queued_replacement;
32891
32892 let replacement_revision = {
32893 let pool_meta = store.pool_meta.read().await;
32894 let info = pool_meta.pools[0]
32895 .decommission
32896 .as_ref()
32897 .expect("queued replacement should remain present");
32898 assert!(info.queued);
32899 assert!(!info.canceled);
32900 assert!(info.start_time.is_none());
32901 pool_meta.pools[0].last_update
32902 };
32903
32904 store.retry_decommission_cancel_for_operation(0, &canceler).await;
32905
32906 let pool_meta = store.pool_meta.read().await;
32907 let info = pool_meta.pools[0]
32908 .decommission
32909 .as_ref()
32910 .expect("stale retry must preserve the queued replacement");
32911 assert_eq!(pool_meta.pools[0].last_update, replacement_revision);
32912 assert!(info.queued);
32913 assert!(!info.canceled);
32914 assert!(info.start_time.is_none());
32915 }
32916
32917 #[tokio::test]
32918 async fn test_decommission_failed_save_failure_preserves_owner_until_retry_succeeds() {
32919 let canceler = DecommissionCanceler::new(CancellationToken::new());
32920 let pool_meta = PoolMeta {
32921 pools: vec![decommission_test_pool_status(
32922 0,
32923 Some(PoolDecommissionInfo {
32924 start_time: Some(OffsetDateTime::UNIX_EPOCH),
32925 ..Default::default()
32926 }),
32927 )],
32928 ..Default::default()
32929 };
32930 let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
32931
32932 store
32933 .decommission_failed_with_owner_and_save(0, Some(&canceler), async { Err(Error::SlowDown) })
32934 .await
32935 .expect_err("injected terminal save failure should be returned");
32936
32937 {
32938 let cancelers = store.decommission_cancelers.read().await;
32939 let current = cancelers[0].as_ref().expect("failed save must retain the exact owner slot");
32940 assert!(current.owns_same_operation(&canceler));
32941 assert!(current.is_active());
32942 }
32943 {
32944 let pool_meta = store.pool_meta.read().await;
32945 let info = pool_meta.pools[0]
32946 .decommission
32947 .as_ref()
32948 .expect("rollback must retain active decommission metadata");
32949 assert!(info.has_decommission_state());
32950 assert!(!info.failed);
32951 assert!(!info.complete);
32952 assert!(!info.canceled);
32953 }
32954 assert!(store.decommission_terminal_retryable_for_operation(0, &canceler).await);
32955
32956 store
32957 .decommission_failed_with_owner_and_save(0, Some(&canceler), async { Ok(()) })
32958 .await
32959 .expect("terminal retry should commit");
32960
32961 let pool_meta = store.pool_meta.read().await;
32962 assert!(
32963 pool_meta.pools[0]
32964 .decommission
32965 .as_ref()
32966 .expect("terminal metadata should remain")
32967 .failed
32968 );
32969 drop(pool_meta);
32970 assert!(store.decommission_cancelers.read().await[0].is_none());
32971 assert!(!canceler.is_active());
32972 assert!(canceler.is_cancelled());
32973 assert_eq!(store.ctx.data_movement_operation_epoch(), 1);
32974 }
32975
32976 #[test]
32977 fn test_stale_decommission_operation_cannot_cancel_replacement() {
32978 let stale = DecommissionCanceler::new(CancellationToken::new());
32979 let replacement = DecommissionCanceler::new(CancellationToken::new());
32980 let cancelers = vec![Some(replacement.clone())];
32981 let mut pool_meta = PoolMeta {
32982 pools: vec![decommission_test_pool_status(
32983 0,
32984 Some(PoolDecommissionInfo {
32985 start_time: Some(OffsetDateTime::UNIX_EPOCH),
32986 ..Default::default()
32987 }),
32988 )],
32989 ..Default::default()
32990 };
32991
32992 let changed = update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, 0, Some(&stale), |pool_meta| {
32993 pool_meta.decommission_cancel(0)
32994 });
32995
32996 assert!(changed.is_none());
32997 assert!(
32998 !pool_meta.pools[0]
32999 .decommission
33000 .as_ref()
33001 .expect("replacement metadata should remain")
33002 .canceled
33003 );
33004 assert!(replacement.is_active());
33005 assert!(!replacement.is_cancelled());
33006 assert!(!stale.is_active());
33007 assert!(stale.is_cancelled());
33008 }
33009
33010 #[test]
33011 fn test_ensure_decommission_routines_scheduled_accepts_positive_bound_count() {
33012 assert!(super::ensure_decommission_routines_scheduled(2, 2).is_ok());
33013 }
33014
33015 #[test]
33016 fn test_ensure_decommission_routines_scheduled_rejects_zero_bound_count() {
33017 let err = super::ensure_decommission_routines_scheduled(0, 1).expect_err("zero bound count should be rejected");
33018 assert!(
33019 err.to_string()
33020 .contains("failed to start decommission routines: scheduled 0 of 1 expected workers")
33021 );
33022 }
33023
33024 #[test]
33025 fn test_ensure_decommission_routines_scheduled_rejects_partial_binding() {
33026 let err = super::ensure_decommission_routines_scheduled(1, 2).expect_err("partial binding should be rejected");
33027 assert!(
33028 err.to_string()
33029 .contains("failed to start decommission routines: scheduled 1 of 2 expected workers")
33030 );
33031 }
33032
33033 #[test]
33034 #[cfg(windows)]
33035 fn test_path2_bucket_object_with_base_path_supports_windows_separators() {
33036 let (bucket, object) = super::path2_bucket_object_with_base_path("C:\\data", "C:\\data\\my-bucket\\nested\\object.txt");
33037
33038 assert_eq!(bucket, "my-bucket");
33039 assert_eq!(object, "nested/object.txt");
33040 }
33041}