1use crate::bucket::replication::replication_state_from_filemeta;
16#[cfg(all(test, feature = "test-util"))]
17use crate::bucket::utils::is_meta_bucketname;
18use crate::bucket::versioning::VersioningApi as _;
19use crate::bucket::versioning_sys::BucketVersioningSys;
20use crate::bucket::{
21 lifecycle::{
22 DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
23 ValidatedDurableIlmRecord,
24 bucket_lifecycle_audit::LcEventSrc,
25 bucket_lifecycle_ops::{
26 LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_for_data_movement, apply_expiry_rule_in,
27 eval_action_from_lifecycle, lifecycle_delete_all_versions_blocked_by_replication,
28 },
29 classify_durable_ilm_record, get_expiry_configs,
30 lifecycle::IlmAction,
31 tier_delete_journal::durable_ilm_v6_topology_generation,
32 validate_durable_ilm_record,
33 },
34 metadata_sys,
35};
36use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
37use crate::config::com::{
38 CONFIG_PREFIX, delete_config, read_config_limited_preserve_empty, read_config_limited_preserve_empty_with_metadata,
39 read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty, save_config_with_opts,
40 save_config_with_opts_and_metadata,
41};
42use crate::data_movement;
43use crate::data_movement::backpressure::{self, DataMovementOperation};
44use crate::data_usage::DATA_USAGE_CACHE_NAME;
45use crate::disk::error::DiskError;
46use crate::disk::{BUCKET_META_PREFIX, DiskAPI, RUSTFS_META_BUCKET};
47use crate::error::{Error, Result};
48use crate::error::{
49 StorageError, is_err_bucket_exists, is_err_bucket_not_found, is_err_object_not_found, is_err_operation_canceled,
50 is_err_version_not_found,
51};
52use crate::layout::endpoints::EndpointServerPools;
53use crate::object_api::{DecommissionCapacityOptions, GetObjectReader, ObjectInfo, ObjectOptions};
54use crate::runtime::sources as runtime_sources;
55use crate::services::notification_sys::{
56 acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches, tier_delete_journal_topology_generation,
57};
58use crate::services::rebalance::{REBAL_META_NAME, RebalanceMeta, is_rebalance_conflicting_with_decommission};
59use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
60use crate::storage_api_contracts::{
61 admin::StorageAdminApi,
62 bucket::{BucketOperations, BucketOptions, MakeBucketOptions},
63 heal::HealOperations as _,
64 list::ListOperations as _,
65 namespace::NamespaceLocking as _,
66 object::{EcstoreObjectIO, HTTPPreconditions, ObjectIO as _, ObjectOperations as _},
67};
68use crate::store::PoolMetaWriteGateStatus;
69use crate::{core::sets::Sets, store::ECStore};
70use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
71use futures::{
72 StreamExt,
73 future::{BoxFuture, join_all},
74 stream::FuturesUnordered,
75};
76use http::HeaderMap;
77#[cfg(test)]
78use rmp_serde::Deserializer;
79use rmp_serde::Serializer;
80use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
81use rustfs_heal_contracts::heal_channel::HealOpts;
82use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
83use rustfs_utils::path::{
84 decode_dir_object, encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path,
85};
86use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ReplicationConfiguration};
87use serde::{Deserialize, Serialize};
88use sha2::{Digest, Sha256};
89use std::collections::{HashMap, HashSet};
90use std::fmt::Display;
91#[cfg(test)]
92use std::io::Cursor;
93use std::io::Write;
94use std::path::PathBuf;
95use std::sync::{
96 Arc,
97 atomic::{AtomicBool, AtomicUsize, Ordering},
98};
99use time::{Duration, OffsetDateTime};
100use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc};
101use tokio_util::sync::CancellationToken;
102use tracing::{debug, error, info, warn};
103
104const LOG_COMPONENT_ECSTORE: &str = "ecstore";
105const LOG_SUBSYSTEM_POOLS: &str = "pools";
106const EVENT_DECOMMISSION_STATE: &str = "decommission_state";
107const EVENT_DECOMMISSION_BUCKET: &str = "decommission_bucket";
108const EVENT_DECOMMISSION_ENTRY: &str = "decommission_entry";
109const POOL_ACTIVATION_FLEET_PROOF_REQUIRED: &str = "pool activation requires a live fleet capability proof";
110const POOL_ACTIVATION_FLEET_PROOF_EXPIRED: &str = "pool activation fleet capability proof expired before commit";
111const DECOMMISSION_TARGET_FLEET_PROOF_EXPIRED: &str =
112 "decommission target fence fleet capability proof expired before reservation commit";
113const DECOMMISSION_STAGE_MIGRATE_OBJECT: &str = "migrate_object";
114const DECOMMISSION_STAGE_CLEANUP_PREFLIGHT: &str = "cleanup_preflight";
115const DECOMMISSION_STAGE_SOURCE_CLEANUP: &str = "source_cleanup";
116const DECOMMISSION_STAGE_ENTRY_FINISHED: &str = "entry_finished";
117const DECOMMISSION_PROGRESS_SAVE_INTERVAL: Duration = Duration::seconds(30);
118const DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD: usize = 1000;
119const DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF: Duration = Duration::seconds(1);
120const DECOMMISSION_BUCKET_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_BUCKET_CONCURRENCY";
121const DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP: usize = 4;
122const DECOMMISSION_ENTRY_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_ENTRY_CONCURRENCY";
123const DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP: usize = 8;
124const DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP: usize = 64;
125const DECOMMISSION_ENTRY_WORKERS_PER_SET: usize = 2;
126const DECOMMISSION_META_PREFIXES: [&str; 3] = [CONFIG_PREFIX, BUCKET_META_PREFIX, ILM_META_PREFIX];
127const DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION: u16 = 1;
128const DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION: u16 = 2;
129#[cfg(test)]
130const DECOMMISSION_CAPACITY_MODEL_VERSION: u16 = DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION;
131const DECOMMISSION_CAPACITY_TEMPORARY_COPIES: usize = 1;
132const DECOMMISSION_CAPACITY_RESERVATION_TTL: Duration = Duration::minutes(10);
133const DECOMMISSION_CAPACITY_RELEASE_CANCELED: &str = "canceled";
134const DECOMMISSION_CAPACITY_RELEASE_FAILED: &str = "failed";
135const DECOMMISSION_CAPACITY_RELEASE_COMPLETED: &str = "completed";
136pub(crate) const DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX: &str = "decommission/capacity-target";
137const DECOMMISSION_CAPACITY_TARGET_LOCK_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(250);
138const DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX: &str = "target pool ";
139const DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX: &str = " target capacity mutation gate is busy";
140const METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL: &str = "rustfs_decommission_capacity_conflicts_total";
141const METRIC_DECOMMISSION_CAPACITY_PREDICTED_BYTES: &str = "rustfs_decommission_capacity_predicted_physical_bytes";
142const METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES: &str = "rustfs_decommission_capacity_reserved_physical_bytes";
143const METRIC_DECOMMISSION_CAPACITY_PREDICTION_ERROR_BYTES: &str = "rustfs_decommission_capacity_prediction_error_bytes";
144const METRIC_DECOMMISSION_CAPACITY_PREDICTION_ABSOLUTE_ERROR_BYTES: &str =
145 "rustfs_decommission_capacity_prediction_absolute_error_bytes";
146const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3;
147const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
148pub(crate) const DECOMMISSION_ENTRY_MAX_ATTEMPTS: usize = 3;
149const DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS: usize = 12;
150const DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(100);
151pub(crate) const DECOMMISSION_VERSION_COPY_ATTEMPTS: usize = 3;
152const DECOMMISSION_COPY_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(50);
153const DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT: usize = 100;
154const DECOMMISSION_TERMINAL_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(1);
155const DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS: usize = 3;
156const DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT: &str = "decommission/ilm-receipts";
157const DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT: &str = "decommission/ilm-manifests";
158const DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA: &str = "v2";
159const DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA: &str = "v1";
160const DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE: usize = 16 * 1024;
161const DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE: usize = 4 * 1024;
162const DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS: usize = 3;
163const DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
166
167pub const POOL_META_NAME: &str = "pool.bin";
168pub(crate) const POOL_META_IDENTITY_NAME: &str = "pool.bin.identity";
169pub const POOL_META_FORMAT: u16 = 1;
170const POOL_META_V1_VERSION: u16 = 1;
171pub const POOL_META_VERSION: u16 = 2;
172const POOL_META_GENERATION_VERSION: u16 = 3;
173const POOL_META_IDENTITY_FORMAT: u16 = 1;
174const POOL_META_IDENTITY_VERSION: u16 = 1;
175const POOL_META_INITIAL_EPOCH: u64 = 1;
176const POOL_META_CAS_MAX_ATTEMPTS: usize = 3;
177const METRIC_POOL_META_STALE_WRITE_REJECTIONS_TOTAL: &str = "rustfs_pool_meta_stale_write_rejections_total";
178
179fn record_pool_meta_stale_write_rejection(reason: &'static str) {
180 metrics::counter!(METRIC_POOL_META_STALE_WRITE_REJECTIONS_TOTAL, "reason" => reason).increment(1);
181}
182
183fn pool_meta_v2_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool {
184 requested && fleet_confirmed
185}
186
187fn pool_meta_v2_writer_enabled() -> bool {
188 pool_meta_v2_writer_enabled_for(
189 rustfs_utils::get_env_bool(rustfs_config::ENV_POOL_META_V2_WRITE, rustfs_config::DEFAULT_POOL_META_V2_WRITE),
190 rustfs_utils::get_env_bool(
191 rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
192 rustfs_config::DEFAULT_POOL_META_V2_FLEET_CONFIRMED,
193 ),
194 )
195}
196
197fn pool_meta_v3_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool {
198 requested && fleet_confirmed
199}
200
201fn pool_meta_v3_writer_enabled() -> bool {
202 pool_meta_v3_writer_enabled_for(
203 rustfs_utils::get_env_bool(rustfs_config::ENV_POOL_META_V3_WRITE, rustfs_config::DEFAULT_POOL_META_V3_WRITE),
204 rustfs_utils::get_env_bool(
205 rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED,
206 rustfs_config::DEFAULT_POOL_META_V3_FLEET_CONFIRMED,
207 ),
208 )
209}
210
211fn decommission_capacity_writer_supported_for(version: u16, v2_writer_enabled: bool, v3_writer_enabled: bool) -> bool {
212 matches!(version, POOL_META_VERSION | POOL_META_GENERATION_VERSION) || v2_writer_enabled || v3_writer_enabled
213}
214
215fn ensure_decommission_ledger_persistence_supported_for(
216 version: u16,
217 v2_writer_enabled: bool,
218 v3_writer_enabled: bool,
219) -> Result<()> {
220 if decommission_capacity_writer_supported_for(version, v2_writer_enabled, v3_writer_enabled) {
221 return Ok(());
222 }
223
224 Err(Error::InvalidArgument(
225 "decommission".to_string(),
226 "pool-metadata-version".to_string(),
227 format!(
228 "durable unresolved-entry recovery requires pool metadata V2 or V3; enable either the {} + {} V2 gate or the {} + {} V3 gate only after every reader and writer supports that format",
229 rustfs_config::ENV_POOL_META_V2_WRITE,
230 rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
231 rustfs_config::ENV_POOL_META_V3_WRITE,
232 rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED,
233 ),
234 ))
235}
236
237fn ensure_decommission_ledger_persistence_supported(pool_meta: &PoolMeta) -> Result<()> {
238 ensure_decommission_ledger_persistence_supported_for(
239 pool_meta.version,
240 pool_meta_v2_writer_enabled(),
241 pool_meta_v3_writer_enabled(),
242 )
243}
244
245#[derive(Clone, Debug)]
246pub struct DecommissionCanceler {
247 operation: Arc<DecommissionOperation>,
248}
249
250#[derive(Debug)]
251struct DecommissionOperation {
252 token: CancellationToken,
253 active: AtomicBool,
254}
255
256impl DecommissionCanceler {
257 pub(crate) fn new(token: CancellationToken) -> Self {
258 Self {
259 operation: Arc::new(DecommissionOperation {
260 token,
261 active: AtomicBool::new(true),
262 }),
263 }
264 }
265
266 #[cfg(test)]
267 pub(crate) fn new_for_test(token: CancellationToken) -> Self {
268 Self::new(token)
269 }
270
271 fn token(&self) -> &CancellationToken {
272 &self.operation.token
273 }
274
275 pub(crate) fn is_active(&self) -> bool {
276 self.operation.active.load(Ordering::Acquire)
277 }
278
279 #[cfg(test)]
280 fn is_cancelled(&self) -> bool {
281 self.token().is_cancelled()
282 }
283
284 fn cancel(&self) {
285 self.token().cancel();
286 }
287
288 fn release(&self) {
289 self.cancel();
290 self.operation.active.store(false, Ordering::Release);
291 }
292
293 fn owns_same_operation(&self, other: &Self) -> bool {
294 Arc::ptr_eq(&self.operation, &other.operation)
295 }
296}
297
298struct DecommissionCancelerGuard {
299 canceler: DecommissionCanceler,
300}
301
302impl DecommissionCancelerGuard {
303 fn new(canceler: DecommissionCanceler) -> Self {
304 Self { canceler }
305 }
306
307 fn canceler(&self) -> &DecommissionCanceler {
308 &self.canceler
309 }
310}
311
312impl Drop for DecommissionCancelerGuard {
313 fn drop(&mut self) {
314 self.canceler.release();
315 }
316}
317
318fn dedup_indices(indices: &[usize]) -> Vec<usize> {
319 let mut seen = HashSet::with_capacity(indices.len());
320 let mut output = Vec::with_capacity(indices.len());
321 for idx in indices {
322 if seen.insert(*idx) {
323 output.push(*idx);
324 }
325 }
326
327 output
328}
329
330fn bind_decommission_cancelers(
331 indices: &[usize],
332 parent: &CancellationToken,
333 cancelers: &mut [Option<DecommissionCanceler>],
334) -> Vec<(usize, DecommissionCanceler)> {
335 let mut bound = Vec::with_capacity(indices.len());
336
337 for idx in indices {
338 if let Some(slot) = cancelers.get_mut(*idx) {
339 if let Some(existing) = slot.take() {
340 existing.release();
341 }
342 let canceler = DecommissionCanceler::new(parent.child_token());
343 *slot = Some(canceler.clone());
344 bound.push((*idx, canceler));
345 }
346 }
347
348 bound
349}
350
351fn bind_missing_decommission_cancelers(
352 indices: &[usize],
353 parent: &CancellationToken,
354 cancelers: &mut [Option<DecommissionCanceler>],
355) -> Vec<(usize, DecommissionCanceler)> {
356 let mut bound = Vec::with_capacity(indices.len());
357
358 for idx in indices {
359 let Some(slot) = cancelers.get_mut(*idx) else {
360 continue;
361 };
362 if slot.as_ref().is_some_and(DecommissionCanceler::is_active) {
363 break;
364 }
365 if let Some(stale) = slot.take() {
366 stale.release();
367 }
368 let canceler = DecommissionCanceler::new(parent.child_token());
369 *slot = Some(canceler.clone());
370 bound.push((*idx, canceler));
371 }
372
373 bound
374}
375
376fn take_decommission_canceler(cancelers: &mut [Option<DecommissionCanceler>], idx: usize) -> Option<DecommissionCanceler> {
377 cancelers.get_mut(idx).and_then(Option::take)
378}
379
380fn take_decommission_canceler_for_operation(
381 cancelers: &mut [Option<DecommissionCanceler>],
382 idx: usize,
383 owner: &DecommissionCanceler,
384) -> Option<DecommissionCanceler> {
385 let slot = cancelers.get_mut(idx)?;
386 if slot.as_ref().is_some_and(|canceler| canceler.owns_same_operation(owner)) {
387 slot.take()
388 } else {
389 None
390 }
391}
392
393fn decommission_canceler_is_owned_by(
394 cancelers: &[Option<DecommissionCanceler>],
395 idx: usize,
396 owner: &DecommissionCanceler,
397) -> bool {
398 cancelers
399 .get(idx)
400 .and_then(Option::as_ref)
401 .is_some_and(|canceler| canceler.owns_same_operation(owner))
402}
403
404fn update_decommission_for_operation<T>(
405 cancelers: &[Option<DecommissionCanceler>],
406 pool_meta: &mut PoolMeta,
407 idx: usize,
408 owner: Option<&DecommissionCanceler>,
409 update: impl FnOnce(&mut PoolMeta) -> T,
410) -> Option<T> {
411 if let Some(owner) = owner
412 && !decommission_canceler_is_owned_by(cancelers, idx, owner)
413 {
414 owner.release();
415 return None;
416 }
417
418 Some(update(pool_meta))
419}
420
421fn has_active_decommission_canceler(cancelers: &[Option<DecommissionCanceler>]) -> bool {
422 cancelers.iter().flatten().any(DecommissionCanceler::is_active)
423}
424
425fn cancel_decommission_canceler(canceler: Option<DecommissionCanceler>) -> bool {
426 if let Some(canceler) = canceler {
427 canceler.release();
428 true
429 } else {
430 false
431 }
432}
433
434fn take_and_cancel_decommission_canceler(cancelers: &mut [Option<DecommissionCanceler>], idx: usize) -> bool {
435 let canceler = take_decommission_canceler(cancelers, idx);
436 cancel_decommission_canceler(canceler)
437}
438
439fn take_and_cancel_decommission_canceler_for_operation(
440 cancelers: &mut [Option<DecommissionCanceler>],
441 idx: usize,
442 owner: &DecommissionCanceler,
443) -> bool {
444 let canceler = take_decommission_canceler_for_operation(cancelers, idx, owner);
445 if canceler.is_none() {
446 owner.release();
447 return false;
448 }
449 cancel_decommission_canceler(canceler)
450}
451
452fn ensure_decommission_routines_scheduled(bound_count: usize, expected_count: usize) -> Result<()> {
453 if bound_count == 0 || bound_count != expected_count {
454 return Err(Error::other(format!(
455 "failed to start decommission routines: scheduled {bound_count} of {expected_count} expected workers"
456 )));
457 }
458
459 Ok(())
460}
461
462fn guard_decommission_cancelers(index_cancelers: Vec<(usize, DecommissionCanceler)>) -> Vec<(usize, DecommissionCancelerGuard)> {
463 index_cancelers
464 .into_iter()
465 .map(|(idx, canceler)| (idx, DecommissionCancelerGuard::new(canceler)))
466 .collect()
467}
468
469async fn await_decommission_worker(idx: usize, worker: tokio::task::JoinHandle<Result<()>>) -> Result<()> {
470 worker
471 .await
472 .map_err(|err| Error::other(format!("decommission worker {idx} task join error: {err}")))?
473}
474
475fn reserve_decommission_start_cancelers(
476 pool_meta: &PoolMeta,
477 indices: &[usize],
478 local_indices: &[usize],
479 parent: &CancellationToken,
480 cancelers: &mut [Option<DecommissionCanceler>],
481) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
482 ensure_decommission_start_pool_states(pool_meta, indices)?;
483 if local_indices.is_empty() {
484 return Ok(Vec::new());
485 }
486 let bound = bind_decommission_cancelers(local_indices, parent, cancelers);
487 let guards = guard_decommission_cancelers(bound);
488 ensure_decommission_routines_scheduled(guards.len(), local_indices.len())?;
489 Ok(guards)
490}
491
492fn default_decommission_bucket_concurrency(cpu_count: usize) -> usize {
493 cpu_count.clamp(1, DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP)
494}
495
496fn decommission_bucket_concurrency_limit() -> usize {
497 let default_limit = default_decommission_bucket_concurrency(num_cpus::get());
498 rustfs_utils::get_env_usize(DECOMMISSION_BUCKET_CONCURRENCY_ENV, default_limit).max(1)
499}
500
501fn default_decommission_entry_concurrency(cpu_count: usize) -> usize {
502 cpu_count.clamp(1, DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP)
503}
504
505fn clamp_decommission_entry_concurrency(limit: usize) -> usize {
506 limit.clamp(1, DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP)
507}
508
509fn decommission_entry_concurrency_limit() -> usize {
510 let default_limit = default_decommission_entry_concurrency(num_cpus::get());
511 clamp_decommission_entry_concurrency(rustfs_utils::get_env_usize(DECOMMISSION_ENTRY_CONCURRENCY_ENV, default_limit))
512}
513
514fn is_decommission_meta_bucket(bucket: &DecomBucketInfo) -> bool {
515 bucket.name == RUSTFS_META_BUCKET
516}
517
518fn decommission_meta_buckets() -> [DecomBucketInfo; DECOMMISSION_META_PREFIXES.len()] {
519 DECOMMISSION_META_PREFIXES.map(|prefix| DecomBucketInfo {
520 name: RUSTFS_META_BUCKET.to_owned(),
521 prefix: prefix.to_owned(),
522 })
523}
524
525fn reconcile_decommission_meta_buckets(meta: &mut PoolMeta, idx: usize) -> bool {
526 let before = meta.pending_buckets(idx).len();
527 meta.queue_buckets(idx, decommission_meta_buckets().into());
528 meta.pending_buckets(idx).len() != before
529}
530
531fn split_decommission_buckets(buckets: Vec<DecomBucketInfo>) -> (Vec<DecomBucketInfo>, Vec<DecomBucketInfo>) {
532 let mut regular = Vec::with_capacity(buckets.len());
533 let mut meta = Vec::new();
534
535 for bucket in buckets {
536 if is_decommission_meta_bucket(&bucket) {
537 meta.push(bucket);
538 } else {
539 regular.push(bucket);
540 }
541 }
542
543 regular.shrink_to_fit();
544 (regular, meta)
545}
546
547fn ensure_decommission_not_rebalancing(rebalance_running: bool) -> Result<()> {
548 if rebalance_running {
549 return Err(Error::RebalanceAlreadyRunning);
550 }
551
552 Ok(())
553}
554
555fn ensure_decommission_start_rebalance_meta_allowed(meta: Option<&RebalanceMeta>) -> Result<()> {
556 ensure_decommission_not_rebalancing(meta.is_some_and(is_rebalance_conflicting_with_decommission))
557}
558
559#[allow(dead_code, reason = "leader precondition asserted by this file's tests (backlog#1823)")]
560fn ensure_local_decommission_pool_leaders(endpoints: &EndpointServerPools, indices: &[usize]) -> Result<()> {
561 for idx in indices {
562 ensure_local_decommission_pool_leader(endpoints, *idx)?;
563 }
564
565 Ok(())
566}
567
568fn ensure_local_decommission_pool_leader(endpoints: &EndpointServerPools, idx: usize) -> Result<()> {
569 let pool = endpoints
570 .as_ref()
571 .get(idx)
572 .ok_or_else(|| invalid_decommission_pool_index_error(endpoints.as_ref().len(), idx))?;
573 let endpoint = pool
574 .endpoints
575 .as_ref()
576 .first()
577 .ok_or_else(|| Error::other(format!("decommission pool {idx} has no configured endpoints")))?;
578
579 if !endpoint.is_local {
580 return Err(Error::other(format!(
581 "decommission for pool {idx} must run on the pool first endpoint {endpoint}"
582 )));
583 }
584
585 Ok(())
586}
587
588fn decommission_pool_first_endpoint_is_local(endpoints: &EndpointServerPools, idx: usize) -> Result<bool> {
589 let pool = endpoints
590 .as_ref()
591 .get(idx)
592 .ok_or_else(|| invalid_decommission_pool_index_error(endpoints.as_ref().len(), idx))?;
593 let endpoint = pool
594 .endpoints
595 .as_ref()
596 .first()
597 .ok_or_else(|| Error::other(format!("decommission pool {idx} has no configured endpoints")))?;
598
599 Ok(endpoint.is_local)
600}
601
602pub(crate) fn local_decommission_queue_prefix(endpoints: &EndpointServerPools, indices: &[usize]) -> Result<Vec<usize>> {
603 let mut local = Vec::with_capacity(indices.len());
604
605 for idx in indices {
606 if decommission_pool_first_endpoint_is_local(endpoints, *idx)? {
607 local.push(*idx);
608 } else {
609 break;
610 }
611 }
612
613 Ok(local)
614}
615
616fn resumable_decommission_queue_indices(meta: &PoolMeta) -> Vec<usize> {
617 let mut indices = Vec::new();
618 for (idx, pool) in meta.pools.iter().enumerate() {
619 if let Some(decommission) = &pool.decommission {
620 if !decommission.has_decommission_state() {
621 continue;
622 }
623 if decommission.complete || decommission.failed || decommission.canceled {
624 continue;
625 }
626 indices.push(idx);
627 }
628 }
629
630 indices
631}
632
633fn missing_decommission_worker_prefix(indices: &[usize], cancelers: &[Option<DecommissionCanceler>]) -> Vec<usize> {
634 let mut missing = Vec::with_capacity(indices.len());
635
636 for idx in indices {
637 if cancelers
638 .get(*idx)
639 .and_then(Option::as_ref)
640 .is_some_and(DecommissionCanceler::is_active)
641 {
642 break;
643 }
644 missing.push(*idx);
645 }
646
647 missing
648}
649
650fn ensure_decommission_start_local_leader(endpoints: &EndpointServerPools, indices: &[usize]) -> Result<()> {
651 if let Some(first) = indices.first() {
652 ensure_local_decommission_pool_leader(endpoints, *first)?;
653 }
654
655 Ok(())
656}
657
658fn build_decommission_start_state(
659 pi: PoolSpaceInfo,
660 queued: bool,
661 now: OffsetDateTime,
662 previous: Option<&PoolDecommissionInfo>,
663) -> PoolDecommissionInfo {
664 let mut info = PoolDecommissionInfo {
665 start_time: if queued { None } else { Some(now) },
666 start_size: pi.free,
667 total_size: pi.total,
668 current_size: pi.free,
669 queued,
670 ..Default::default()
671 };
672
673 if let Some(previous) = previous
674 && (previous.failed || previous.canceled)
675 {
676 info.decommissioned_buckets = previous.decommissioned_buckets.clone();
677 info.items_decommissioned = previous.items_decommissioned;
678 info.bytes_done = previous.bytes_done;
679 info.unresolved_entries = previous.unresolved_entries.clone();
680 if let Some(generation) = info.start_time {
681 for entry in &mut info.unresolved_entries {
682 entry.source_generation = generation;
683 }
684 }
685 info.mark_progress_saved();
686 }
687
688 info
689}
690
691fn spawn_decommission_index_cancelers(
692 store: Arc<ECStore>,
693 rx: CancellationToken,
694 index_cancelers: Vec<(usize, DecommissionCancelerGuard)>,
695 entry_budget: Arc<Semaphore>,
696) -> tokio::task::JoinHandle<()> {
697 tokio::spawn(async move {
698 let mut stop_queue = false;
699
700 for (idx, canceler_guard) in index_cancelers {
701 let canceler = canceler_guard.canceler().clone();
702 if stop_queue || rx.is_cancelled() {
703 canceler.cancel();
704 store.retry_decommission_cancel_for_operation(idx, &canceler).await;
705 continue;
706 }
707
708 let worker = tokio::spawn({
709 let store = store.clone();
710 let canceler = canceler.clone();
711 let entry_budget = entry_budget.clone();
712 async move { store.do_decommission_in_routine(canceler, idx, entry_budget).await }
713 });
714 if let Err(err) = await_decommission_worker(idx, worker).await {
715 if is_decommission_capacity_blocked_error(&err) || is_decommission_target_capacity_error(&err) {
716 warn!(
717 event = EVENT_DECOMMISSION_STATE,
718 component = LOG_COMPONENT_ECSTORE,
719 subsystem = LOG_SUBSYSTEM_POOLS,
720 pool_index = idx,
721 state = "capacity_blocked",
722 error = %err,
723 "Decommission routine paused without a terminal transition"
724 );
725 store.release_decommission_canceler_slot(idx, &canceler).await;
726 stop_queue = true;
727 continue;
728 }
729 if let Err(blocked) = store
730 .ensure_pool_meta_side_effects_safe("decommission paused because pool metadata requires recovery")
731 .await
732 {
733 warn!(
734 event = EVENT_DECOMMISSION_STATE,
735 component = LOG_COMPONENT_ECSTORE,
736 subsystem = LOG_SUBSYSTEM_POOLS,
737 pool_index = idx,
738 state = "routine_blocked",
739 error = %blocked,
740 "Decommission routine paused without changing terminal state"
741 );
742 store.release_decommission_canceler_slot(idx, &canceler).await;
743 stop_queue = true;
744 continue;
745 }
746 error!(
747 event = EVENT_DECOMMISSION_STATE,
748 component = LOG_COMPONENT_ECSTORE,
749 subsystem = LOG_SUBSYSTEM_POOLS,
750 pool_index = idx,
751 state = "routine_failed",
752 error = %err,
753 "Decommission routine failed"
754 );
755 store.quiesce_decommission_worker_after_join_error(&canceler).await;
756 store.retry_decommission_failed_for_operation(idx, &canceler).await;
757 stop_queue = true;
758 continue;
759 }
760
761 stop_queue = {
762 let pool_meta = store.pool_meta.read().await;
763 !should_continue_decommission_queue(&pool_meta, idx)
764 };
765 }
766 })
767}
768
769fn decommission_meta_bucket_options() -> MakeBucketOptions {
770 MakeBucketOptions {
771 force_create: true,
772 ..Default::default()
773 }
774}
775
776fn is_decommission_active(complete: bool, failed: bool, canceled: bool) -> bool {
777 !complete && !failed && !canceled
778}
779
780pub(crate) fn pool_meta_has_active_decommission(meta: &PoolMeta) -> bool {
781 meta.pools.iter().any(|pool| {
782 pool.decommission.as_ref().is_some_and(|info| {
783 info.has_decommission_state() && is_decommission_active(info.complete, info.failed, info.canceled)
784 })
785 })
786}
787
788fn is_decommission_suspended(info: &PoolDecommissionInfo) -> bool {
789 info.has_decommission_state()
790}
791
792fn validate_decommission_terminal_state(complete: bool, failed: bool, canceled: bool) -> Result<()> {
793 let terminal_count = [complete, failed, canceled].into_iter().filter(|terminal| *terminal).count();
794 if terminal_count > 1 {
795 return Err(Error::other(format!(
796 "pool metadata load failed: invalid decommission terminal state complete={complete} failed={failed} canceled={canceled}"
797 )));
798 }
799 Ok(())
800}
801
802fn invalid_decommission_pool_index_error(pool_count: usize, idx: usize) -> Error {
803 Error::other(format!("invalid decommission pool index {idx} for {pool_count} pools"))
804}
805
806#[derive(Debug, Clone, Copy, PartialEq, Eq)]
807enum DecommissionStartPoolState {
808 Missing,
809 Active,
810 Retryable,
811 Decommissioning,
812 Decommissioned,
813 Blocked,
814}
815
816fn decommission_start_pool_state(pool: Option<&PoolStatus>) -> DecommissionStartPoolState {
817 let Some(pool) = pool else {
818 return DecommissionStartPoolState::Missing;
819 };
820 let Some(info) = pool.decommission.as_ref() else {
821 return DecommissionStartPoolState::Active;
822 };
823 if !info.has_decommission_state() {
824 return DecommissionStartPoolState::Active;
825 }
826
827 if info.complete {
828 DecommissionStartPoolState::Decommissioned
829 } else if info.failed || info.canceled {
830 if info.unresolved_entries.is_empty() {
831 DecommissionStartPoolState::Blocked
832 } else {
833 DecommissionStartPoolState::Retryable
834 }
835 } else {
836 DecommissionStartPoolState::Decommissioning
837 }
838}
839
840fn is_decommission_start_active_pool(pool: &PoolStatus) -> bool {
841 decommission_start_pool_state(Some(pool)) == DecommissionStartPoolState::Active
842}
843
844fn invalid_decommission_request(reason: impl Into<String>) -> Error {
845 Error::InvalidArgument("decommission".to_string(), "pool-state".to_string(), reason.into())
846}
847
848fn ensure_decommission_start_allowed(state: DecommissionStartPoolState) -> Result<()> {
849 match state {
850 DecommissionStartPoolState::Missing => {
851 Err(invalid_decommission_request("failed to start decommission: target pool was not found"))
852 }
853 DecommissionStartPoolState::Active | DecommissionStartPoolState::Retryable => Ok(()),
854 DecommissionStartPoolState::Decommissioning => Err(StorageError::DecommissionAlreadyRunning),
855 DecommissionStartPoolState::Decommissioned => Err(invalid_decommission_request(
856 "failed to start decommission: target pool is already decommissioned",
857 )),
858 DecommissionStartPoolState::Blocked => Err(invalid_decommission_request(
859 "failed to start decommission: target pool decommission is blocked; clear failed or canceled metadata before starting again",
860 )),
861 }
862}
863
864fn ensure_decommission_start_keeps_active_pool(meta: &PoolMeta, indices: &[usize]) -> Result<()> {
865 let active_count = meta
866 .pools
867 .iter()
868 .filter(|pool| is_decommission_start_active_pool(pool))
869 .count();
870 let active_target_count = indices
871 .iter()
872 .filter(|idx| meta.pools.get(**idx).is_some_and(is_decommission_start_active_pool))
873 .count();
874 if active_count.saturating_sub(active_target_count) == 0 {
875 return Err(invalid_decommission_request(
876 "failed to start decommission: at least one active pool must remain after decommission start",
877 ));
878 }
879
880 Ok(())
881}
882
883fn ensure_decommission_start_pool_states(meta: &PoolMeta, indices: &[usize]) -> Result<()> {
884 for idx in indices.iter().copied() {
885 ensure_decommission_start_allowed(decommission_start_pool_state(meta.pools.get(idx)))?;
886 }
887 ensure_decommission_start_keeps_active_pool(meta, indices)
888}
889
890fn capacity_mul_div_ceil(value: usize, multiplier: usize, divisor: usize) -> usize {
891 if value == 0 || multiplier == 0 {
892 return 0;
893 }
894 if divisor == 0 {
895 return usize::MAX;
896 }
897 value
898 .checked_mul(multiplier)
899 .map(|product| product.div_ceil(divisor))
900 .unwrap_or(usize::MAX)
901}
902
903fn capacity_source_data_equivalent(physical_bytes: usize, layout: DecommissionErasureLayout) -> Result<usize> {
904 if !layout.is_valid() {
905 return Err(Error::DecommissionCapacity(format!(
906 "failed to model decommission capacity: invalid source layout data={} parity={}",
907 layout.data, layout.parity
908 )));
909 }
910 Ok(capacity_mul_div_ceil(physical_bytes, layout.data, layout.width()))
911}
912
913fn capacity_target_physical_bytes(data_bytes: usize, layout: DecommissionErasureLayout) -> Result<usize> {
914 if !layout.is_valid() {
915 return Err(Error::DecommissionCapacity(format!(
916 "failed to model decommission capacity: invalid target layout data={} parity={}",
917 layout.data, layout.parity
918 )));
919 }
920 Ok(capacity_mul_div_ceil(data_bytes, layout.width(), layout.data))
921}
922
923fn worst_decommission_target_layout(targets: &[DecommissionPoolCapacityInfo]) -> Result<DecommissionErasureLayout> {
924 targets
925 .iter()
926 .map(|target| target.layout)
927 .filter(|layout| layout.is_valid())
928 .max_by(|left, right| {
929 ((left.width() as u128) * (right.data as u128)).cmp(&((right.width() as u128) * (left.data as u128)))
930 })
931 .ok_or_else(|| Error::other("failed to start decommission: no valid target erasure layout is available"))
932}
933
934fn decommission_capacity_writer_supported(meta: &PoolMeta) -> bool {
935 decommission_capacity_writer_supported_for(meta.version, pool_meta_v2_writer_enabled(), pool_meta_v3_writer_enabled())
936}
937
938fn ensure_decommission_capacity_writer_supported(meta: &PoolMeta) -> Result<()> {
939 if decommission_capacity_writer_supported(meta) {
940 return Ok(());
941 }
942 Err(Error::DecommissionCapacity(format!(
943 "failed to start decommission: durable capacity reservations require pool metadata V2 or V3; enable either the {} + {} V2 gate or the {} + {} V3 gate only after every reader and writer supports that format",
944 rustfs_config::ENV_POOL_META_V2_WRITE,
945 rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
946 rustfs_config::ENV_POOL_META_V3_WRITE,
947 rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED,
948 )))
949}
950
951fn decommission_capacity_blocked_error(message: impl Display) -> Error {
952 Error::DecommissionCapacityBlocked {
953 message: message.to_string(),
954 }
955}
956
957fn is_decommission_capacity_blocked_error(err: &Error) -> bool {
958 if matches!(err, Error::DecommissionCapacityBlocked { .. }) {
959 return true;
960 }
961 data_movement::data_movement_stage_source(err).is_some_and(is_decommission_capacity_blocked_error)
962}
963
964fn is_decommission_capacity_intent_conflict(err: &Error) -> bool {
965 if let Error::DecommissionCapacityBlocked { message } = err {
966 return message.contains("unresolved target capacity intent")
967 || message.contains("pending capacity intent belongs to another mutation");
968 }
969 data_movement::data_movement_stage_source(err).is_some_and(is_decommission_capacity_intent_conflict)
970 || err.to_string().contains("unresolved target capacity intent")
971}
972
973fn decommission_capacity_target_gate_busy_index(err: &Error) -> Option<usize> {
974 if let Error::DecommissionCapacityBlocked { message } = err {
975 return message
976 .strip_prefix(DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX)?
977 .strip_suffix(DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX)?
978 .parse()
979 .ok();
980 }
981 data_movement::data_movement_stage_source(err).and_then(decommission_capacity_target_gate_busy_index)
982}
983
984fn is_decommission_capacity_target_gate_busy(err: &Error) -> bool {
985 decommission_capacity_target_gate_busy_index(err).is_some()
986}
987
988#[derive(Debug, Clone, Copy, PartialEq, Eq)]
989enum DecommissionCapacityRetryKind {
990 IntentConflict,
991}
992
993fn decommission_capacity_retry_kind(err: &Error, intent_conflict_attempt: usize) -> Option<DecommissionCapacityRetryKind> {
994 (intent_conflict_attempt < DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS
995 && is_decommission_capacity_intent_conflict(err))
996 .then_some(DecommissionCapacityRetryKind::IntentConflict)
997}
998
999fn ensure_decommission_capacity_target_fence(
1000 guard: &rustfs_lock::NamespaceLockGuard,
1001 target_pool_index: usize,
1002 phase: &str,
1003) -> Result<()> {
1004 if guard.is_lock_lost() {
1005 return Err(Error::DecommissionCapacity(format!(
1006 "target pool {target_pool_index} capacity mutation fence was lost during {phase}"
1007 )));
1008 }
1009 Ok(())
1010}
1011
1012fn ensure_decommission_capacity_mutation_intent_current(
1013 meta: &PoolMeta,
1014 owner: DecommissionCapacityOwner,
1015 target_pool_index: usize,
1016 expected_target_physical_bytes: usize,
1017 mutation_id: uuid::Uuid,
1018 temporary_release: bool,
1019 model_version: u16,
1020) -> Result<()> {
1021 let reservation = meta
1022 .pools
1023 .get(owner.source_pool_index)
1024 .and_then(|pool| pool.decommission.as_ref())
1025 .and_then(|info| info.capacity_reservation.as_ref())
1026 .filter(|reservation| {
1027 reservation.active()
1032 && reservation.source_pool_index == owner.source_pool_index
1033 && reservation.operation_id == owner.operation_id
1034 && reservation.generation == owner.generation
1035 && reservation.model_version == model_version
1036 })
1037 .ok_or_else(|| {
1038 decommission_capacity_blocked_error("decommission target mutation owner changed before capacity finalize")
1039 })?;
1040 let target = reservation
1041 .targets
1042 .iter()
1043 .find(|target| target.pool_index == target_pool_index)
1044 .ok_or_else(|| decommission_capacity_blocked_error("decommission target allocation changed before capacity finalize"))?;
1045
1046 if temporary_release {
1047 return Ok(());
1052 }
1053
1054 if expected_target_physical_bytes == 0
1055 || (target.pending_mutation_id == Some(mutation_id) && target.pending_physical_bytes >= expected_target_physical_bytes)
1056 {
1057 return Ok(());
1058 }
1059 Err(decommission_capacity_blocked_error(
1060 "pending capacity intent belongs to another mutation before capacity finalize",
1061 ))
1062}
1063
1064fn validate_decommission_capacity_reservation(reservation: Option<&DecommissionCapacityReservation>) -> Result<()> {
1065 let Some(reservation) = reservation else {
1066 return Ok(());
1067 };
1068 if !matches!(
1069 reservation.model_version,
1070 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
1071 ) {
1072 return Err(Error::DecommissionCapacity(format!(
1073 "pool metadata load failed: unsupported decommission capacity model version {}",
1074 reservation.model_version
1075 )));
1076 }
1077 if reservation.operation_id.is_nil() || reservation.generation == 0 || reservation.owner_nonce.is_nil() {
1078 return Err(Error::other(
1079 "pool metadata load failed: decommission capacity reservation identity is invalid",
1080 ));
1081 }
1082 if !reservation.source_layout.is_valid() || reservation.targets.iter().any(|target| !target.layout.is_valid()) {
1083 return Err(Error::other(
1084 "pool metadata load failed: decommission capacity reservation contains an invalid erasure layout",
1085 ));
1086 }
1087 if reservation.temporary_copies != DECOMMISSION_CAPACITY_TEMPORARY_COPIES {
1088 return Err(Error::other(
1089 "pool metadata load failed: decommission capacity reservation temporary-copy model is invalid",
1090 ));
1091 }
1092 if reservation.source_data_equivalent_bytes
1093 != capacity_source_data_equivalent(reservation.source_physical_bytes, reservation.source_layout)?
1094 {
1095 return Err(Error::other(
1096 "pool metadata load failed: decommission capacity reservation source estimate is invalid",
1097 ));
1098 }
1099 if reservation.peak_physical_bytes
1100 != reservation
1101 .predicted_physical_bytes
1102 .saturating_add(reservation.temporary_physical_bytes)
1103 || reservation.temporary_physical_bytes
1104 != reservation
1105 .predicted_physical_bytes
1106 .saturating_mul(reservation.temporary_copies)
1107 {
1108 return Err(Error::other(
1109 "pool metadata load failed: decommission capacity reservation peak does not match its physical model",
1110 ));
1111 }
1112 if reservation.committed_data_bytes > reservation.source_data_equivalent_bytes {
1113 return Err(Error::other(
1114 "pool metadata load failed: decommission capacity committed bytes exceed the source estimate",
1115 ));
1116 }
1117 if reservation.consumed_target_physical_bytes > reservation.predicted_physical_bytes {
1118 return Err(Error::other(
1119 "pool metadata load failed: decommission capacity consumed target bytes exceed the prediction",
1120 ));
1121 }
1122 let mut target_indices = HashSet::with_capacity(reservation.targets.len());
1123 let mut reserved_physical_bytes = 0usize;
1124 let mut consumed_physical_bytes = 0usize;
1125 let mut observed_physical_bytes = 0usize;
1126 let mut inflight_physical_bytes = 0usize;
1127 let mut pending_physical_bytes = 0usize;
1128 for target in &reservation.targets {
1129 let mut temporary_mutation_ids = HashSet::with_capacity(target.temporary_mutations.len());
1130 let temporary_mutation_bytes = target.temporary_mutations.iter().try_fold(0usize, |total, mutation| {
1131 if mutation.mutation_id.is_nil()
1132 || (mutation.physical_bytes == 0 && reservation.model_version != DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION)
1133 || !temporary_mutation_ids.insert(mutation.mutation_id)
1134 {
1135 return Err(Error::other(
1136 "pool metadata load failed: decommission capacity temporary mutation is invalid",
1137 ));
1138 }
1139 Ok(total.saturating_add(mutation.physical_bytes))
1140 })?;
1141 if target.pool_index == reservation.source_pool_index
1142 || !target_indices.insert(target.pool_index)
1143 || target.reserved_physical_bytes == 0
1144 || target.reserved_physical_bytes > target.physical_free_at_reservation
1145 || target.physical_free_at_reservation > target.physical_total_at_reservation
1146 || target.consumed_physical_bytes > reservation.consumed_target_physical_bytes
1147 || target.observed_physical_bytes > reservation.observed_target_physical_bytes
1148 || target.inflight_physical_bytes > target.observed_physical_bytes
1149 || temporary_mutation_bytes > target.inflight_physical_bytes
1150 {
1151 return Err(Error::other(
1152 "pool metadata load failed: decommission capacity reservation target allocation is invalid",
1153 ));
1154 }
1155 if reservation.predicted_physical_bytes
1156 < capacity_target_physical_bytes(reservation.source_data_equivalent_bytes, target.layout)?
1157 {
1158 return Err(Error::other(
1159 "pool metadata load failed: decommission capacity reservation target estimate is unsafe",
1160 ));
1161 }
1162 reserved_physical_bytes =
1163 reserved_physical_bytes.saturating_add(target.remaining_reserved_physical_bytes(reservation.temporary_copies));
1164 consumed_physical_bytes = consumed_physical_bytes.saturating_add(target.consumed_physical_bytes);
1165 observed_physical_bytes = observed_physical_bytes.saturating_add(target.observed_physical_bytes);
1166 inflight_physical_bytes = inflight_physical_bytes.saturating_add(target.inflight_physical_bytes);
1167 pending_physical_bytes = pending_physical_bytes.saturating_add(target.pending_physical_bytes);
1168 }
1169 if reserved_physical_bytes != reservation.remaining_peak_physical_bytes() {
1170 return Err(Error::other(
1171 "pool metadata load failed: decommission capacity reservation target allocation is incomplete",
1172 ));
1173 }
1174 if inflight_physical_bytes != reservation.inflight_target_physical_bytes {
1175 return Err(Error::other(
1176 "pool metadata load failed: decommission capacity inflight target accounting is incomplete",
1177 ));
1178 }
1179 if pending_physical_bytes != reservation.pending_target_physical_bytes {
1180 return Err(Error::other(
1181 "pool metadata load failed: decommission capacity pending target accounting is incomplete",
1182 ));
1183 }
1184 if consumed_physical_bytes != reservation.consumed_target_physical_bytes
1185 || observed_physical_bytes != reservation.observed_target_physical_bytes
1186 {
1187 return Err(Error::other(
1188 "pool metadata load failed: decommission capacity target progress accounting is incomplete",
1189 ));
1190 }
1191 if reservation.released_at.is_some() != reservation.release_reason.is_some() {
1192 return Err(Error::other(
1193 "pool metadata load failed: decommission capacity reservation release state is incomplete",
1194 ));
1195 }
1196 Ok(())
1197}
1198
1199fn active_decommission_capacity_model(meta: &PoolMeta) -> Result<Option<u16>> {
1200 let mut active_model = None;
1201 for reservation in meta
1202 .pools
1203 .iter()
1204 .filter_map(|pool| pool.decommission.as_ref()?.capacity_reservation.as_ref())
1205 .filter(|reservation| reservation.active())
1206 {
1207 if !matches!(
1208 reservation.model_version,
1209 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
1210 ) {
1211 return Err(Error::DecommissionCapacity(format!(
1212 "pool metadata load failed: unsupported active decommission capacity model version {}",
1213 reservation.model_version
1214 )));
1215 }
1216 match active_model {
1217 Some(model_version) if model_version != reservation.model_version => {
1218 return Err(Error::DecommissionCapacity(
1219 "pool metadata load failed: active decommission capacity reservations use mixed lock models".to_string(),
1220 ));
1221 }
1222 Some(_) => {}
1223 None => active_model = Some(reservation.model_version),
1224 }
1225 }
1226 Ok(active_model)
1227}
1228
1229fn validate_decommission_capacity_model_cohort(meta: &PoolMeta) -> Result<()> {
1230 active_decommission_capacity_model(meta).map(|_| ())
1231}
1232
1233fn select_decommission_capacity_model(meta: &PoolMeta, target_fence_proof_available: bool) -> Result<u16> {
1234 match active_decommission_capacity_model(meta)? {
1235 Some(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION) => Ok(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION),
1236 Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) if target_fence_proof_available => {
1237 Ok(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION)
1238 }
1239 Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) => Err(Error::DecommissionCapacity(
1240 "failed to start decommission: an active per-target capacity cohort requires a live all-v4 fleet proof".to_string(),
1241 )),
1242 Some(version) => Err(Error::DecommissionCapacity(format!(
1243 "failed to start decommission: unsupported active capacity model version {version}"
1244 ))),
1245 None if target_fence_proof_available => Ok(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION),
1246 None => Ok(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION),
1247 }
1248}
1249
1250fn ensure_decommission_target_fence_fleet_proof(
1251 proof: Option<&crate::services::notification_sys::DecommissionTargetFenceFleetProofToken>,
1252 required: bool,
1253) -> Result<()> {
1254 if !required {
1255 return Ok(());
1256 }
1257 if proof.is_some_and(crate::services::notification_sys::decommission_target_fence_fleet_proof_matches) {
1258 return Ok(());
1259 }
1260 Err(Error::other(DECOMMISSION_TARGET_FLEET_PROOF_EXPIRED))
1261}
1262
1263fn release_decommission_capacity_reservation(info: &mut PoolDecommissionInfo, reason: &str, now: OffsetDateTime) -> bool {
1264 let Some(reservation) = info.capacity_reservation.as_mut() else {
1265 return false;
1266 };
1267 if !reservation.active() {
1268 return false;
1269 }
1270 reservation.released_at = Some(now);
1271 reservation.release_reason = Some(reason.to_string());
1272 metrics::gauge!(
1273 METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES,
1274 "pool_index" => reservation.source_pool_index.to_string()
1275 )
1276 .set(0.0);
1277 true
1278}
1279
1280fn renew_decommission_capacity_reservation(
1281 reservation: &mut DecommissionCapacityReservation,
1282 now: OffsetDateTime,
1283 recover_expired: bool,
1284) -> bool {
1285 if !reservation.active() {
1286 return false;
1287 }
1288 let expired = reservation.expires_at <= now;
1289 if expired && recover_expired {
1290 reservation.owner_nonce = uuid::Uuid::new_v4();
1291 reservation.recovered_at = Some(now);
1292 }
1293 reservation.renewed_at = now;
1294 reservation.expires_at = now + DECOMMISSION_CAPACITY_RESERVATION_TTL;
1295 true
1296}
1297
1298fn next_decommission_capacity_generation(meta: &PoolMeta) -> Result<u64> {
1299 meta.pools
1300 .iter()
1301 .filter_map(|pool| pool.decommission.as_ref()?.capacity_reservation.as_ref())
1302 .map(|reservation| reservation.generation)
1303 .max()
1304 .unwrap_or_default()
1305 .checked_add(1)
1306 .ok_or_else(|| Error::other("failed to start decommission: capacity reservation generation overflow"))
1307}
1308
1309fn active_decommission_source_indices(meta: &PoolMeta) -> HashSet<usize> {
1310 meta.pools
1311 .iter()
1312 .enumerate()
1313 .filter_map(|(idx, pool)| {
1314 pool.decommission.as_ref().and_then(|info| {
1315 (info.has_decommission_state() && is_decommission_active(info.complete, info.failed, info.canceled))
1316 .then_some(idx)
1317 })
1318 })
1319 .collect()
1320}
1321
1322#[cfg(test)]
1323fn build_decommission_capacity_reservation(
1324 source: DecommissionPoolCapacityInfo,
1325 target_layout: DecommissionErasureLayout,
1326 operation_id: uuid::Uuid,
1327 generation: u64,
1328 now: OffsetDateTime,
1329) -> Result<DecommissionCapacityReservation> {
1330 build_decommission_capacity_reservation_with_model(
1331 source,
1332 target_layout,
1333 operation_id,
1334 generation,
1335 now,
1336 DECOMMISSION_CAPACITY_MODEL_VERSION,
1337 )
1338}
1339
1340fn build_decommission_capacity_reservation_with_model(
1341 source: DecommissionPoolCapacityInfo,
1342 target_layout: DecommissionErasureLayout,
1343 operation_id: uuid::Uuid,
1344 generation: u64,
1345 now: OffsetDateTime,
1346 model_version: u16,
1347) -> Result<DecommissionCapacityReservation> {
1348 if !matches!(
1349 model_version,
1350 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
1351 ) {
1352 return Err(Error::DecommissionCapacity(format!(
1353 "failed to build decommission capacity reservation: unsupported lock model {model_version}"
1354 )));
1355 }
1356 let source_data_equivalent_bytes = capacity_source_data_equivalent(source.physical_used, source.layout)?;
1357 let predicted_physical_bytes = capacity_target_physical_bytes(source_data_equivalent_bytes, target_layout)?;
1358 let temporary_physical_bytes = predicted_physical_bytes.saturating_mul(DECOMMISSION_CAPACITY_TEMPORARY_COPIES);
1359 Ok(DecommissionCapacityReservation {
1360 model_version,
1361 operation_id,
1362 generation,
1363 owner_nonce: uuid::Uuid::new_v4(),
1364 source_pool_index: source.pool_index,
1365 source_layout: source.layout,
1366 source_physical_total_bytes: source.physical_total,
1367 source_physical_bytes: source.physical_used,
1368 source_data_equivalent_bytes,
1369 predicted_physical_bytes,
1370 temporary_copies: DECOMMISSION_CAPACITY_TEMPORARY_COPIES,
1371 temporary_physical_bytes,
1372 peak_physical_bytes: predicted_physical_bytes.saturating_add(temporary_physical_bytes),
1373 committed_data_bytes: 0,
1374 consumed_target_physical_bytes: 0,
1375 observed_target_physical_bytes: 0,
1376 inflight_target_physical_bytes: 0,
1377 pending_target_physical_bytes: 0,
1378 prediction_error_bytes: 0,
1379 targets: Vec::new(),
1380 created_at: now,
1381 renewed_at: now,
1382 expires_at: now + DECOMMISSION_CAPACITY_RESERVATION_TTL,
1383 recovered_at: None,
1384 released_at: None,
1385 release_reason: None,
1386 })
1387}
1388
1389fn reserve_decommission_start_target_capacity(
1390 meta: &mut PoolMeta,
1391 requested_indices: &[usize],
1392 capacity_infos: &[DecommissionPoolCapacityInfo],
1393 operation_id: uuid::Uuid,
1394 generation: u64,
1395 now: OffsetDateTime,
1396 model_version: u16,
1397) -> Result<()> {
1398 validate_decommission_capacity_model_cohort(meta)?;
1399 let active_sources = active_decommission_source_indices(meta);
1400 let targets = capacity_infos
1401 .iter()
1402 .copied()
1403 .filter(|capacity| {
1404 !active_sources.contains(&capacity.pool_index)
1405 && meta
1406 .pools
1407 .get(capacity.pool_index)
1408 .is_some_and(is_decommission_start_active_pool)
1409 })
1410 .collect::<Vec<_>>();
1411 let target_layout = worst_decommission_target_layout(&targets)?;
1412 let requested = requested_indices.iter().copied().collect::<HashSet<_>>();
1413 let mut reservations = Vec::with_capacity(active_sources.len());
1414
1415 let mut source_indices = active_sources.into_iter().collect::<Vec<_>>();
1416 source_indices.sort_unstable();
1417 for source_index in source_indices {
1418 let source = capacity_infos
1419 .iter()
1420 .copied()
1421 .find(|capacity| capacity.pool_index == source_index)
1422 .ok_or_else(|| {
1423 Error::DecommissionCapacity(format!(
1424 "failed to start decommission: capacity snapshot is missing source pool {source_index}"
1425 ))
1426 })?;
1427 let pool = meta
1428 .pools
1429 .get(source_index)
1430 .ok_or_else(|| invalid_decommission_pool_index_error(meta.pools.len(), source_index))?;
1431 let info = pool
1432 .decommission
1433 .as_ref()
1434 .ok_or_else(|| decommission_metadata_not_initialized_error("reserve decommission capacity"))?;
1435 let new_reservation = requested.contains(&source_index);
1436 let mut reservation = if new_reservation {
1437 build_decommission_capacity_reservation_with_model(
1438 source,
1439 target_layout,
1440 operation_id,
1441 generation,
1442 now,
1443 model_version,
1444 )?
1445 } else {
1446 info.capacity_reservation.clone().ok_or_else(|| {
1447 Error::DecommissionCapacity(format!(
1448 "failed to start decommission: active source pool {source_index} has no durable capacity reservation"
1449 ))
1450 })?
1451 };
1452 if !reservation.active() {
1453 return Err(Error::DecommissionCapacity(format!(
1454 "failed to start decommission: active source pool {source_index} has a released capacity reservation"
1455 )));
1456 }
1457 renew_decommission_capacity_reservation(&mut reservation, now, true);
1458 reservations.push((source_index, reservation, new_reservation));
1459 }
1460
1461 let mut remaining_target_capacity = targets
1462 .iter()
1463 .map(|target| (target.pool_index, target.physical_free))
1464 .collect::<HashMap<_, _>>();
1465 for (_, reservation, new_reservation) in &reservations {
1466 if *new_reservation {
1467 continue;
1468 }
1469 for target in &reservation.targets {
1470 let available = remaining_target_capacity.entry(target.pool_index).or_default();
1471 *available = available
1472 .saturating_add(target.inflight_physical_bytes)
1473 .saturating_add(target.pending_physical_bytes);
1474 let required = target.remaining_reserved_physical_bytes(reservation.temporary_copies);
1475 if *available < required {
1476 return Err(decommission_capacity_blocked_error(format!(
1477 "failed to start decommission: existing reservation for source pool {} requires {required} bytes on target pool {}, but only {} bytes remain",
1478 reservation.source_pool_index, target.pool_index, *available
1479 )));
1480 }
1481 *available -= required;
1482 }
1483 }
1484 let required = reservations
1485 .iter()
1486 .filter(|(_, _, new_reservation)| *new_reservation)
1487 .fold(0usize, |total, (_, reservation, _)| {
1488 total.saturating_add(reservation.remaining_peak_physical_bytes())
1489 });
1490 let available = remaining_target_capacity
1491 .values()
1492 .copied()
1493 .fold(0usize, usize::saturating_add);
1494 if available < required {
1495 metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => "activation").increment(1);
1496 return Err(decommission_capacity_blocked_error(format!(
1497 "failed to start decommission: insufficient reserved physical target capacity: operation {operation_id} generation {generation} requires {required} bytes, but {available} bytes are available after source/target parity and temporary-copy accounting"
1498 )));
1499 }
1500
1501 for (_, reservation, new_reservation) in &mut reservations {
1502 if !*new_reservation {
1503 continue;
1504 }
1505 let mut remaining = reservation.remaining_peak_physical_bytes();
1506 for target in &targets {
1507 if remaining == 0 {
1508 break;
1509 }
1510 let target_remaining = remaining_target_capacity.entry(target.pool_index).or_default();
1511 let allocated = remaining.min(*target_remaining);
1512 if allocated == 0 {
1513 continue;
1514 }
1515 *target_remaining -= allocated;
1516 remaining -= allocated;
1517 reservation.targets.push(DecommissionCapacityTarget {
1518 pool_index: target.pool_index,
1519 layout: target.layout,
1520 physical_total_at_reservation: target.physical_total,
1521 physical_free_at_reservation: target.physical_free,
1522 reserved_physical_bytes: allocated,
1523 consumed_physical_bytes: 0,
1524 observed_physical_bytes: 0,
1525 inflight_physical_bytes: 0,
1526 pending_physical_bytes: 0,
1527 pending_mutation_id: None,
1528 temporary_mutations: Vec::new(),
1529 });
1530 }
1531 debug_assert_eq!(remaining, 0);
1532 }
1533
1534 for (source_index, reservation, _) in reservations {
1535 metrics::gauge!(
1536 METRIC_DECOMMISSION_CAPACITY_PREDICTED_BYTES,
1537 "pool_index" => source_index.to_string()
1538 )
1539 .set(reservation.predicted_physical_bytes as f64);
1540 metrics::gauge!(
1541 METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES,
1542 "pool_index" => source_index.to_string()
1543 )
1544 .set(reservation.remaining_peak_physical_bytes() as f64);
1545 let pool_count = meta.pools.len();
1546 let info = meta
1547 .pools
1548 .get_mut(source_index)
1549 .and_then(|pool| pool.decommission.as_mut())
1550 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_index))?;
1551 info.capacity_blocked_reason = None;
1552 info.capacity_reservation = Some(reservation);
1553 }
1554 if meta.version != POOL_META_GENERATION_VERSION {
1555 meta.version = POOL_META_VERSION;
1556 }
1557 validate_decommission_capacity_model_cohort(meta)?;
1558 Ok(())
1559}
1560
1561fn ensure_decommission_start_target_capacity(
1562 meta: &PoolMeta,
1563 indices: &[usize],
1564 capacity_infos: &[DecommissionPoolCapacityInfo],
1565 target_fence_proof_available: bool,
1566) -> Result<()> {
1567 let generation = next_decommission_capacity_generation(meta)?;
1568 let mut projected = meta.clone();
1569 let first_idx = indices.first().copied();
1570 for idx in indices.iter().copied() {
1571 let capacity = capacity_infos
1572 .iter()
1573 .find(|capacity| capacity.pool_index == idx)
1574 .ok_or_else(|| Error::DecommissionCapacity(format!("decommission capacity snapshot is missing pool {idx}")))?;
1575 if Some(idx) == first_idx {
1576 projected.decommission(idx, capacity.space)?;
1577 } else {
1578 projected.queue_decommission(idx, capacity.space)?;
1579 }
1580 }
1581 let model_version = select_decommission_capacity_model(&projected, target_fence_proof_available)?;
1582 reserve_decommission_start_target_capacity(
1583 &mut projected,
1584 indices,
1585 capacity_infos,
1586 uuid::Uuid::new_v4(),
1587 generation,
1588 OffsetDateTime::now_utc(),
1589 model_version,
1590 )
1591}
1592
1593fn recover_decommission_capacity_reservations(
1594 meta: &mut PoolMeta,
1595 capacity_infos: &[DecommissionPoolCapacityInfo],
1596 now: OffsetDateTime,
1597 target_fence_proof_available: bool,
1598) -> Result<Vec<usize>> {
1599 ensure_decommission_capacity_writer_supported(meta)?;
1600 let mut active_indices = active_decommission_source_indices(meta).into_iter().collect::<Vec<_>>();
1601 active_indices.sort_unstable();
1602 let missing_indices = active_indices
1603 .iter()
1604 .copied()
1605 .filter(|idx| {
1606 meta.pools
1607 .get(*idx)
1608 .and_then(|pool| pool.decommission.as_ref())
1609 .and_then(|info| info.capacity_reservation.as_ref())
1610 .is_none_or(|reservation| !reservation.active())
1611 })
1612 .collect::<Vec<_>>();
1613 if missing_indices.is_empty() {
1614 validate_decommission_capacity_model_cohort(meta)?;
1615 return Ok(active_indices);
1616 }
1617 let generation = next_decommission_capacity_generation(meta)?;
1618 let model_version = select_decommission_capacity_model(meta, target_fence_proof_available)?;
1619 reserve_decommission_start_target_capacity(
1620 meta,
1621 &missing_indices,
1622 capacity_infos,
1623 uuid::Uuid::new_v4(),
1624 generation,
1625 now,
1626 model_version,
1627 )?;
1628 for idx in missing_indices {
1629 if let Some(reservation) = meta
1630 .pools
1631 .get_mut(idx)
1632 .and_then(|pool| pool.decommission.as_mut())
1633 .and_then(|info| info.capacity_reservation.as_mut())
1634 {
1635 reservation.recovered_at = Some(now);
1636 }
1637 }
1638 Ok(active_indices)
1639}
1640
1641fn signed_capacity_difference(observed: usize, predicted: usize) -> i64 {
1642 if observed >= predicted {
1643 i64::try_from(observed.saturating_sub(predicted)).unwrap_or(i64::MAX)
1644 } else {
1645 -i64::try_from(predicted.saturating_sub(observed)).unwrap_or(i64::MAX)
1646 }
1647}
1648
1649fn active_decommission_target_reservations(meta: &PoolMeta) -> HashMap<usize, usize> {
1650 let mut target_reservations = HashMap::new();
1651 for reservation in meta
1652 .pools
1653 .iter()
1654 .filter_map(|pool| pool.decommission.as_ref()?.capacity_reservation.as_ref())
1655 .filter(|reservation| reservation.active())
1656 {
1657 for target in &reservation.targets {
1658 let reserved = target_reservations.entry(target.pool_index).or_insert(0usize);
1659 *reserved = reserved.saturating_add(target.remaining_reserved_physical_bytes(reservation.temporary_copies));
1660 }
1661 }
1662 target_reservations
1663}
1664
1665fn observe_decommission_capacity_reservation(
1666 info: &mut PoolDecommissionInfo,
1667 _capacity_infos: &[DecommissionPoolCapacityInfo],
1668 _active_target_reservations: &HashMap<usize, usize>,
1669) {
1670 let Some(reservation) = info.capacity_reservation.as_mut() else {
1671 return;
1672 };
1673 reservation.prediction_error_bytes =
1674 signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes);
1675 metrics::gauge!(
1676 METRIC_DECOMMISSION_CAPACITY_PREDICTION_ERROR_BYTES,
1677 "pool_index" => reservation.source_pool_index.to_string()
1678 )
1679 .set(reservation.prediction_error_bytes as f64);
1680 metrics::histogram!(
1681 METRIC_DECOMMISSION_CAPACITY_PREDICTION_ABSOLUTE_ERROR_BYTES,
1682 "pool_index" => reservation.source_pool_index.to_string()
1683 )
1684 .record(reservation.prediction_error_bytes.unsigned_abs() as f64);
1685}
1686
1687fn ensure_decommission_capacity_reservations_available(
1688 meta: &PoolMeta,
1689 capacity_infos: &[DecommissionPoolCapacityInfo],
1690 phase: &'static str,
1691) -> Result<()> {
1692 let mut required_by_target = HashMap::<usize, usize>::new();
1693 let mut inflight_by_target = HashMap::<usize, usize>::new();
1694 let mut pending_by_target = HashMap::<usize, usize>::new();
1695 for source_index in active_decommission_source_indices(meta) {
1696 let info = meta.pools[source_index]
1697 .decommission
1698 .as_ref()
1699 .ok_or_else(|| decommission_metadata_not_initialized_error("check decommission capacity reservation"))?;
1700 let reservation = info.capacity_reservation.as_ref().ok_or_else(|| {
1701 decommission_capacity_blocked_error(format!(
1702 "active decommission source pool {source_index} has no durable capacity reservation"
1703 ))
1704 })?;
1705 if !reservation.active() {
1706 return Err(decommission_capacity_blocked_error(format!(
1707 "active decommission source pool {source_index} has a released capacity reservation"
1708 )));
1709 }
1710 metrics::gauge!(
1711 METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES,
1712 "pool_index" => source_index.to_string()
1713 )
1714 .set(reservation.remaining_peak_physical_bytes() as f64);
1715 for target in &reservation.targets {
1716 let temporary_budget = target.remaining_reserved_physical_bytes(reservation.temporary_copies)
1717 / 1usize.saturating_add(reservation.temporary_copies);
1718 if target.inflight_physical_bytes > temporary_budget {
1719 return Err(decommission_capacity_blocked_error(format!(
1720 "source pool {source_index} target pool {} has {} inflight physical bytes, exceeding its {temporary_budget}-byte temporary-copy reservation during {phase}",
1721 target.pool_index, target.inflight_physical_bytes
1722 )));
1723 }
1724 if target.pending_physical_bytes > temporary_budget {
1725 return Err(decommission_capacity_blocked_error(format!(
1726 "source pool {source_index} target pool {} has {} pending physical bytes, exceeding its {temporary_budget}-byte committed-copy reservation during {phase}",
1727 target.pool_index, target.pending_physical_bytes
1728 )));
1729 }
1730 let required = required_by_target.entry(target.pool_index).or_default();
1731 *required = required.saturating_add(target.remaining_reserved_physical_bytes(reservation.temporary_copies));
1732 let inflight = inflight_by_target.entry(target.pool_index).or_default();
1733 *inflight = inflight.saturating_add(target.inflight_physical_bytes);
1734 let pending = pending_by_target.entry(target.pool_index).or_default();
1735 *pending = pending.saturating_add(target.pending_physical_bytes);
1736 }
1737 }
1738 for (target_pool_index, required) in required_by_target {
1739 let available = capacity_infos
1740 .iter()
1741 .find(|capacity| capacity.pool_index == target_pool_index)
1742 .map(|capacity| capacity.physical_free)
1743 .ok_or_else(|| {
1744 decommission_capacity_blocked_error(format!(
1745 "capacity snapshot is missing target pool {target_pool_index} during {phase}"
1746 ))
1747 })?
1748 .saturating_add(inflight_by_target.get(&target_pool_index).copied().unwrap_or_default())
1749 .saturating_add(pending_by_target.get(&target_pool_index).copied().unwrap_or_default());
1750 if available >= required {
1751 continue;
1752 }
1753 metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => phase).increment(1);
1754 return Err(decommission_capacity_blocked_error(format!(
1755 "target pool {target_pool_index} reservation requires {required} physical bytes, but only {available} bytes remain during {phase}"
1756 )));
1757 }
1758 Ok(())
1759}
1760
1761#[derive(Clone, Copy)]
1762pub(crate) enum DecommissionCapacityAdmission {
1763 Mutation,
1764 ExistingMultipart,
1765 ScannerBacklog,
1766 BatchDelete,
1767 Heal,
1768}
1769
1770impl DecommissionCapacityAdmission {
1771 fn phase(self) -> &'static str {
1772 match self {
1773 Self::Mutation => "mutation",
1774 Self::ExistingMultipart => "existing_multipart",
1775 Self::ScannerBacklog => "scanner_backlog",
1776 Self::BatchDelete => "batch_delete",
1777 Self::Heal => "heal",
1778 }
1779 }
1780}
1781
1782fn ensure_external_decommission_target_admission(
1783 meta: &PoolMeta,
1784 target_pool_index: usize,
1785 admission: DecommissionCapacityAdmission,
1786) -> Result<()> {
1787 let phase = admission.phase();
1788 if matches!(admission, DecommissionCapacityAdmission::ScannerBacklog)
1792 && !meta.scanner_pause_backlog_pool_writable(target_pool_index)
1793 {
1794 return Err(Error::SlowDown);
1795 }
1796 let active_sources = active_decommission_source_indices(meta);
1797 if meta.is_suspended(target_pool_index) {
1798 let active_source = active_sources.contains(&target_pool_index);
1799 if !matches!(
1800 admission,
1801 DecommissionCapacityAdmission::Heal | DecommissionCapacityAdmission::ScannerBacklog
1802 ) && !(matches!(admission, DecommissionCapacityAdmission::ExistingMultipart) && active_source)
1803 {
1804 return Err(Error::SlowDown);
1805 }
1806 }
1807 if active_sources.into_iter().any(|source_pool_index| {
1808 meta.pools
1809 .get(source_pool_index)
1810 .and_then(|pool| pool.decommission.as_ref())
1811 .and_then(|info| info.capacity_reservation.as_ref())
1812 .is_none_or(|reservation| !reservation.active())
1813 }) {
1814 metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => phase).increment(1);
1815 return Err(Error::SlowDown);
1816 }
1817 if !matches!(admission, DecommissionCapacityAdmission::Heal) {
1822 return Ok(());
1823 }
1824 let reserved = active_decommission_target_reservations(meta)
1825 .get(&target_pool_index)
1826 .copied()
1827 .unwrap_or_default();
1828 if reserved == 0 {
1829 return Ok(());
1830 }
1831 metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => phase).increment(1);
1832 Err(Error::SlowDown)
1833}
1834
1835fn ensure_decommission_target_owner_admission(
1836 meta: &PoolMeta,
1837 owner: DecommissionCapacityOwner,
1838 target_pool_index: usize,
1839 target_physical_bytes: usize,
1840 now: OffsetDateTime,
1841) -> Result<()> {
1842 let reservation = meta
1843 .pools
1844 .get(owner.source_pool_index)
1845 .and_then(|pool| pool.decommission.as_ref())
1846 .and_then(|info| info.capacity_reservation.as_ref())
1847 .filter(|reservation| reservation.admits_owner(owner, now))
1848 .ok_or_else(|| Error::SlowDown)?;
1849 let target = reservation
1850 .targets
1851 .iter()
1852 .find(|target| target.pool_index == target_pool_index)
1853 .ok_or_else(|| Error::SlowDown)?;
1854 let required_peak = target_physical_bytes.saturating_mul(1usize.saturating_add(reservation.temporary_copies));
1855 let remaining = target.remaining_reserved_physical_bytes(reservation.temporary_copies);
1856 if required_peak <= remaining {
1857 return Ok(());
1858 }
1859 Err(decommission_capacity_blocked_error(format!(
1860 "source pool {} target pool {target_pool_index} operation requires {required_peak} physical bytes, but only {remaining} reserved bytes remain",
1861 owner.source_pool_index
1862 )))
1863}
1864
1865fn reserve_decommission_target_pending(
1866 meta: &mut PoolMeta,
1867 source_pool_index: usize,
1868 target_pool_index: usize,
1869 requested_physical_bytes: usize,
1870 mutation_id: uuid::Uuid,
1871 now: OffsetDateTime,
1872) -> Result<usize> {
1873 if requested_physical_bytes == 0 {
1874 return Ok(0);
1875 }
1876 let pool_count = meta.pools.len();
1877 let pool = meta
1878 .pools
1879 .get_mut(source_pool_index)
1880 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
1881 let info = pool
1882 .decommission
1883 .as_mut()
1884 .ok_or_else(|| decommission_metadata_not_initialized_error("reserve target capacity mutation"))?;
1885 let reservation = info
1886 .capacity_reservation
1887 .as_mut()
1888 .filter(|reservation| reservation.active())
1889 .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared before target write"))?;
1890 let target = reservation
1891 .targets
1892 .iter_mut()
1893 .find(|target| target.pool_index == target_pool_index)
1894 .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared before target write"))?;
1895 if target.pending_physical_bytes == 0 {
1896 target.pending_mutation_id = None;
1897 }
1898 if target.pending_physical_bytes > 0 {
1899 match target.pending_mutation_id {
1900 Some(pending_mutation_id) if pending_mutation_id == mutation_id => {
1901 if requested_physical_bytes <= target.pending_physical_bytes {
1902 return Ok(0);
1903 }
1904 let budget = target.remaining_reserved_physical_bytes(reservation.temporary_copies)
1905 / 1usize.saturating_add(reservation.temporary_copies);
1906 if requested_physical_bytes > budget {
1907 return Err(decommission_capacity_blocked_error(format!(
1908 "source pool {source_pool_index} target pool {target_pool_index} mutation intent size changed during retry"
1909 )));
1910 }
1911 let additional = requested_physical_bytes.saturating_sub(target.pending_physical_bytes);
1912 target.pending_physical_bytes = requested_physical_bytes;
1913 reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_add(additional);
1914 renew_decommission_capacity_reservation(reservation, now, true);
1915 pool.last_update = pool.last_update.max(now);
1916 return Ok(additional);
1917 }
1918 pending_mutation_id => {
1919 return Err(decommission_capacity_blocked_error(format!(
1920 "source pool {source_pool_index} target pool {target_pool_index} has an unresolved target capacity intent {pending_mutation_id:?} while mutation {mutation_id} is waiting"
1921 )));
1922 }
1923 }
1924 }
1925 let budget = target.remaining_reserved_physical_bytes(reservation.temporary_copies)
1926 / 1usize.saturating_add(reservation.temporary_copies);
1927 if requested_physical_bytes > budget {
1928 return Err(decommission_capacity_blocked_error(format!(
1929 "source pool {source_pool_index} target pool {target_pool_index} mutation requires {requested_physical_bytes} physical bytes, but only {budget} committed-copy bytes remain"
1930 )));
1931 }
1932 let additional = requested_physical_bytes;
1933 target.pending_mutation_id = Some(mutation_id);
1934 target.pending_physical_bytes = target.pending_physical_bytes.saturating_add(additional);
1935 reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_add(additional);
1936 renew_decommission_capacity_reservation(reservation, now, true);
1937 pool.last_update = pool.last_update.max(now);
1938 Ok(additional)
1939}
1940
1941fn resolve_decommission_target_pending(
1942 meta: &mut PoolMeta,
1943 source_pool_index: usize,
1944 target_pool_index: usize,
1945 resolved_physical_bytes: usize,
1946 mutation_id: uuid::Uuid,
1947) -> Result<()> {
1948 if resolved_physical_bytes == 0 {
1949 return Ok(());
1950 }
1951 let pool_count = meta.pools.len();
1952 let pool = meta
1953 .pools
1954 .get_mut(source_pool_index)
1955 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
1956 let reservation = pool
1957 .decommission
1958 .as_mut()
1959 .and_then(|info| info.capacity_reservation.as_mut())
1960 .filter(|reservation| reservation.active())
1961 .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared after target write"))?;
1962 let target = reservation
1963 .targets
1964 .iter_mut()
1965 .find(|target| target.pool_index == target_pool_index)
1966 .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared after target write"))?;
1967 if target.pending_physical_bytes > 0 && target.pending_mutation_id != Some(mutation_id) {
1968 return Err(decommission_capacity_blocked_error(format!(
1969 "source pool {source_pool_index} target pool {target_pool_index} pending capacity intent belongs to another mutation"
1970 )));
1971 }
1972 let resolved = target.pending_physical_bytes.min(resolved_physical_bytes);
1973 target.pending_physical_bytes = target.pending_physical_bytes.saturating_sub(resolved);
1974 reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_sub(resolved);
1975 if target.pending_physical_bytes == 0 {
1976 target.pending_mutation_id = None;
1977 }
1978 Ok(())
1979}
1980
1981fn release_decommission_target_temporary_mutation(
1982 target: &mut DecommissionCapacityTarget,
1983 mutation_id: uuid::Uuid,
1984 maximum_physical_bytes: usize,
1985) -> (usize, bool) {
1986 let Some(index) = target
1987 .temporary_mutations
1988 .iter()
1989 .position(|mutation| mutation.mutation_id == mutation_id)
1990 else {
1991 return (0, false);
1992 };
1993 let released = target.temporary_mutations[index].physical_bytes.min(maximum_physical_bytes);
1994 target.temporary_mutations[index].physical_bytes = target.temporary_mutations[index].physical_bytes.saturating_sub(released);
1995 let removed = target.temporary_mutations[index].physical_bytes == 0;
1996 if removed {
1997 target.temporary_mutations.remove(index);
1998 }
1999 target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released);
2000 (released, released > 0 || removed)
2001}
2002
2003fn settle_decommission_target_non_growing_replacement(
2004 meta: &mut PoolMeta,
2005 source_pool_index: usize,
2006 target_pool_index: usize,
2007 mutation_id: uuid::Uuid,
2008 now: OffsetDateTime,
2009) -> Result<bool> {
2010 let pool_count = meta.pools.len();
2011 let pool = meta
2012 .pools
2013 .get_mut(source_pool_index)
2014 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2015 let info = pool
2016 .decommission
2017 .as_mut()
2018 .ok_or_else(|| decommission_metadata_not_initialized_error("settle non-growing target replacement"))?;
2019 let reservation = info
2020 .capacity_reservation
2021 .as_mut()
2022 .filter(|reservation| reservation.active())
2023 .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target replacement"))?;
2024 let target = reservation
2025 .targets
2026 .iter_mut()
2027 .find(|target| target.pool_index == target_pool_index)
2028 .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target replacement"))?;
2029 let (released, changed) = release_decommission_target_temporary_mutation(target, mutation_id, usize::MAX);
2030 reservation.inflight_target_physical_bytes = reservation.inflight_target_physical_bytes.saturating_sub(released);
2031 if changed {
2032 renew_decommission_capacity_reservation(reservation, now, true);
2033 pool.last_update = pool.last_update.max(now);
2034 }
2035 Ok(changed)
2036}
2037
2038struct DecommissionTargetConsumption {
2039 committed_data_bytes: usize,
2040 target_physical_bytes: usize,
2041 observed_physical_bytes: usize,
2042}
2043
2044fn record_decommission_target_consumption(
2045 meta: &mut PoolMeta,
2046 source_pool_index: usize,
2047 target_pool_index: usize,
2048 consumption: DecommissionTargetConsumption,
2049 mutation_id: uuid::Uuid,
2050 now: OffsetDateTime,
2051) -> Result<()> {
2052 let DecommissionTargetConsumption {
2053 committed_data_bytes,
2054 target_physical_bytes,
2055 observed_physical_bytes,
2056 } = consumption;
2057 let pool_count = meta.pools.len();
2058 let pool = meta
2059 .pools
2060 .get_mut(source_pool_index)
2061 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2062 let info = pool
2063 .decommission
2064 .as_mut()
2065 .ok_or_else(|| decommission_metadata_not_initialized_error("record target capacity consumption"))?;
2066 let reservation = info
2067 .capacity_reservation
2068 .as_mut()
2069 .filter(|reservation| reservation.active())
2070 .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target write"))?;
2071 let target = reservation
2072 .targets
2073 .iter_mut()
2074 .find(|target| target.pool_index == target_pool_index)
2075 .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target write"))?;
2076 let remaining_target_bytes = target.remaining_reserved_physical_bytes(reservation.temporary_copies)
2077 / 1usize.saturating_add(reservation.temporary_copies);
2078 let remaining_total_bytes = reservation
2079 .predicted_physical_bytes
2080 .saturating_sub(reservation.consumed_target_physical_bytes);
2081 let consumed = target_physical_bytes.min(remaining_target_bytes).min(remaining_total_bytes);
2082 target.consumed_physical_bytes = target.consumed_physical_bytes.saturating_add(consumed);
2083 target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes);
2084 let has_scoped_temporary_mutations = !target.temporary_mutations.is_empty();
2085 let (released_inflight, _) = release_decommission_target_temporary_mutation(target, mutation_id, usize::MAX);
2086 let released_inflight = if released_inflight == 0 && !has_scoped_temporary_mutations {
2087 let released = target.inflight_physical_bytes.min(consumed);
2088 target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released);
2089 released
2090 } else {
2091 released_inflight
2092 };
2093 reservation.consumed_target_physical_bytes = reservation.consumed_target_physical_bytes.saturating_add(consumed);
2094 let committed = committed_data_bytes.min(
2095 reservation
2096 .source_data_equivalent_bytes
2097 .saturating_sub(reservation.committed_data_bytes),
2098 );
2099 reservation.committed_data_bytes = reservation.committed_data_bytes.saturating_add(committed);
2100 reservation.observed_target_physical_bytes = reservation
2101 .observed_target_physical_bytes
2102 .saturating_add(observed_physical_bytes);
2103 reservation.inflight_target_physical_bytes = reservation.inflight_target_physical_bytes.saturating_sub(released_inflight);
2104 reservation.prediction_error_bytes =
2105 signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes);
2106 renew_decommission_capacity_reservation(reservation, now, true);
2107 info.capacity_blocked_reason = None;
2108 pool.last_update = pool.last_update.max(now);
2109 Ok(())
2110}
2111
2112fn record_decommission_target_inflight(
2113 meta: &mut PoolMeta,
2114 source_pool_index: usize,
2115 target_pool_index: usize,
2116 observed_physical_bytes: usize,
2117 mutation_id: uuid::Uuid,
2118 now: OffsetDateTime,
2119) -> Result<bool> {
2120 let pool_count = meta.pools.len();
2121 let pool = meta
2122 .pools
2123 .get_mut(source_pool_index)
2124 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2125 let info = pool
2126 .decommission
2127 .as_mut()
2128 .ok_or_else(|| decommission_metadata_not_initialized_error("record target capacity inflight bytes"))?;
2129 let reservation = info
2130 .capacity_reservation
2131 .as_mut()
2132 .filter(|reservation| reservation.active())
2133 .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target write"))?;
2134 let target = reservation
2135 .targets
2136 .iter_mut()
2137 .find(|target| target.pool_index == target_pool_index)
2138 .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target write"))?;
2139 let ledger_changed = if let Some(mutation) = target
2140 .temporary_mutations
2141 .iter_mut()
2142 .find(|mutation| mutation.mutation_id == mutation_id)
2143 {
2144 mutation.physical_bytes = mutation.physical_bytes.saturating_add(observed_physical_bytes);
2145 observed_physical_bytes > 0
2146 } else if observed_physical_bytes > 0 || reservation.model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION {
2147 target.temporary_mutations.push(DecommissionCapacityTemporaryMutation {
2151 mutation_id,
2152 physical_bytes: observed_physical_bytes,
2153 });
2154 true
2155 } else {
2156 false
2157 };
2158 target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes);
2159 target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_add(observed_physical_bytes);
2160 reservation.observed_target_physical_bytes = reservation
2161 .observed_target_physical_bytes
2162 .saturating_add(observed_physical_bytes);
2163 reservation.inflight_target_physical_bytes = reservation
2164 .inflight_target_physical_bytes
2165 .saturating_add(observed_physical_bytes);
2166 reservation.prediction_error_bytes =
2167 signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes);
2168 renew_decommission_capacity_reservation(reservation, now, true);
2169 pool.last_update = pool.last_update.max(now);
2170 Ok(ledger_changed)
2171}
2172
2173fn record_decommission_target_observation(
2174 meta: &mut PoolMeta,
2175 source_pool_index: usize,
2176 target_pool_index: usize,
2177 observed_physical_bytes: usize,
2178 now: OffsetDateTime,
2179) -> Result<()> {
2180 let pool_count = meta.pools.len();
2181 let pool = meta
2182 .pools
2183 .get_mut(source_pool_index)
2184 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2185 let info = pool
2186 .decommission
2187 .as_mut()
2188 .ok_or_else(|| decommission_metadata_not_initialized_error("record target capacity observation"))?;
2189 let reservation = info
2190 .capacity_reservation
2191 .as_mut()
2192 .filter(|reservation| reservation.active())
2193 .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target write"))?;
2194 let target = reservation
2195 .targets
2196 .iter_mut()
2197 .find(|target| target.pool_index == target_pool_index)
2198 .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target write"))?;
2199 target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes);
2200 reservation.observed_target_physical_bytes = reservation
2201 .observed_target_physical_bytes
2202 .saturating_add(observed_physical_bytes);
2203 reservation.prediction_error_bytes =
2204 signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes);
2205 renew_decommission_capacity_reservation(reservation, now, true);
2206 pool.last_update = pool.last_update.max(now);
2207 Ok(())
2208}
2209
2210fn release_decommission_target_inflight(
2211 meta: &mut PoolMeta,
2212 source_pool_index: usize,
2213 target_pool_index: usize,
2214 released_physical_bytes: usize,
2215 mutation_id: uuid::Uuid,
2216 proof: DecommissionCapacityReleaseProof,
2217 now: OffsetDateTime,
2218) -> Result<bool> {
2219 let DecommissionCapacityReleaseProof {
2220 confirmed_absent,
2221 clear_pending,
2222 } = proof;
2223 let pool_count = meta.pools.len();
2224 let pool = meta
2225 .pools
2226 .get_mut(source_pool_index)
2227 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2228 let info = pool
2229 .decommission
2230 .as_mut()
2231 .ok_or_else(|| decommission_metadata_not_initialized_error("release target capacity inflight bytes"))?;
2232 let reservation = info
2233 .capacity_reservation
2234 .as_mut()
2235 .filter(|reservation| reservation.active())
2236 .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target cleanup"))?;
2237 let model_version = reservation.model_version;
2238 let target = reservation
2239 .targets
2240 .iter_mut()
2241 .find(|target| target.pool_index == target_pool_index)
2242 .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target cleanup"))?;
2243 let has_temporary_mutations = !target.temporary_mutations.is_empty();
2244 let pending_belongs_to_mutation = target.pending_mutation_id == Some(mutation_id);
2245 let zero_ledger_cleanup = confirmed_absent
2246 && model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
2247 && !has_temporary_mutations
2248 && target.pending_mutation_id.is_none()
2249 && target.pending_physical_bytes == 0;
2250 let (released, temporary_mutation_changed) = release_decommission_target_temporary_mutation(
2251 target,
2252 mutation_id,
2253 if confirmed_absent {
2254 usize::MAX
2255 } else {
2256 released_physical_bytes
2257 },
2258 );
2259 let released = if released == 0 && !has_temporary_mutations && model_version == DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION {
2260 let released = target.inflight_physical_bytes.min(released_physical_bytes);
2261 target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released);
2262 released
2263 } else {
2264 released
2265 };
2266 let pending_matches = clear_pending && confirmed_absent && pending_belongs_to_mutation;
2267 let published_pending_delayed_release = confirmed_absent
2268 && model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
2269 && !clear_pending
2270 && !has_temporary_mutations
2271 && pending_belongs_to_mutation;
2272 if released == 0
2273 && released_physical_bytes > 0
2274 && !zero_ledger_cleanup
2275 && !published_pending_delayed_release
2276 && (has_temporary_mutations || !pending_matches)
2277 {
2278 return Err(decommission_capacity_blocked_error(
2279 "temporary target cleanup released bytes that cannot be attributed to its mutation",
2280 ));
2281 }
2282 reservation.inflight_target_physical_bytes = reservation.inflight_target_physical_bytes.saturating_sub(released);
2283 let cleared_pending = if pending_matches {
2284 let cleared = target.pending_physical_bytes;
2285 target.pending_physical_bytes = 0;
2286 target.pending_mutation_id = None;
2287 reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_sub(cleared);
2288 cleared
2289 } else {
2290 0
2291 };
2292 let changed = released > 0 || temporary_mutation_changed || cleared_pending > 0;
2293 if changed {
2294 renew_decommission_capacity_reservation(reservation, now, true);
2295 pool.last_update = pool.last_update.max(now);
2296 }
2297 Ok(changed)
2298}
2299
2300fn ensure_valid_decommission_pool_index(pool_count: usize, idx: usize) -> Result<()> {
2301 if idx >= pool_count {
2302 return Err(invalid_decommission_pool_index_error(pool_count, idx));
2303 }
2304
2305 Ok(())
2306}
2307
2308fn get_by_index<'a, T>(items: &'a [T], idx: usize, operation: &'static str) -> Result<&'a T> {
2309 items.get(idx).ok_or_else(|| {
2310 Error::other(format!(
2311 "failed to {operation}: invalid decommission pool index {idx} for {pool_count} pools",
2312 pool_count = items.len()
2313 ))
2314 })
2315}
2316
2317fn decommission_metadata_not_initialized_error(operation: &str) -> Error {
2318 Error::other(format!("failed to {operation}: decommission metadata not initialized"))
2319}
2320
2321fn resolve_decommission_bucket_state(meta: &PoolMeta, idx: usize, bucket: &DecomBucketInfo) -> Result<bool> {
2322 let pool_count = meta.pools.len();
2323 ensure_valid_decommission_pool_index(pool_count, idx)?;
2324
2325 let Some(pool) = meta.pools.get(idx) else {
2326 return Err(invalid_decommission_pool_index_error(pool_count, idx));
2327 };
2328 let Some(info) = pool.decommission.as_ref() else {
2329 return Err(decommission_metadata_not_initialized_error("resolve decommission bucket state"));
2330 };
2331
2332 Ok(info.is_bucket_decommissioned(&bucket.to_string()))
2333}
2334
2335fn mark_decommission_bucket_done(meta: &mut PoolMeta, idx: usize, bucket: &DecomBucketInfo) -> Result<bool> {
2336 let pool_count = meta.pools.len();
2337 ensure_valid_decommission_pool_index(pool_count, idx)?;
2338
2339 let Some(pool) = meta.pools.get_mut(idx) else {
2340 return Err(invalid_decommission_pool_index_error(pool_count, idx));
2341 };
2342 let Some(info) = pool.decommission.as_mut() else {
2343 return Err(decommission_metadata_not_initialized_error("mark decommission bucket done"));
2344 };
2345
2346 Ok(info.bucket_pop(&bucket.to_string()))
2347}
2348
2349fn count_decommission_item(meta: &mut PoolMeta, idx: usize, size: usize, failed: bool) -> Result<()> {
2350 let pool_count = meta.pools.len();
2351 ensure_valid_decommission_pool_index(pool_count, idx)?;
2352
2353 let Some(pool) = meta.pools.get_mut(idx) else {
2354 return Err(invalid_decommission_pool_index_error(pool_count, idx));
2355 };
2356 let Some(info) = pool.decommission.as_mut() else {
2357 return Err(decommission_metadata_not_initialized_error("count decommission item"));
2358 };
2359
2360 if failed {
2361 info.items_decommission_failed += 1;
2362 info.bytes_failed += size;
2363 } else {
2364 info.items_decommissioned += 1;
2365 info.bytes_done += size;
2366 }
2367
2368 Ok(())
2369}
2370
2371fn ensure_decommission_generation(meta: &PoolMeta, idx: usize, generation: OffsetDateTime) -> Result<()> {
2372 let Some(pool) = meta.pools.get(idx) else {
2373 return Err(invalid_decommission_pool_index_error(meta.pools.len(), idx));
2374 };
2375 let Some(info) = pool.decommission.as_ref() else {
2376 return Err(decommission_metadata_not_initialized_error("check decommission generation"));
2377 };
2378
2379 if info.start_time == Some(generation) && !info.queued && is_decommission_active(info.complete, info.failed, info.canceled) {
2380 Ok(())
2381 } else {
2382 Err(Error::OperationCanceled)
2383 }
2384}
2385
2386fn record_decommission_unresolved_entry(
2387 meta: &mut PoolMeta,
2388 idx: usize,
2389 generation: OffsetDateTime,
2390 entry: DecommissionUnresolvedEntry,
2391 now: OffsetDateTime,
2392 rebalance_meta: Option<&RebalanceMeta>,
2393) -> Result<bool> {
2394 ensure_decommission_generation(meta, idx, generation)?;
2395 if entry.pool_index != idx || entry.source_generation != generation {
2396 return Err(Error::other("decommission unresolved entry does not match the active pool generation"));
2397 }
2398
2399 let pool_count = meta.pools.len();
2400 let Some(pool) = meta.pools.get(idx) else {
2401 return Err(invalid_decommission_pool_index_error(pool_count, idx));
2402 };
2403 let Some(info) = pool.decommission.as_ref() else {
2404 return Err(decommission_metadata_not_initialized_error("record decommission unresolved entry"));
2405 };
2406 let existing_index = info.unresolved_entries.iter().position(|existing| {
2407 existing.bucket == entry.bucket
2408 && existing.object == entry.object
2409 && existing.pool_index == entry.pool_index
2410 && existing.set_index == entry.set_index
2411 && existing.source_generation == entry.source_generation
2412 });
2413 if existing_index.is_some_and(|index| info.unresolved_entries[index] == entry) {
2414 return Ok(false);
2415 }
2416
2417 let last_update = meta.next_scanner_data_movement_update(now, rebalance_meta);
2418 let Some(pool) = meta.pools.get_mut(idx) else {
2419 return Err(invalid_decommission_pool_index_error(pool_count, idx));
2420 };
2421 let Some(info) = pool.decommission.as_mut() else {
2422 return Err(decommission_metadata_not_initialized_error("record decommission unresolved entry"));
2423 };
2424 if let Some(index) = existing_index {
2425 info.unresolved_entries[index] = entry;
2426 } else {
2427 info.unresolved_entries.push(entry);
2428 }
2429 pool.last_update = last_update;
2430 Ok(true)
2431}
2432
2433type DecommissionUnresolvedEntryIdentity = (usize, String, String);
2434
2435fn decommission_unresolved_entry_identity(entry: &DecommissionUnresolvedEntry) -> DecommissionUnresolvedEntryIdentity {
2436 (entry.set_index, entry.bucket.clone(), entry.object.clone())
2437}
2438
2439fn reconcile_decommission_unresolved_entries_for_completion(
2440 meta: &mut PoolMeta,
2441 idx: usize,
2442 verified_generation: Option<OffsetDateTime>,
2443 verified_entries: Option<&[DecommissionUnresolvedEntry]>,
2444) -> Result<()> {
2445 let pool_count = meta.pools.len();
2446 let Some(pool) = meta.pools.get(idx) else {
2447 return Err(invalid_decommission_pool_index_error(pool_count, idx));
2448 };
2449 let Some(info) = pool.decommission.as_ref() else {
2450 return Err(decommission_metadata_not_initialized_error("reconcile decommission unresolved entries"));
2451 };
2452 if info.unresolved_entries.is_empty() {
2453 return Ok(());
2454 }
2455
2456 let Some(generation) = verified_generation else {
2457 return Err(Error::other(format!(
2458 "failed to complete decommission for pool {idx}: {} unresolved listing entries remain",
2459 info.unresolved_entries.len()
2460 )));
2461 };
2462 ensure_decommission_generation(meta, idx, generation)?;
2463 if info
2464 .unresolved_entries
2465 .iter()
2466 .any(|entry| entry.source_generation != generation)
2467 {
2468 return Err(Error::other(format!(
2469 "failed to complete decommission for pool {idx}: unresolved listing ledger contains a different generation"
2470 )));
2471 }
2472
2473 let verified_entries = verified_entries.unwrap_or_default();
2474 let unverified_count = info
2475 .unresolved_entries
2476 .iter()
2477 .filter(|entry| !verified_entries.contains(entry))
2478 .count();
2479 if unverified_count > 0 {
2480 return Err(Error::other(format!(
2481 "failed to complete decommission for pool {idx}: {unverified_count} unresolved listing entries were not individually verified"
2482 )));
2483 }
2484
2485 let Some(info) = meta.pools.get_mut(idx).and_then(|pool| pool.decommission.as_mut()) else {
2486 return Err(decommission_metadata_not_initialized_error("reconcile decommission unresolved entries"));
2487 };
2488 info.unresolved_entries.clear();
2489 Ok(())
2490}
2491
2492#[cfg(test)]
2493async fn run_decommission_side_effect<T, E, F, Fut>(
2494 rx: &CancellationToken,
2495 operation_gate: &Arc<tokio::sync::RwLock<()>>,
2496 operation: F,
2497) -> std::result::Result<T, E>
2498where
2499 F: FnOnce() -> Fut,
2500 Fut: std::future::Future<Output = std::result::Result<T, E>>,
2501 E: From<Error>,
2502{
2503 let _operation_guard = tokio::select! {
2504 biased;
2505 _ = rx.cancelled() => return Err(Error::OperationCanceled.into()),
2506 guard = operation_gate.read() => guard,
2507 };
2508
2509 if rx.is_cancelled() {
2510 return Err(Error::OperationCanceled.into());
2511 }
2512
2513 let result = operation().await;
2514 if rx.is_cancelled() {
2515 return Err(Error::OperationCanceled.into());
2516 }
2517 result
2518}
2519
2520fn track_decommission_current_object_stage(
2521 meta: &mut PoolMeta,
2522 idx: usize,
2523 bucket: &str,
2524 object: &str,
2525 stage: &str,
2526) -> Result<()> {
2527 let pool_count = meta.pools.len();
2528 ensure_valid_decommission_pool_index(pool_count, idx)?;
2529
2530 let Some(pool) = meta.pools.get_mut(idx) else {
2531 return Err(invalid_decommission_pool_index_error(pool_count, idx));
2532 };
2533 let Some(info) = pool.decommission.as_mut() else {
2534 return Err(decommission_metadata_not_initialized_error("track decommission current object"));
2535 };
2536
2537 info.object = object.to_string();
2538 info.bucket = bucket.to_string();
2539 info.stage = stage.to_string();
2540 Ok(())
2541}
2542
2543fn track_decommission_current_object(meta: &mut PoolMeta, idx: usize, bucket: &str, object: &str) -> Result<()> {
2544 track_decommission_current_object_stage(meta, idx, bucket, object, "")
2545}
2546
2547fn resolve_decommission_update_after_result(result: Result<bool>) -> Result<bool> {
2548 result.map_err(|err| Error::other(format!("decommission metadata update failed: {err}")))
2549}
2550
2551fn resolve_decommission_progress_save_result(result: Result<()>) -> Option<Error> {
2552 result
2553 .err()
2554 .map(|err| Error::other(format!("decommission progress save failed: {err}")))
2555}
2556
2557fn resolve_decommission_preflight_heal_result<T>(bucket: &str, result: Result<T>) -> Result<T> {
2558 result.map_err(|err| Error::other(format!("decommission preflight heal failed for bucket {bucket}: {err}")))
2559}
2560
2561fn resolve_decommission_optional_bucket_config_result<T>(bucket: &str, stage: &str, result: Result<T>) -> Result<Option<T>> {
2562 match result {
2563 Ok(config) => Ok(Some(config)),
2564 Err(Error::ConfigNotFound) => Ok(None),
2565 Err(err) => Err(Error::other(format!(
2566 "decommission {stage} config load failed for bucket {bucket}: {err}"
2567 ))),
2568 }
2569}
2570
2571fn resolve_decommission_entry_cleanup_delete_result<T>(result: Result<T>, bucket: &str, object_name: &str) -> Result<()> {
2572 match result {
2573 Ok(_) => Ok(()),
2574 Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(()),
2575 Err(err) => Err(Error::other(format!(
2576 "decommission cleanup_delete_object failed for {bucket}/{object_name}: {err}"
2577 ))),
2578 }
2579}
2580
2581fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
2582 result.map_err(|err| Error::other(format!("decommission reload_pool_meta failed for {bucket}/{object_name}: {err}")))
2583}
2584
2585fn resolve_decommission_terminal_mark_result(result: Result<()>, stage: &str, pool_label: &str) -> Result<()> {
2586 result.map_err(|err| Error::other(format!("decommission terminal mark {stage} failed for pool {pool_label}: {err}")))
2587}
2588
2589fn resolve_decommission_terminal_mark_after_error_result(result: Result<()>, idx: usize, primary_err: &Error) -> Result<()> {
2590 result.map_err(|err| {
2591 Error::other(format!(
2592 "decommission terminal mark failed after background error on pool {idx}: {primary_err}; mark error: {err}"
2593 ))
2594 })
2595}
2596
2597fn observe_decommission_terminal_reload_result(result: Result<()>, stage: &str) -> Option<Error> {
2598 result
2599 .err()
2600 .map(|err| Error::other(format!("decommission terminal pool meta reload failed during {stage}: {err}")))
2601}
2602
2603fn decommission_item_size<T>(size: T) -> usize
2604where
2605 usize: TryFrom<T>,
2606{
2607 usize::try_from(size).unwrap_or_default()
2608}
2609
2610fn is_decommission_set_local_usage_cache(bucket: &str, object: &str) -> bool {
2611 if bucket != RUSTFS_META_BUCKET {
2612 return false;
2613 }
2614 let Some(path) = object
2615 .strip_prefix(BUCKET_META_PREFIX)
2616 .and_then(|path| path.strip_prefix('/'))
2617 else {
2618 return false;
2619 };
2620 let name = match path.rsplit_once('/') {
2621 Some((bucket, name)) if !bucket.is_empty() && !bucket.contains('/') && bucket != "." && bucket != ".." => name,
2622 Some(_) => return false,
2623 None => path,
2624 };
2625 name.strip_suffix(".bkp").unwrap_or(name) == DATA_USAGE_CACHE_NAME
2626}
2627
2628fn with_decommission_entry_context<E: Display>(stage: &str, bucket: &str, object: &str, err: E) -> Error {
2629 Error::other(format!("decommission entry {stage} failed for bucket {bucket} object {object}: {err}"))
2630}
2631
2632#[cfg(test)]
2633fn load_decommission_entry_versions(entry: &MetaCacheEntry, bucket: &str, stage: &str) -> Result<FileInfoVersions> {
2634 entry
2635 .file_info_versions(bucket)
2636 .map_err(|err| with_decommission_entry_context(stage, bucket, &entry.name, err))
2637}
2638
2639fn empty_decommission_entry_versions(bucket: &str, object: &str) -> FileInfoVersions {
2640 FileInfoVersions {
2641 volume: bucket.to_string(),
2642 name: object.to_string(),
2643 versions: Vec::new(),
2644 ..Default::default()
2645 }
2646}
2647
2648fn resolve_decommission_entry_exact_versions(
2649 result: Result<Option<FileInfoVersions>>,
2650 entry: &MetaCacheEntry,
2651 bucket: &str,
2652 stage: &str,
2653) -> Result<FileInfoVersions> {
2654 match result {
2655 Ok(Some(fivs)) => Ok(fivs),
2656 Ok(None) => Ok(empty_decommission_entry_versions(bucket, &entry.name)),
2657 Err(err) => Err(with_decommission_entry_context(stage, bucket, &entry.name, err)),
2658 }
2659}
2660
2661async fn load_decommission_entry_exact_versions(
2662 set: &SetDisks,
2663 entry: &MetaCacheEntry,
2664 bucket: &str,
2665 stage: &str,
2666) -> Result<FileInfoVersions> {
2667 resolve_decommission_entry_exact_versions(set.load_file_info_versions_exact(bucket, &entry.name).await, entry, bucket, stage)
2668}
2669
2670fn resolve_decommission_check_after_list_result(list_result: Result<()>, entry_error: Option<Error>) -> Result<()> {
2671 match list_result {
2672 Ok(()) => entry_error.map_or(Ok(()), Err),
2673 Err(list_err) => resolve_decommission_listing_error(Some(list_err), entry_error).map_or(Ok(()), Err),
2674 }
2675}
2676
2677fn resolve_decommission_listing_error(listing_error: Option<Error>, entry_error: Option<Error>) -> Option<Error> {
2678 match (listing_error, entry_error) {
2679 (Some(listing_error), Some(entry_error)) if is_err_operation_canceled(&listing_error) => Some(entry_error),
2680 (Some(listing_error), Some(entry_error)) if is_err_operation_canceled(&entry_error) => Some(listing_error),
2681 (Some(listing_error), _) => Some(listing_error),
2682 (None, entry_error) => entry_error,
2683 }
2684}
2685
2686fn decommission_unresolved_listing_error(entry: &DecommissionUnresolvedEntry) -> Error {
2687 Error::other(format!(
2688 "decommission listing could not resolve metadata for {bucket}/{object} on pool {pool_index} set {set_index} ({candidate_count} candidate(s), {disk_error_count} disk error(s))",
2689 bucket = entry.bucket,
2690 object = entry.object,
2691 pool_index = entry.pool_index,
2692 set_index = entry.set_index,
2693 candidate_count = entry.candidate_count,
2694 disk_error_count = entry.disk_error_count,
2695 ))
2696}
2697
2698#[allow(clippy::result_large_err)]
2701#[allow(clippy::too_many_arguments)]
2702fn resolve_decommission_partial_listing_entry(
2703 entries: MetaCacheEntries,
2704 resolver: MetadataResolutionParams,
2705 bucket: &str,
2706 prefix: &str,
2707 disk_error_count: usize,
2708 pool_index: usize,
2709 set_index: usize,
2710 source_generation: OffsetDateTime,
2711) -> std::result::Result<MetaCacheEntry, DecommissionUnresolvedEntry> {
2712 let candidate_count = entries.as_ref().iter().flatten().count();
2713 if let Some(entry) = entries.resolve(resolver) {
2714 return Ok(entry);
2715 }
2716
2717 let object = entries
2718 .as_ref()
2719 .iter()
2720 .flatten()
2721 .map(|entry| entry.name.as_str())
2722 .next()
2723 .unwrap_or(prefix)
2724 .to_string();
2725 Err(DecommissionUnresolvedEntry {
2726 bucket: bucket.to_string(),
2727 object,
2728 candidate_count,
2729 disk_error_count,
2730 pool_index,
2731 set_index,
2732 source_generation,
2733 observed_at: OffsetDateTime::now_utc(),
2734 reason: "metadata_resolution_failed".to_string(),
2735 })
2736}
2737
2738fn validate_decommission_durable_ilm_copy(
2739 path: &str,
2740 source_record: &ValidatedDurableIlmRecord,
2741 target: &[u8],
2742) -> Result<ValidatedDurableIlmRecord> {
2743 let target_record = validate_durable_ilm_record(path, target).map_err(|err| {
2744 Error::other(format!(
2745 "target durable ILM record is invalid at path `{path}` {}: {err}",
2746 source_record.context()
2747 ))
2748 })?;
2749 source_record
2750 .checkpoint
2751 .validate_successor(&target_record.checkpoint)
2752 .map_err(|err| {
2753 Error::other(format!(
2754 "target durable ILM record generation mismatch at path `{path}` {}: {err}",
2755 source_record.context()
2756 ))
2757 })?;
2758 Ok(target_record)
2759}
2760
2761#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
2762#[serde(deny_unknown_fields)]
2763struct DecommissionDurableIlmReceipt {
2764 source_path: String,
2765 namespace: String,
2766 id_kind: String,
2767 id: String,
2768 checkpoint: DurableIlmRecordCheckpoint,
2769 terminal_checkpoint: Option<DurableIlmRecordCheckpoint>,
2770 #[serde(default, skip_serializing_if = "Option::is_none")]
2771 fleet_topology_generation: Option<String>,
2772}
2773
2774impl DecommissionDurableIlmReceipt {
2775 fn new(path: &str, record: &ValidatedDurableIlmRecord, fleet_topology_generation: Option<String>) -> Self {
2776 Self {
2777 source_path: path.to_string(),
2778 namespace: record.namespace.to_string(),
2779 id_kind: record.id_kind.to_string(),
2780 id: record.id.clone(),
2781 checkpoint: record.checkpoint.clone(),
2782 terminal_checkpoint: None,
2783 fleet_topology_generation,
2784 }
2785 }
2786
2787 fn context(&self) -> String {
2788 format!("namespace `{}` {} `{}`", self.namespace, self.id_kind, self.id)
2789 }
2790
2791 fn validate(&self) -> Result<()> {
2792 let namespace = classify_durable_ilm_record(&self.source_path)?
2793 .ok_or_else(|| Error::other(format!("receipt source path `{}` is not a durable ILM record", self.source_path)))?;
2794 if namespace.name != self.namespace {
2795 return Err(Error::other(format!(
2796 "receipt namespace `{}` does not match source path `{}`",
2797 self.namespace, self.source_path
2798 )));
2799 }
2800 if self.id_kind.is_empty() || self.id.is_empty() {
2801 return Err(Error::other(format!(
2802 "receipt identity is missing for source path `{}`",
2803 self.source_path
2804 )));
2805 }
2806 if !is_sha256_checksum(self.checkpoint.content_sha256()) {
2807 return Err(Error::other(format!(
2808 "receipt target checksum is invalid for source path `{}` {}",
2809 self.source_path,
2810 self.context()
2811 )));
2812 }
2813 if let Some(terminal_checkpoint) = &self.terminal_checkpoint {
2814 self.checkpoint.validate_successor(terminal_checkpoint).map_err(|err| {
2815 Error::other(format!(
2816 "receipt terminal checkpoint is invalid for source path `{}` {}: {err}",
2817 self.source_path,
2818 self.context()
2819 ))
2820 })?;
2821 }
2822 if self
2823 .fleet_topology_generation
2824 .as_deref()
2825 .is_some_and(|generation| !is_sha256_checksum(generation))
2826 {
2827 return Err(Error::other_with_context(
2828 "receipt fleet topology generation is invalid",
2829 format!("source path `{}` {}", self.source_path, self.context()),
2830 ));
2831 }
2832 Ok(())
2833 }
2834
2835 fn encode(&self) -> Result<Vec<u8>> {
2836 let mut receipt = self.clone();
2837 receipt.checkpoint = receipt.checkpoint.compacted()?;
2838 receipt.terminal_checkpoint = receipt
2839 .terminal_checkpoint
2840 .as_ref()
2841 .map(DurableIlmRecordCheckpoint::compacted)
2842 .transpose()?;
2843 receipt.validate()?;
2844 let receipt_bytes = serde_json::to_vec(&receipt)?;
2845 let persisted = PersistedDecommissionDurableIlmReceipt {
2846 schema: DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA.to_string(),
2847 content_sha256: hex_sha256(&receipt_bytes, ToOwned::to_owned),
2848 receipt,
2849 };
2850 let encoded = serde_json::to_vec(&persisted)?;
2851 if encoded.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE {
2852 return Err(Error::other(format!(
2853 "durable ILM receipt exceeds maximum size for source path `{}` {}",
2854 self.source_path,
2855 self.context()
2856 )));
2857 }
2858 Ok(encoded)
2859 }
2860
2861 fn decode(data: &[u8]) -> Result<Self> {
2862 if data.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE {
2863 return Err(Error::other("durable ILM receipt exceeds maximum size"));
2864 }
2865 let persisted: PersistedDecommissionDurableIlmReceipt = serde_json::from_slice(data)?;
2866 if persisted.schema != DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA {
2867 return Err(Error::other(format!("unsupported durable ILM receipt schema `{}`", persisted.schema)));
2868 }
2869 if !is_sha256_checksum(&persisted.content_sha256) {
2870 return Err(Error::other("durable ILM receipt checksum is invalid"));
2871 }
2872 let receipt_bytes = serde_json::to_vec(&persisted.receipt)?;
2873 let actual_checksum = hex_sha256(&receipt_bytes, ToOwned::to_owned);
2874 if persisted.content_sha256 != actual_checksum {
2875 return Err(Error::other("durable ILM receipt checksum mismatch"));
2876 }
2877 persisted.receipt.validate()?;
2878 Ok(persisted.receipt)
2879 }
2880}
2881
2882fn merge_decommission_durable_ilm_receipts(
2883 existing: &DecommissionDurableIlmReceipt,
2884 incoming: &DecommissionDurableIlmReceipt,
2885) -> Result<DecommissionDurableIlmReceipt> {
2886 if existing.source_path != incoming.source_path
2887 || existing.namespace != incoming.namespace
2888 || existing.id_kind != incoming.id_kind
2889 || existing.id != incoming.id
2890 {
2891 return Err(Error::other(format!(
2892 "durable ILM receipt identity conflict for source path `{}` {}; incoming {}",
2893 existing.source_path,
2894 existing.context(),
2895 incoming.context()
2896 )));
2897 }
2898
2899 let checkpoint =
2900 if existing.checkpoint == incoming.checkpoint || incoming.checkpoint.validate_successor(&existing.checkpoint).is_ok() {
2901 existing.checkpoint.clone()
2902 } else {
2903 existing.checkpoint.validate_successor(&incoming.checkpoint).map_err(|err| {
2904 Error::other(format!(
2905 "durable ILM receipt checkpoint conflict for source path `{}` {}: {err}",
2906 existing.source_path,
2907 existing.context()
2908 ))
2909 })?;
2910 incoming.checkpoint.clone()
2911 };
2912 let terminal_checkpoint = match (&existing.terminal_checkpoint, &incoming.terminal_checkpoint) {
2913 (Some(existing_terminal), Some(incoming_terminal)) if existing_terminal == incoming_terminal => {
2914 Some(existing_terminal.clone())
2915 }
2916 (Some(existing_terminal), Some(incoming_terminal)) if incoming_terminal.validate_successor(existing_terminal).is_ok() => {
2917 Some(existing_terminal.clone())
2918 }
2919 (Some(existing_terminal), Some(incoming_terminal)) => {
2920 existing_terminal.validate_successor(incoming_terminal).map_err(|err| {
2921 Error::other(format!(
2922 "durable ILM receipt terminal checkpoint conflict for source path `{}` {}: {err}",
2923 existing.source_path,
2924 existing.context()
2925 ))
2926 })?;
2927 Some(incoming_terminal.clone())
2928 }
2929 (Some(existing_terminal), None) => Some(existing_terminal.clone()),
2930 (None, Some(incoming_terminal)) => Some(incoming_terminal.clone()),
2931 (None, None) => None,
2932 };
2933 let merged = DecommissionDurableIlmReceipt {
2934 source_path: existing.source_path.clone(),
2935 namespace: existing.namespace.clone(),
2936 id_kind: existing.id_kind.clone(),
2937 id: existing.id.clone(),
2938 checkpoint,
2939 terminal_checkpoint,
2940 fleet_topology_generation: match (
2941 existing.fleet_topology_generation.as_ref(),
2942 incoming.fleet_topology_generation.as_ref(),
2943 ) {
2944 (Some(existing_generation), Some(incoming_generation)) if existing_generation == incoming_generation => {
2945 Some(existing_generation.clone())
2946 }
2947 (None, None) => None,
2948 _ => {
2949 return Err(Error::other_with_context(
2950 "durable ILM receipt fleet topology conflict",
2951 format!("source path `{}` {}", existing.source_path, existing.context()),
2952 ));
2953 }
2954 },
2955 };
2956 merged.validate()?;
2957 Ok(merged)
2958}
2959
2960#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
2961#[serde(deny_unknown_fields)]
2962struct PersistedDecommissionDurableIlmReceipt {
2963 schema: String,
2964 content_sha256: String,
2965 receipt: DecommissionDurableIlmReceipt,
2966}
2967
2968#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
2969#[serde(deny_unknown_fields)]
2970struct DecommissionDurableIlmManifest {
2971 schema: String,
2972 run_token: String,
2973 receipt_count: u64,
2974 receipt_paths_sha256: String,
2975}
2976
2977impl DecommissionDurableIlmManifest {
2978 fn new(run_token: &str, receipt_paths: &[String]) -> Result<Self> {
2979 let manifest = Self {
2980 schema: DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA.to_string(),
2981 run_token: run_token.to_string(),
2982 receipt_count: u64::try_from(receipt_paths.len())
2983 .map_err(|_| Error::other("durable ILM expected manifest receipt count exceeds u64"))?,
2984 receipt_paths_sha256: decommission_durable_ilm_manifest_paths_sha256(receipt_paths)?,
2985 };
2986 manifest.validate(run_token, receipt_paths)?;
2987 Ok(manifest)
2988 }
2989
2990 fn validate(&self, run_token: &str, receipt_paths: &[String]) -> Result<()> {
2991 if self.schema != DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA {
2992 return Err(Error::other(format!(
2993 "unsupported durable ILM expected manifest schema `{}`",
2994 self.schema
2995 )));
2996 }
2997 if self.run_token != run_token || !is_sha256_checksum(&self.run_token) {
2998 return Err(Error::other("durable ILM expected manifest run token is invalid"));
2999 }
3000 let receipt_count = u64::try_from(receipt_paths.len())
3001 .map_err(|_| Error::other("durable ILM expected manifest receipt count exceeds u64"))?;
3002 if self.receipt_count != receipt_count {
3003 return Err(Error::other(format!(
3004 "durable ILM expected manifest receipt count mismatch: expected {}, found {receipt_count}",
3005 self.receipt_count
3006 )));
3007 }
3008 if !is_sha256_checksum(&self.receipt_paths_sha256)
3009 || self.receipt_paths_sha256 != decommission_durable_ilm_manifest_paths_sha256(receipt_paths)?
3010 {
3011 return Err(Error::other("durable ILM expected manifest receipt paths checksum mismatch"));
3012 }
3013 Ok(())
3014 }
3015
3016 fn encode(&self) -> Result<Vec<u8>> {
3017 let encoded = serde_json::to_vec(self)?;
3018 if encoded.len() > DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE {
3019 return Err(Error::other("durable ILM expected manifest exceeds maximum size"));
3020 }
3021 Ok(encoded)
3022 }
3023
3024 fn decode(data: &[u8], run_token: &str, receipt_paths: &[String]) -> Result<Self> {
3025 if data.len() > DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE {
3026 return Err(Error::other("durable ILM expected manifest exceeds maximum size"));
3027 }
3028 let manifest: Self = serde_json::from_slice(data)?;
3029 manifest.validate(run_token, receipt_paths)?;
3030 Ok(manifest)
3031 }
3032}
3033
3034#[derive(Debug, Clone, PartialEq, Eq)]
3035struct DecommissionDurableIlmReceiptLocator {
3036 run_token: String,
3037 source_path: String,
3038 id_kind: String,
3039 id: String,
3040}
3041
3042impl DecommissionDurableIlmReceiptLocator {
3043 fn context(&self) -> String {
3044 format!("source path `{}` {} `{}`", self.source_path, self.id_kind, self.id)
3045 }
3046}
3047
3048fn decommission_durable_ilm_receipt_run_token(cmd_line: &str, start_time: OffsetDateTime) -> String {
3049 let identity = format!("{cmd_line}\0{}", start_time.unix_timestamp_nanos());
3050 hex_sha256(identity.as_bytes(), ToOwned::to_owned)
3051}
3052
3053fn decommission_durable_ilm_receipt_run_prefix(run_token: &str) -> String {
3054 format!("{DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT}/{run_token}/")
3055}
3056
3057fn decommission_durable_ilm_receipt_path(run_token: &str, source_path: &str, id_kind: &str, id: &str) -> String {
3058 format!(
3059 "{}{}/{}/{}.json",
3060 decommission_durable_ilm_receipt_run_prefix(run_token),
3061 source_path,
3062 id_kind,
3063 id
3064 )
3065}
3066
3067fn decommission_durable_ilm_manifest_path(run_token: &str) -> String {
3068 format!("{DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT}/{run_token}.json")
3069}
3070
3071fn decommission_durable_ilm_manifest_paths_sha256(receipt_paths: &[String]) -> Result<String> {
3072 let mut sorted_paths = receipt_paths.iter().map(String::as_str).collect::<Vec<_>>();
3073 sorted_paths.sort_unstable();
3074 let encoded = serde_json::to_vec(&sorted_paths)?;
3075 Ok(hex_sha256(&encoded, ToOwned::to_owned))
3076}
3077
3078fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result<DecommissionDurableIlmReceiptLocator> {
3079 let prefix = format!("{DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT}/");
3080 let suffix = path
3081 .strip_prefix(&prefix)
3082 .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has the wrong root")))?;
3083 let (run_token, record_path) = suffix
3084 .split_once('/')
3085 .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record path")))?;
3086 let mut parts = record_path.rsplitn(3, '/');
3087 let id = parts
3088 .next()
3089 .and_then(|file| file.strip_suffix(".json"))
3090 .filter(|id| !id.is_empty())
3091 .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record id")))?;
3092 let id_kind = parts
3093 .next()
3094 .filter(|id_kind| matches!(*id_kind, "operation_id" | "transaction_id" | "job_id" | "control_id"))
3095 .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has an invalid id kind")))?;
3096 let source_path = parts
3097 .next()
3098 .filter(|source_path| !source_path.is_empty())
3099 .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its source path")))?;
3100 if !is_sha256_checksum(run_token) {
3101 return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid run token")));
3102 }
3103 match id_kind {
3104 "operation_id" | "control_id" if !is_sha256_checksum(id) => {
3105 let id_label = id_kind.trim_end_matches("_id");
3106 return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid {id_label} id")));
3107 }
3108 "transaction_id" | "job_id" if uuid::Uuid::parse_str(id).is_err() => {
3109 return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid UUID")));
3110 }
3111 _ => {}
3112 }
3113 classify_durable_ilm_record(source_path)?
3114 .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` does not identify a durable ILM source path")))?;
3115 Ok(DecommissionDurableIlmReceiptLocator {
3116 run_token: run_token.to_string(),
3117 source_path: source_path.to_string(),
3118 id_kind: id_kind.to_string(),
3119 id: id.to_string(),
3120 })
3121}
3122
3123fn resolve_decommission_pool_meta_reload_result(result: Result<()>, stage: &str) -> Result<()> {
3124 result.map_err(|err| Error::other(format!("decommission pool meta reload failed during {stage}: {err}")))
3125}
3126
3127fn apply_decommission_status_space_info(mut pool_info: PoolStatus, space_info: PoolSpaceInfo) -> PoolStatus {
3128 match pool_info.decommission.as_mut() {
3129 Some(d) => {
3130 d.total_size = space_info.total;
3131 d.current_size = space_info.free;
3132 }
3133 None => {
3134 pool_info.decommission = Some(PoolDecommissionInfo {
3135 total_size: space_info.total,
3136 current_size: space_info.free,
3137 ..Default::default()
3138 });
3139 }
3140 }
3141
3142 pool_info
3143}
3144
3145fn should_replace_pool_status_for_status_refresh(
3146 current: Option<&PoolStatus>,
3147 persisted: &PoolStatus,
3148 has_active_worker: bool,
3149) -> bool {
3150 let Some(current) = current else {
3151 return true;
3152 };
3153
3154 !has_active_worker && persisted.last_update > current.last_update
3155}
3156
3157fn pool_decommission_movement_snapshot(
3158 info: Option<&PoolDecommissionInfo>,
3159) -> (bool, bool, bool, bool, bool, Option<OffsetDateTime>) {
3160 info.map(|info| {
3161 (
3162 info.has_decommission_state(),
3163 info.complete,
3164 info.failed,
3165 info.canceled,
3166 info.queued,
3167 info.start_time,
3168 )
3169 })
3170 .unwrap_or_default()
3171}
3172
3173pub(crate) fn pool_meta_movement_snapshot_changed(before: &PoolMeta, after: &PoolMeta) -> bool {
3174 before.pools.len() != after.pools.len()
3175 || before.pools.iter().zip(after.pools.iter()).any(|(before, after)| {
3176 pool_decommission_movement_snapshot(before.decommission.as_ref())
3177 != pool_decommission_movement_snapshot(after.decommission.as_ref())
3178 })
3179}
3180
3181pub(crate) fn merge_pool_status_refresh(current: &mut PoolMeta, persisted: PoolMeta, active_workers: &[bool]) -> bool {
3187 let observed_version = current.version.max(persisted.version);
3188 if persisted.pools.is_empty() {
3189 current.version = observed_version;
3190 return false;
3191 }
3192
3193 if current.pools.is_empty() {
3194 *current = persisted;
3195 current.version = observed_version;
3196 return true;
3197 }
3198
3199 current.version = observed_version;
3200
3201 let mut merged_newer = false;
3202 for (idx, persisted_pool) in persisted.pools.into_iter().enumerate() {
3203 if persisted_pool.id != idx {
3204 continue;
3205 }
3206
3207 let has_active_worker = active_workers.get(idx).copied().unwrap_or(false);
3208 if idx < current.pools.len() {
3209 if should_replace_pool_status_for_status_refresh(current.pools.get(idx), &persisted_pool, has_active_worker) {
3210 current.pools[idx] = persisted_pool;
3211 merged_newer = true;
3212 }
3213 } else if idx == current.pools.len() && !has_active_worker {
3214 current.pools.push(persisted_pool);
3215 merged_newer = true;
3216 }
3217 }
3218 merged_newer
3219}
3220
3221fn merge_pool_meta_updates_for_save(
3222 persisted: &mut PoolMeta,
3223 current: &PoolMeta,
3224 indices: &[usize],
3225 operation: &str,
3226) -> Result<()> {
3227 if persisted.pools.is_empty() {
3228 *persisted = current.clone();
3229 return Ok(());
3230 }
3231 if persisted.version != current.version {
3232 return Err(Error::other(format!(
3233 "{operation}: pool metadata version changed from {} to {}",
3234 current.version, persisted.version
3235 )));
3236 }
3237
3238 for &idx in indices {
3239 let current_pool = current
3240 .pools
3241 .get(idx)
3242 .ok_or_else(|| invalid_decommission_pool_index_error(current.pools.len(), idx))?;
3243 let persisted_count = persisted.pools.len();
3244 let persisted_pool = persisted
3245 .pools
3246 .get_mut(idx)
3247 .ok_or_else(|| invalid_decommission_pool_index_error(persisted_count, idx))?;
3248 if current_pool.id != idx || persisted_pool.id != idx || current_pool.cmd_line != persisted_pool.cmd_line {
3249 return Err(Error::other(format!("{operation}: pool metadata layout changed for pool {idx}")));
3250 }
3251 let current_clears_decommission = current_pool
3252 .decommission
3253 .as_ref()
3254 .is_none_or(|info| !info.has_decommission_state());
3255 if current_clears_decommission
3256 && persisted_pool.decommission.as_ref().is_some_and(|info| {
3257 info.has_decommission_state() && is_decommission_active(info.complete, info.failed, info.canceled)
3258 })
3259 {
3260 record_pool_meta_stale_write_rejection("nonterminal_decommission_clear");
3261 return Err(Error::other(format!(
3262 "{operation}: stale pool metadata update rejected for pool {idx}; persisted active or queued decommission cannot be cleared"
3263 )));
3264 }
3265 if current_clears_decommission
3266 && persisted_pool
3267 .decommission
3268 .as_ref()
3269 .is_some_and(|info| info.complete || !info.unresolved_entries.is_empty())
3270 {
3271 record_pool_meta_stale_write_rejection("unsafe_terminal_decommission_clear");
3272 return Err(Error::StalePoolMetadataUpdate {
3273 operation: operation.to_string(),
3274 pool_index: idx,
3275 reason: "completed or unresolved decommission state cannot be cleared",
3276 });
3277 }
3278 if current_pool.last_update < persisted_pool.last_update {
3279 record_pool_meta_stale_write_rejection("older_pool_revision");
3280 return Err(Error::other(format!(
3281 "{operation}: stale pool metadata update rejected for pool {idx}; persisted update is newer"
3282 )));
3283 }
3284 if let (Some(persisted_info), Some(current_info)) =
3285 (persisted_pool.decommission.as_ref(), current_pool.decommission.as_ref())
3286 && current_info.has_decommission_state()
3287 {
3288 let persisted_terminal = (persisted_info.complete, persisted_info.failed, persisted_info.canceled);
3289 let current_terminal = (current_info.complete, current_info.failed, current_info.canceled);
3290 if persisted_terminal != (false, false, false) && persisted_terminal != current_terminal {
3291 record_pool_meta_stale_write_rejection("terminal_state_regression");
3292 return Err(Error::other(format!(
3293 "{operation}: stale pool metadata update rejected for pool {idx}; terminal state cannot be replaced"
3294 )));
3295 }
3296 if current_info.items_decommissioned < persisted_info.items_decommissioned
3297 || current_info.items_decommission_failed < persisted_info.items_decommission_failed
3298 || current_info.bytes_done < persisted_info.bytes_done
3299 || current_info.bytes_failed < persisted_info.bytes_failed
3300 {
3301 record_pool_meta_stale_write_rejection("progress_regression");
3302 return Err(Error::other(format!(
3303 "{operation}: stale pool metadata update rejected for pool {idx}; durable progress cannot decrease"
3304 )));
3305 }
3306 }
3307 *persisted_pool = current_pool.clone();
3308 }
3309
3310 Ok(())
3311}
3312
3313fn publish_pool_meta_updates(current: &mut PoolMeta, saved: &PoolMeta, indices: &[usize]) {
3314 current.version = current.version.max(saved.version);
3315 for &idx in indices {
3316 let Some(saved_pool) = saved.pools.get(idx) else {
3317 continue;
3318 };
3319 let Some(current_pool) = current.pools.get_mut(idx) else {
3320 continue;
3321 };
3322 if current_pool.id == idx && saved_pool.id == idx && current_pool.cmd_line == saved_pool.cmd_line {
3323 *current_pool = saved_pool.clone();
3324 }
3325 }
3326}
3327
3328fn publish_decommission_capacity_update(current: &mut PoolMeta, saved: &PoolMeta, idx: usize) -> Result<()> {
3331 let stale = |reason| Error::StalePoolMetadataUpdate {
3332 operation: "publish decommission capacity".to_string(),
3333 pool_index: idx,
3334 reason,
3335 };
3336 let saved_pool = saved
3337 .pools
3338 .get(idx)
3339 .ok_or_else(|| invalid_decommission_pool_index_error(saved.pools.len(), idx))?;
3340 let current_count = current.pools.len();
3341 let pool = current
3342 .pools
3343 .get_mut(idx)
3344 .ok_or_else(|| invalid_decommission_pool_index_error(current_count, idx))?;
3345 if pool.id != idx || saved_pool.id != idx || pool.cmd_line != saved_pool.cmd_line || saved.version < current.version {
3346 return Err(stale("pool identity changed before capacity publication"));
3347 }
3348 let info = pool
3349 .decommission
3350 .as_mut()
3351 .ok_or_else(|| stale("decommission cleared before capacity publication"))?;
3352 let saved_info = saved_pool
3353 .decommission
3354 .as_ref()
3355 .ok_or_else(|| stale("persisted decommission is missing during capacity publication"))?;
3356 if (info.start_time, info.queued, info.complete, info.failed, info.canceled)
3357 != (
3358 saved_info.start_time,
3359 saved_info.queued,
3360 saved_info.complete,
3361 saved_info.failed,
3362 saved_info.canceled,
3363 )
3364 {
3365 return Err(stale("decommission generation or terminal state changed before capacity publication"));
3366 }
3367 let identity = |reservation: &DecommissionCapacityReservation| {
3368 (
3369 reservation.source_pool_index,
3370 reservation.operation_id,
3371 reservation.generation,
3372 reservation.owner_nonce,
3373 reservation.model_version,
3374 )
3375 };
3376 if info.capacity_reservation.as_ref().map(identity) != saved_info.capacity_reservation.as_ref().map(identity) {
3377 return Err(stale("decommission capacity owner changed before publication"));
3378 }
3379
3380 info.progress_save_last_at.get_or_insert(pool.last_update);
3383 info.capacity_reservation = saved_info.capacity_reservation.clone();
3384 info.capacity_blocked_reason = saved_info.capacity_blocked_reason.clone();
3385 pool.last_update = pool.last_update.max(saved_pool.last_update);
3386 current.version = current.version.max(saved.version);
3387 Ok(())
3388}
3389
3390fn resolve_start_decommission_pool_meta_reload_result(result: Result<()>) -> Result<()> {
3391 resolve_decommission_pool_meta_reload_result(result, "start_decommission")
3392}
3393
3394fn activation_rebalance_meta_lock_error(err: rustfs_lock::LockError) -> Error {
3395 match err {
3396 rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
3397 mode: "write",
3398 bucket: RUSTFS_META_BUCKET.to_string(),
3399 object: REBAL_META_NAME.to_string(),
3400 required,
3401 achieved,
3402 },
3403 other => Error::other(format!(
3404 "failed to acquire rebalance activation lock on {RUSTFS_META_BUCKET}/{REBAL_META_NAME}: {other}"
3405 )),
3406 }
3407}
3408
3409fn activation_pool_meta_lock_error(err: rustfs_lock::LockError) -> Error {
3410 match err {
3411 rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
3412 mode: "write",
3413 bucket: RUSTFS_META_BUCKET.to_string(),
3414 object: POOL_META_NAME.to_string(),
3415 required,
3416 achieved,
3417 },
3418 other => Error::other(format!(
3419 "failed to acquire pool activation lock on {RUSTFS_META_BUCKET}/{POOL_META_NAME}: {other}"
3420 )),
3421 }
3422}
3423
3424pub(crate) struct PoolRebalanceActivationFence {
3425 pool_meta_guard: rustfs_lock::NamespaceLockGuard,
3426 rebalance_meta_guard: rustfs_lock::NamespaceLockGuard,
3427 fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
3428 decommission_target_fence_proof: Option<crate::services::notification_sys::DecommissionTargetFenceFleetProofToken>,
3429 #[cfg(test)]
3430 forced_lost: Arc<AtomicBool>,
3431}
3432
3433impl PoolRebalanceActivationFence {
3434 pub(crate) fn set_fleet_proof(
3435 &mut self,
3436 fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
3437 ) {
3438 self.fleet_proof = fleet_proof;
3439 }
3440
3441 fn set_decommission_target_fence_proof(
3442 &mut self,
3443 fleet_proof: Option<crate::services::notification_sys::DecommissionTargetFenceFleetProofToken>,
3444 ) {
3445 self.decommission_target_fence_proof = fleet_proof;
3446 }
3447
3448 pub(crate) fn ensure_held(&self) -> Result<()> {
3449 #[cfg(test)]
3450 let forced_lost = self.forced_lost.load(Ordering::Acquire);
3451 #[cfg(not(test))]
3452 let forced_lost = false;
3453 if forced_lost || self.pool_meta_guard.is_lock_lost() || self.rebalance_meta_guard.is_lock_lost() {
3454 return Err(Error::other("activation lock lost before metadata commit or worker admission"));
3455 }
3456 if self
3457 .fleet_proof
3458 .as_ref()
3459 .is_some_and(|proof| !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof))
3460 {
3461 return Err(Error::other(POOL_ACTIVATION_FLEET_PROOF_EXPIRED));
3462 }
3463 if self
3464 .decommission_target_fence_proof
3465 .as_ref()
3466 .is_some_and(|proof| !crate::services::notification_sys::decommission_target_fence_fleet_proof_matches(proof))
3467 {
3468 return Err(Error::other(DECOMMISSION_TARGET_FLEET_PROOF_EXPIRED));
3469 }
3470
3471 Ok(())
3472 }
3473
3474 pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
3475 opts.add_namespace_lock_guard(&self.pool_meta_guard);
3476 opts.add_namespace_lock_guard(&self.rebalance_meta_guard);
3477 }
3478
3479 #[cfg(test)]
3480 fn force_lost_for_test(&self) {
3481 self.forced_lost.store(true, Ordering::Release);
3482 }
3483}
3484
3485#[cfg(test)]
3486tokio::task_local! {
3487 pub(crate) static REBALANCE_ACTIVATION_LOCK_ATTEMPT: Arc<tokio::sync::Notify>;
3488}
3489
3490pub(crate) async fn acquire_pool_rebalance_activation_locks<S>(
3491 pool: Arc<S>,
3492 fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
3493) -> Result<PoolRebalanceActivationFence>
3494where
3495 S: crate::storage_api_contracts::namespace::NamespaceLocking<
3496 Error = Error,
3497 NamespaceLock = rustfs_lock::NamespaceLockWrapper,
3498 >,
3499{
3500 let rebalance_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
3504 #[cfg(test)]
3505 let _ = REBALANCE_ACTIVATION_LOCK_ATTEMPT.try_with(|attempted| attempted.notify_one());
3506 let rebalance_meta_guard = rebalance_meta_lock
3507 .get_write_lock(get_lock_acquire_timeout())
3508 .await
3509 .map_err(activation_rebalance_meta_lock_error)?;
3510 let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
3511 let pool_meta_guard = pool_meta_lock
3512 .get_write_lock(get_lock_acquire_timeout())
3513 .await
3514 .map_err(activation_pool_meta_lock_error)?;
3515
3516 Ok(PoolRebalanceActivationFence {
3517 pool_meta_guard,
3518 rebalance_meta_guard,
3519 fleet_proof,
3520 decommission_target_fence_proof: None,
3521 #[cfg(test)]
3522 forced_lost: Arc::new(AtomicBool::new(false)),
3523 })
3524}
3525
3526pub(crate) async fn acquire_pool_activation_fleet_proof(
3527 ctx: &crate::runtime::instance::InstanceContext,
3528) -> Result<Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>> {
3529 if !ctx.is_dist_erasure().await {
3530 return Ok(None);
3531 }
3532 crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof()
3533 .map(Some)
3534 .ok_or_else(|| Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED))
3535}
3536
3537pub fn is_pool_activation_fleet_proof_error(err: &Error) -> bool {
3538 matches!(err, Error::Io(io_error) if io_error.kind() == std::io::ErrorKind::Other && {
3542 let message = io_error.to_string();
3543 message.contains(POOL_ACTIVATION_FLEET_PROOF_REQUIRED) || message.contains(POOL_ACTIVATION_FLEET_PROOF_EXPIRED)
3544 })
3545}
3546
3547#[cfg(test)]
3548#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3549pub(crate) enum PoolActivationStartKind {
3550 Rebalance,
3551 Decommission,
3552}
3553
3554#[cfg(test)]
3555struct PoolActivationDurableSaveBarrierState {
3556 pool_key: usize,
3557 arrived: tokio::sync::Notify,
3558 release: tokio::sync::Notify,
3559 force_fence_loss: AtomicBool,
3560}
3561
3562#[cfg(test)]
3563static POOL_ACTIVATION_DURABLE_SAVE_BARRIER: std::sync::OnceLock<
3564 std::sync::Mutex<Option<Arc<PoolActivationDurableSaveBarrierState>>>,
3565> = std::sync::OnceLock::new();
3566
3567#[cfg(test)]
3568pub(crate) struct PoolActivationDurableSaveBarrier {
3569 state: Arc<PoolActivationDurableSaveBarrierState>,
3570}
3571
3572#[cfg(test)]
3573fn pool_activation_test_pool_key<S>(pool: &Arc<S>) -> usize {
3574 Arc::as_ptr(pool).cast::<()>() as usize
3575}
3576
3577#[cfg(test)]
3578impl PoolActivationDurableSaveBarrier {
3579 pub(crate) fn install<S>(pool: &Arc<S>) -> Self {
3580 let state = Arc::new(PoolActivationDurableSaveBarrierState {
3581 pool_key: pool_activation_test_pool_key(pool),
3582 arrived: tokio::sync::Notify::new(),
3583 release: tokio::sync::Notify::new(),
3584 force_fence_loss: AtomicBool::new(false),
3585 });
3586 let mut barrier = POOL_ACTIVATION_DURABLE_SAVE_BARRIER
3587 .get_or_init(|| std::sync::Mutex::new(None))
3588 .lock()
3589 .expect("pool activation durable save barrier should not be poisoned");
3590 assert!(barrier.is_none(), "pool activation durable save barrier must be unique");
3591 *barrier = Some(Arc::clone(&state));
3592 Self { state }
3593 }
3594
3595 pub(crate) async fn wait_until_paused(&self) {
3596 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.arrived.notified())
3597 .await
3598 .expect("activation should reach the post-durable-save barrier");
3599 }
3600
3601 pub(crate) fn release_after_fence_loss(&self) {
3602 self.state.force_fence_loss.store(true, Ordering::Release);
3603 self.state.release.notify_one();
3604 }
3605
3606 pub(crate) fn release_without_fence_loss(&self) {
3607 self.state.release.notify_one();
3608 }
3609}
3610
3611#[cfg(test)]
3612impl Drop for PoolActivationDurableSaveBarrier {
3613 fn drop(&mut self) {
3614 self.state.release.notify_one();
3615 let mut barrier = POOL_ACTIVATION_DURABLE_SAVE_BARRIER
3616 .get_or_init(|| std::sync::Mutex::new(None))
3617 .lock()
3618 .expect("pool activation durable save barrier should not be poisoned");
3619 if barrier.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
3620 *barrier = None;
3621 }
3622 }
3623}
3624
3625#[cfg(test)]
3626pub(crate) async fn pause_pool_activation_after_durable_save<S>(pool: &Arc<S>, fence: &PoolRebalanceActivationFence) {
3627 let pool_key = pool_activation_test_pool_key(pool);
3628 let barrier = {
3629 let mut barrier = POOL_ACTIVATION_DURABLE_SAVE_BARRIER
3630 .get_or_init(|| std::sync::Mutex::new(None))
3631 .lock()
3632 .expect("pool activation durable save barrier should not be poisoned");
3633 if barrier.as_ref().is_some_and(|state| state.pool_key == pool_key) {
3634 barrier.take()
3635 } else {
3636 None
3637 }
3638 };
3639 if let Some(barrier) = barrier {
3640 barrier.arrived.notify_one();
3641 barrier.release.notified().await;
3642 if barrier.force_fence_loss.load(Ordering::Acquire) {
3643 fence.force_lost_for_test();
3644 }
3645 }
3646}
3647
3648#[cfg(test)]
3649struct PoolActivationStartProbeState {
3650 kind: PoolActivationStartKind,
3651 preflight_side_effect_attempted: std::sync::atomic::AtomicBool,
3652 attempted: std::sync::atomic::AtomicBool,
3653 notify: tokio::sync::Notify,
3654}
3655
3656#[cfg(test)]
3657static POOL_ACTIVATION_START_PROBES: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<PoolActivationStartProbeState>>>> =
3658 std::sync::OnceLock::new();
3659
3660#[cfg(test)]
3661pub(crate) struct PoolActivationStartProbe {
3662 state: Arc<PoolActivationStartProbeState>,
3663}
3664
3665#[cfg(test)]
3666impl PoolActivationStartProbe {
3667 pub(crate) fn install(kind: PoolActivationStartKind) -> Self {
3668 let state = Arc::new(PoolActivationStartProbeState {
3669 kind,
3670 preflight_side_effect_attempted: std::sync::atomic::AtomicBool::new(false),
3671 attempted: std::sync::atomic::AtomicBool::new(false),
3672 notify: tokio::sync::Notify::new(),
3673 });
3674 POOL_ACTIVATION_START_PROBES
3675 .get_or_init(|| std::sync::Mutex::new(Vec::new()))
3676 .lock()
3677 .expect("pool activation start probe should not be poisoned")
3678 .push(Arc::clone(&state));
3679 Self { state }
3680 }
3681
3682 pub(crate) async fn wait_until_attempted(&self) {
3683 while !self.state.attempted.load(Ordering::Acquire) {
3684 self.state.notify.notified().await;
3685 }
3686 }
3687
3688 pub(crate) fn preflight_side_effect_was_attempted(&self) -> bool {
3689 self.state.preflight_side_effect_attempted.load(Ordering::Acquire)
3690 }
3691
3692 pub(crate) fn activation_was_attempted(&self) -> bool {
3693 self.state.attempted.load(Ordering::Acquire)
3694 }
3695}
3696
3697#[cfg(test)]
3698impl Drop for PoolActivationStartProbe {
3699 fn drop(&mut self) {
3700 let mut probes = POOL_ACTIVATION_START_PROBES
3701 .get_or_init(|| std::sync::Mutex::new(Vec::new()))
3702 .lock()
3703 .expect("pool activation start probe should not be poisoned");
3704 probes.retain(|state| !Arc::ptr_eq(state, &self.state));
3705 }
3706}
3707
3708#[cfg(test)]
3709pub(crate) fn observe_pool_activation_start_attempt(kind: PoolActivationStartKind) {
3710 let probes = POOL_ACTIVATION_START_PROBES
3711 .get_or_init(|| std::sync::Mutex::new(Vec::new()))
3712 .lock()
3713 .expect("pool activation start probe should not be poisoned")
3714 .iter()
3715 .filter(|state| state.kind == kind)
3716 .cloned()
3717 .collect::<Vec<_>>();
3718 for state in probes {
3719 state.attempted.store(true, Ordering::Release);
3720 state.notify.notify_one();
3721 }
3722}
3723
3724#[cfg(test)]
3725fn observe_pool_activation_preflight_side_effect_attempt(kind: PoolActivationStartKind) {
3726 let probes = POOL_ACTIVATION_START_PROBES
3727 .get_or_init(|| std::sync::Mutex::new(Vec::new()))
3728 .lock()
3729 .expect("pool activation start probe should not be poisoned")
3730 .iter()
3731 .filter(|state| state.kind == kind)
3732 .cloned()
3733 .collect::<Vec<_>>();
3734 for state in probes {
3735 state.preflight_side_effect_attempted.store(true, Ordering::Release);
3736 }
3737}
3738
3739fn rollback_decommission_pool_meta(pool_meta: &mut PoolMeta, previous_pool_meta: &PoolMeta, indices: &[usize]) {
3740 publish_pool_meta_updates(pool_meta, previous_pool_meta, indices);
3741}
3742
3743#[derive(Debug)]
3744struct DecommissionCancelCommit {
3745 previous_start_time: Option<OffsetDateTime>,
3746 previous_queued: bool,
3747 previous_last_update: OffsetDateTime,
3748 canceled_pool: PoolStatus,
3749}
3750
3751fn commit_decommission_cancel(pool_meta: &mut PoolMeta, idx: usize, commit: DecommissionCancelCommit) -> Result<()> {
3752 let pool_count = pool_meta.pools.len();
3753 let Some(current) = pool_meta.pools.get(idx) else {
3754 return Err(invalid_decommission_pool_index_error(pool_count, idx));
3755 };
3756 let cancel_already_published = commit
3758 .canceled_pool
3759 .decommission
3760 .as_ref()
3761 .is_some_and(|info| info.canceled && !info.complete && !info.failed && info.start_time.is_none())
3762 && PersistedPoolStatus::from(current) == PersistedPoolStatus::from(&commit.canceled_pool);
3763 if cancel_already_published {
3764 return Ok(());
3765 }
3766
3767 let matches_generation = current.id == commit.canceled_pool.id
3768 && current.cmd_line == commit.canceled_pool.cmd_line
3769 && current.decommission.as_ref().is_some_and(|info| {
3770 info.start_time == commit.previous_start_time
3771 && info.queued == commit.previous_queued
3772 && is_decommission_active(info.complete, info.failed, info.canceled)
3773 && (commit.previous_start_time.is_some() || current.last_update == commit.previous_last_update)
3774 });
3775 if !matches_generation {
3776 return Err(Error::other(format!(
3777 "failed to publish decommission cancel for pool {idx}: operation generation changed"
3778 )));
3779 }
3780
3781 pool_meta.pools[idx] = commit.canceled_pool;
3782 Ok(())
3783}
3784
3785fn rollback_start_decommission_pool_meta(pool_meta: &mut PoolMeta, previous_pool_meta: &PoolMeta, indices: &[usize]) {
3786 let active_updates = indices
3787 .iter()
3788 .filter_map(|&idx| pool_meta.pools.get(idx).map(|pool| (idx, pool.last_update)))
3789 .collect::<Vec<_>>();
3790 rollback_decommission_pool_meta(pool_meta, previous_pool_meta, indices);
3791 let rollback_at = OffsetDateTime::now_utc();
3792 for (idx, active_update) in active_updates {
3793 if let Some(pool) = pool_meta.pools.get_mut(idx) {
3794 pool.last_update = std::cmp::max(rollback_at, active_update + Duration::nanoseconds(1));
3795 }
3796 }
3797}
3798
3799fn ensure_pool_meta_write_fence(guard: &rustfs_lock::NamespaceLockGuard, operation: &str) -> Result<()> {
3800 if guard.is_lock_lost() {
3801 return Err(Error::other(format!("{operation}: pool metadata distributed fence was lost")));
3802 }
3803 Ok(())
3804}
3805
3806fn ensure_pool_not_left_in_cmdline_after_decommission(position: usize, cmd_line: &str, completed: bool) -> Result<()> {
3807 if completed {
3808 return Err(Error::other(format!(
3809 "pool({}) = {} is decommissioned, please remove from server command line",
3810 position + 1,
3811 cmd_line
3812 )));
3813 }
3814
3815 Ok(())
3816}
3817
3818fn resolve_decommission_listing_worker_result(
3819 set_idx: usize,
3820 worker_result: std::result::Result<Result<()>, tokio::task::JoinError>,
3821) -> Result<()> {
3822 worker_result.map_err(|err| Error::other(format!("decommission listing worker {set_idx} task join error: {err}")))?
3823}
3824
3825fn should_retry_decommission_listing(err: &Error, attempt: usize, max_attempts: usize) -> bool {
3826 !is_err_bucket_not_found(err) && attempt + 1 < max_attempts
3827}
3828
3829async fn wait_decommission_retry_backoff(rx: &CancellationToken, delay: std::time::Duration) -> bool {
3830 tokio::select! {
3831 _ = rx.cancelled() => true,
3832 _ = tokio::time::sleep(delay) => false,
3833 }
3834}
3835
3836fn decommission_retry_backoff_delay(base: std::time::Duration, attempt: usize) -> std::time::Duration {
3837 base.saturating_mul(u32::try_from(attempt).unwrap_or(u32::MAX))
3838}
3839
3840#[cfg(test)]
3841async fn run_decommission_listing_with_retry<List, ListFuture>(
3842 rx: CancellationToken,
3843 bucket: String,
3844 cb: ListCallback,
3845 pool_idx: usize,
3846 set_idx: usize,
3847 max_attempts: usize,
3848 list: List,
3849) -> Result<()>
3850where
3851 List: FnMut(ListCallback) -> ListFuture,
3852 ListFuture: std::future::Future<Output = Result<()>>,
3853{
3854 run_decommission_listing_with_retry_and_drain(rx, bucket, cb, pool_idx, set_idx, max_attempts, list, || async { false }).await
3855}
3856
3857#[allow(clippy::too_many_arguments)]
3858async fn run_decommission_listing_with_retry_and_drain<List, ListFuture, Drain, DrainFuture>(
3859 rx: CancellationToken,
3860 bucket: String,
3861 cb: ListCallback,
3862 pool_idx: usize,
3863 set_idx: usize,
3864 max_attempts: usize,
3865 mut list: List,
3866 mut drain: Drain,
3867) -> Result<()>
3868where
3869 List: FnMut(ListCallback) -> ListFuture,
3870 ListFuture: std::future::Future<Output = Result<()>>,
3871 Drain: FnMut() -> DrainFuture,
3872 DrainFuture: std::future::Future<Output = bool>,
3873{
3874 let max_attempts = max_attempts.max(1);
3875
3876 for attempt in 0..max_attempts {
3877 if rx.is_cancelled() {
3878 debug!(
3879 event = EVENT_DECOMMISSION_BUCKET,
3880 component = LOG_COMPONENT_ECSTORE,
3881 subsystem = LOG_SUBSYSTEM_POOLS,
3882 pool_index = pool_idx,
3883 set_index = set_idx,
3884 bucket = %bucket,
3885 state = "listing_worker_cancelled",
3886 "Decommission listing worker cancelled"
3887 );
3888 return Ok(());
3889 }
3890
3891 debug!(
3892 event = EVENT_DECOMMISSION_BUCKET,
3893 component = LOG_COMPONENT_ECSTORE,
3894 subsystem = LOG_SUBSYSTEM_POOLS,
3895 pool_index = pool_idx,
3896 set_index = set_idx,
3897 bucket = %bucket,
3898 attempt = attempt + 1,
3899 max_attempts,
3900 state = "listing_started",
3901 "Decommission listing started"
3902 );
3903
3904 let list_result = list(cb.clone()).await;
3905 if drain().await {
3906 return Ok(());
3907 }
3908
3909 match list_result {
3910 Ok(()) => {
3911 debug!(
3912 event = EVENT_DECOMMISSION_BUCKET,
3913 component = LOG_COMPONENT_ECSTORE,
3914 subsystem = LOG_SUBSYSTEM_POOLS,
3915 pool_index = pool_idx,
3916 set_index = set_idx,
3917 bucket = %bucket,
3918 attempt = attempt + 1,
3919 max_attempts,
3920 state = "listing_completed",
3921 "Decommission listing completed"
3922 );
3923 return Ok(());
3924 }
3925 Err(err) if is_err_bucket_not_found(&err) => {
3926 warn!(
3927 event = EVENT_DECOMMISSION_BUCKET,
3928 component = LOG_COMPONENT_ECSTORE,
3929 subsystem = LOG_SUBSYSTEM_POOLS,
3930 pool_index = pool_idx,
3931 set_index = set_idx,
3932 bucket = %bucket,
3933 attempt = attempt + 1,
3934 max_attempts,
3935 state = "listing_bucket_missing",
3936 "Decommission listing bucket missing"
3937 );
3938 return Ok(());
3939 }
3940 Err(err) if should_retry_decommission_listing(&err, attempt, max_attempts) => {
3941 error!(
3942 event = EVENT_DECOMMISSION_BUCKET,
3943 component = LOG_COMPONENT_ECSTORE,
3944 subsystem = LOG_SUBSYSTEM_POOLS,
3945 pool_index = pool_idx,
3946 set_index = set_idx,
3947 bucket = %bucket,
3948 attempt = attempt + 1,
3949 max_attempts,
3950 retry_delay_ms = DECOMMISSION_LISTING_RETRY_DELAY.as_millis(),
3951 state = "listing_failed_retrying",
3952 error = ?err,
3953 "Decommission listing failed; retrying"
3954 );
3955 if wait_decommission_retry_backoff(&rx, DECOMMISSION_LISTING_RETRY_DELAY).await {
3956 debug!(
3957 event = EVENT_DECOMMISSION_BUCKET,
3958 component = LOG_COMPONENT_ECSTORE,
3959 subsystem = LOG_SUBSYSTEM_POOLS,
3960 pool_index = pool_idx,
3961 set_index = set_idx,
3962 bucket = %bucket,
3963 state = "listing_worker_cancelled",
3964 "Decommission listing worker cancelled during retry wait"
3965 );
3966 return Ok(());
3967 }
3968 }
3969 Err(err) => {
3970 error!(
3971 event = EVENT_DECOMMISSION_BUCKET,
3972 component = LOG_COMPONENT_ECSTORE,
3973 subsystem = LOG_SUBSYSTEM_POOLS,
3974 pool_index = pool_idx,
3975 set_index = set_idx,
3976 bucket = %bucket,
3977 attempt = attempt + 1,
3978 max_attempts,
3979 state = "listing_failed",
3980 error = ?err,
3981 "Decommission listing failed"
3982 );
3983 return Err(Error::other(format!(
3984 "decommission listing failed for bucket {bucket} pool {pool_idx} set {set_idx} attempt {}/{}: {err}",
3985 attempt + 1,
3986 max_attempts
3987 )));
3988 }
3989 }
3990 }
3991
3992 Ok(())
3993}
3994
3995fn should_count_decommission_version_complete(ignore: bool, cleanup_ignored: bool, failure: bool) -> bool {
3996 cleanup_ignored || (!ignore && !failure)
3997}
3998
3999fn is_decommission_copy_cleanup_safe_error(err: &Error) -> bool {
4000 if is_err_object_not_found(err) || is_err_version_not_found(err) {
4003 return true;
4004 }
4005
4006 data_movement::data_movement_stage_source(err).is_some_and(is_decommission_copy_cleanup_safe_error)
4009}
4010
4011fn is_decommission_target_capacity_error(err: &Error) -> bool {
4012 if matches!(err, Error::DiskFull | Error::StorageFull) {
4013 return true;
4014 }
4015
4016 if let Some(source) = data_movement::data_movement_stage_source(err) {
4020 return is_decommission_target_capacity_error(source);
4021 }
4022
4023 let message = err.to_string();
4024 let disk_full = Error::DiskFull.to_string();
4025 let storage_full = Error::StorageFull.to_string();
4026 message.contains(&disk_full) || message.contains(&storage_full)
4027}
4028
4029fn should_cleanup_decommission_source_entry(decommissioned: usize, total_versions: usize, expired: usize) -> bool {
4030 decommissioned.saturating_add(expired) == total_versions
4031}
4032
4033fn should_fail_decommission_pool_after_exhausted_source_changed(exhausted_entries: usize) -> bool {
4034 exhausted_entries > DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT
4035}
4036
4037#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4038enum DecommissionEntryAttemptOutcome {
4039 Complete,
4040 SourceChanged,
4041}
4042
4043#[cfg(test)]
4044pub(crate) type DecommissionTestFaultDecision = Arc<dyn Fn(&'static str, &str, &str, usize, bool) -> bool + Send + Sync>;
4045
4046#[cfg(test)]
4047static DECOMMISSION_TEST_FAULT_HOOK: std::sync::OnceLock<std::sync::Mutex<Option<DecommissionTestFaultDecision>>> =
4048 std::sync::OnceLock::new();
4049
4050#[cfg(test)]
4051pub(crate) struct DecommissionTestFaultGuard(DecommissionTestFaultDecision);
4052
4053#[cfg(test)]
4054impl DecommissionTestFaultGuard {
4055 pub(crate) fn install(decision: DecommissionTestFaultDecision) -> Self {
4056 let mut slot = DECOMMISSION_TEST_FAULT_HOOK
4057 .get_or_init(|| std::sync::Mutex::new(None))
4058 .lock()
4059 .expect("decommission test fault hook mutex should not poison");
4060 assert!(slot.is_none(), "decommission test fault hook must be unique");
4061 let stored = Arc::clone(&decision);
4062 *slot = Some(decision);
4063 Self(stored)
4064 }
4065}
4066
4067#[cfg(test)]
4068impl Drop for DecommissionTestFaultGuard {
4069 fn drop(&mut self) {
4070 let mut slot = DECOMMISSION_TEST_FAULT_HOOK
4071 .get_or_init(|| std::sync::Mutex::new(None))
4072 .lock()
4073 .expect("decommission test fault hook mutex should not poison");
4074 if slot.as_ref().is_some_and(|decision| Arc::ptr_eq(decision, &self.0)) {
4075 *slot = None;
4076 }
4077 }
4078}
4079
4080#[cfg(test)]
4081fn decommission_test_wrap_result<T>(
4082 stage: &'static str,
4083 bucket: &str,
4084 object: &str,
4085 attempt: usize,
4086 result: Result<T>,
4087) -> Result<T> {
4088 let decision = DECOMMISSION_TEST_FAULT_HOOK
4089 .get_or_init(|| std::sync::Mutex::new(None))
4090 .lock()
4091 .expect("decommission test fault hook mutex should not poison")
4092 .clone();
4093 let inject = decision.is_some_and(|decision| decision(stage, bucket, object, attempt, result.is_ok()));
4094 if result.is_ok() && inject {
4095 return Err(Error::other(format!(
4096 "injected decommission test fault at {stage} attempt {attempt} for {bucket}/{object}"
4097 )));
4098 }
4099 result
4100}
4101
4102#[cfg(test)]
4103pub(crate) type DecommissionCleanupMutationHook = Arc<dyn Fn(&str, &str, usize) -> BoxFuture<'static, ()> + Send + Sync>;
4104
4105#[cfg(test)]
4106static DECOMMISSION_CLEANUP_MUTATION_HOOK: std::sync::OnceLock<std::sync::Mutex<Option<DecommissionCleanupMutationHook>>> =
4107 std::sync::OnceLock::new();
4108
4109#[cfg(test)]
4110pub(crate) struct DecommissionCleanupMutationGuard(DecommissionCleanupMutationHook);
4111
4112#[cfg(test)]
4113impl DecommissionCleanupMutationGuard {
4114 pub(crate) fn install(hook: DecommissionCleanupMutationHook) -> Self {
4115 let mut slot = DECOMMISSION_CLEANUP_MUTATION_HOOK
4116 .get_or_init(|| std::sync::Mutex::new(None))
4117 .lock()
4118 .expect("decommission cleanup mutation hook mutex should not poison");
4119 assert!(slot.is_none(), "decommission cleanup mutation hook must be unique");
4120 let stored = Arc::clone(&hook);
4121 *slot = Some(hook);
4122 Self(stored)
4123 }
4124}
4125
4126#[cfg(test)]
4127impl Drop for DecommissionCleanupMutationGuard {
4128 fn drop(&mut self) {
4129 let mut slot = DECOMMISSION_CLEANUP_MUTATION_HOOK
4130 .get_or_init(|| std::sync::Mutex::new(None))
4131 .lock()
4132 .expect("decommission cleanup mutation hook mutex should not poison");
4133 if slot.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.0)) {
4134 *slot = None;
4135 }
4136 }
4137}
4138
4139#[cfg(test)]
4140async fn run_decommission_cleanup_mutation_hook(bucket: &str, object: &str, attempt: usize) {
4141 let hook = DECOMMISSION_CLEANUP_MUTATION_HOOK
4142 .get_or_init(|| std::sync::Mutex::new(None))
4143 .lock()
4144 .expect("decommission cleanup mutation hook mutex should not poison")
4145 .clone();
4146 if let Some(hook) = hook {
4147 hook(bucket, object, attempt).await;
4148 }
4149}
4150
4151const DECOMMISSION_FREE_VERSION_MIGRATED_REASON: &str = "tier_free_version_migrated";
4152const DECOMMISSION_FREE_VERSION_CONSUMED_REASON: &str = "tier_free_version_already_consumed";
4153const DECOMMISSION_FREE_VERSION_RETAINED_REASON: &str = "tier_free_version_migration_failed";
4154const DECOMMISSION_FREE_VERSION_SWEEP_REASON: &str = "tier_free_version_unresolved_after_decommission";
4155const DECOMMISSION_FREE_VERSION_DISPOSITION_REASON: &str = "tier_free_version_disposition_recorded";
4156
4157#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
4158struct DecommissionFreeVersionDisposition {
4159 migrated: usize,
4160 consumed: usize,
4161 retained: usize,
4162}
4163
4164impl DecommissionFreeVersionDisposition {
4165 fn record_migrated(&mut self) {
4166 self.migrated += 1;
4167 }
4168
4169 fn record_consumed(&mut self) {
4170 self.consumed += 1;
4171 }
4172
4173 fn record_retained(&mut self) {
4174 self.retained += 1;
4175 }
4176
4177 fn total(self) -> usize {
4178 self.migrated.saturating_add(self.consumed).saturating_add(self.retained)
4179 }
4180}
4181
4182enum DecommissionFreeVersionAttempt {
4183 Migrated,
4184 Consumed,
4185 CapacityFailure(Error),
4186 Retry(Error),
4187}
4188
4189fn classify_decommission_free_version_attempt(result: Result<()>) -> DecommissionFreeVersionAttempt {
4190 match result {
4191 Ok(()) => DecommissionFreeVersionAttempt::Migrated,
4192 Err(err) if is_decommission_copy_cleanup_safe_error(&err) => DecommissionFreeVersionAttempt::Consumed,
4193 Err(err) if is_decommission_target_capacity_error(&err) => DecommissionFreeVersionAttempt::CapacityFailure(err),
4194 Err(err) => DecommissionFreeVersionAttempt::Retry(err),
4195 }
4196}
4197
4198#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4199#[allow(
4200 dead_code,
4201 reason = "terminal-state classification asserted by this file's tests (backlog#1823)"
4202)]
4203enum DecommissionTerminalState {
4204 Completed,
4205 Failed,
4206}
4207
4208#[allow(
4209 dead_code,
4210 reason = "terminal-state classification asserted by this file's tests (backlog#1823)"
4211)]
4212fn classify_decommission_terminal_state(failed_items_present: bool) -> DecommissionTerminalState {
4213 if failed_items_present {
4214 DecommissionTerminalState::Failed
4215 } else {
4216 DecommissionTerminalState::Completed
4217 }
4218}
4219
4220fn should_preserve_decommission_canceled_state(meta_canceled: bool, _cancel_signal: bool) -> bool {
4221 meta_canceled
4222}
4223
4224fn should_continue_decommission_queue(meta: &PoolMeta, idx: usize) -> bool {
4225 meta.pools
4226 .get(idx)
4227 .and_then(|pool| pool.decommission.as_ref())
4228 .is_some_and(|info| info.complete && !info.failed && !info.canceled)
4229}
4230
4231fn decommission_cancel_signal_result(cancel_signal: bool) -> Result<()> {
4232 if cancel_signal {
4233 Err(StorageError::OperationCanceled)
4234 } else {
4235 Ok(())
4236 }
4237}
4238
4239fn is_decommission_cancel_requested(cancel_signal: bool, pool: Option<&PoolStatus>) -> bool {
4240 cancel_signal
4241 || pool
4242 .and_then(|pool| pool.decommission.as_ref())
4243 .is_some_and(|info| info.canceled)
4244}
4245
4246fn should_skip_canceled_decommission_routine(cancel_signal: bool, pool: Option<&PoolStatus>) -> bool {
4247 cancel_signal
4248 && pool
4249 .and_then(|pool| pool.decommission.as_ref())
4250 .is_some_and(|info| info.canceled)
4251}
4252
4253async fn run_decommission_buckets_bounded<F>(
4254 rx: CancellationToken,
4255 buckets: Vec<DecomBucketInfo>,
4256 limit: usize,
4257 mut start_bucket: F,
4258) -> Result<()>
4259where
4260 F: FnMut(DecomBucketInfo, CancellationToken) -> BoxFuture<'static, Result<()>>,
4261{
4262 let mut pending = buckets.into_iter();
4263 let mut active: FuturesUnordered<BoxFuture<'static, Result<()>>> = FuturesUnordered::new();
4264 let mut first_err = None;
4265 let limit = limit.max(1);
4266
4267 for _ in 0..limit {
4268 let Some(bucket) = pending.next() else {
4269 break;
4270 };
4271
4272 active.push(start_bucket(bucket, rx.clone()));
4273 }
4274
4275 while let Some(result) = active.next().await {
4276 if let Err(err) = result {
4277 rx.cancel();
4278 if first_err.is_none() {
4279 first_err = Some(err);
4280 }
4281 continue;
4282 }
4283
4284 if first_err.is_some() || rx.is_cancelled() {
4285 continue;
4286 }
4287
4288 let Some(bucket) = pending.next() else {
4289 continue;
4290 };
4291
4292 active.push(start_bucket(bucket, rx.clone()));
4293 }
4294
4295 if first_err.is_none() && rx.is_cancelled() && pending.len() > 0 {
4296 return decommission_cancel_signal_result(true);
4297 }
4298
4299 if let Some(err) = first_err {
4300 return Err(err);
4301 }
4302
4303 Ok(())
4304}
4305
4306async fn run_decommission_phases<F>(
4307 rx: CancellationToken,
4308 regular_buckets: Vec<DecomBucketInfo>,
4309 meta_buckets: Vec<DecomBucketInfo>,
4310 bucket_concurrency: usize,
4311 mut start_bucket: F,
4312) -> Result<()>
4313where
4314 F: FnMut(DecomBucketInfo, CancellationToken) -> BoxFuture<'static, Result<()>>,
4315{
4316 decommission_cancel_signal_result(rx.is_cancelled())?;
4317
4318 for bucket in meta_buckets {
4319 decommission_cancel_signal_result(rx.is_cancelled())?;
4320 start_bucket(bucket, rx.clone()).await?;
4321 }
4322
4323 decommission_cancel_signal_result(rx.is_cancelled())?;
4324
4325 if bucket_concurrency <= 1 {
4326 for bucket in regular_buckets {
4327 decommission_cancel_signal_result(rx.is_cancelled())?;
4328 start_bucket(bucket, rx.clone()).await?;
4329 }
4330 return Ok(());
4331 }
4332
4333 run_decommission_buckets_bounded(rx, regular_buckets, bucket_concurrency, start_bucket).await
4334}
4335
4336#[cfg(test)]
4337async fn wait_decommission_worker_drain(workers: &Semaphore, limit: usize) -> Result<()> {
4338 let permits = u32::try_from(limit)
4339 .map_err(|_| Error::other(format!("decommission worker limit {limit} exceeds semaphore drain capacity")))?;
4340 let _drain = workers
4341 .acquire_many(permits)
4342 .await
4343 .map_err(|err| Error::other(format!("decommission worker drain failed: {err}")))?;
4344 Ok(())
4345}
4346
4347fn should_reject_decommission_cancel_as_terminal(complete: bool, failed: bool) -> bool {
4348 complete || failed
4349}
4350
4351fn should_retry_decommission_cancel_reload(changed: bool, already_canceled: bool) -> bool {
4352 changed || already_canceled
4353}
4354
4355fn ensure_decommission_cancel_allowed(pool_present: bool, decommission_present: bool, terminal: bool) -> Result<()> {
4356 if !pool_present {
4357 return Err(invalid_decommission_request("failed to cancel decommission: target pool was not found"));
4358 }
4359
4360 if !decommission_present || terminal {
4361 return Err(StorageError::DecommissionNotStarted);
4362 }
4363
4364 Ok(())
4365}
4366
4367fn ensure_decommission_clear_allowed(
4368 pool_present: bool,
4369 decommission_present: bool,
4370 complete: bool,
4371 failed: bool,
4372 canceled: bool,
4373 unresolved_entries: usize,
4374) -> Result<()> {
4375 if !pool_present {
4376 return Err(invalid_decommission_request("failed to clear decommission: target pool was not found"));
4377 }
4378
4379 if !decommission_present {
4380 return Err(StorageError::DecommissionNotStarted);
4381 }
4382
4383 if complete {
4384 return Err(StorageError::DecommissionNotStarted);
4385 }
4386
4387 if !failed && !canceled {
4388 return Err(StorageError::DecommissionAlreadyRunning);
4389 }
4390
4391 if unresolved_entries > 0 {
4392 return Err(invalid_decommission_request(format!(
4393 "failed to clear decommission: {unresolved_entries} unresolved listing entries must be reconciled by retrying decommission"
4394 )));
4395 }
4396
4397 Ok(())
4398}
4399
4400fn ensure_decommission_terminal_operation_supported(single_pool: bool, operation: &str) -> Result<()> {
4401 if single_pool {
4402 return Err(invalid_decommission_request(format!(
4403 "failed to {operation}: single pool deployments do not support decommission"
4404 )));
4405 }
4406
4407 Ok(())
4408}
4409
4410fn validate_start_decommission_request(indices: &[usize], single_pool: bool) -> Result<()> {
4411 if indices.is_empty() {
4412 return Err(invalid_decommission_request(
4413 "failed to start decommission: no target pools were provided",
4414 ));
4415 }
4416
4417 ensure_decommission_terminal_operation_supported(single_pool, "start decommission")
4418}
4419
4420fn require_decommission_store<T>(store: Option<T>, operation: &str) -> Result<T> {
4421 store.ok_or_else(|| Error::other(format!("failed to {operation}: store not initialized")))
4422}
4423
4424fn ensure_decommission_listing_disks_available(has_disks: bool, bucket: &str) -> Result<()> {
4425 if !has_disks {
4426 return Err(Error::other(format!(
4427 "failed to list objects to decommission for bucket {bucket}: no disks available"
4428 )));
4429 }
4430
4431 Ok(())
4432}
4433
4434#[derive(Debug, Clone, Serialize, Deserialize)]
4435pub struct PoolStatus {
4436 #[serde(rename = "id")]
4437 pub id: usize,
4438 #[serde(rename = "cmdline")]
4439 pub cmd_line: String,
4440 #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
4441 pub last_update: OffsetDateTime,
4442 #[serde(rename = "decommissionInfo")]
4443 pub decommission: Option<PoolDecommissionInfo>,
4444}
4445
4446#[derive(Debug, Clone, Default, Serialize, Deserialize)]
4447pub struct PoolMeta {
4448 pub version: u16,
4449 pub pools: Vec<PoolStatus>,
4450 pub dont_save: bool,
4451}
4452
4453#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4454struct PoolMetaRevision {
4455 version: u16,
4456 cluster_id: Option<uuid::Uuid>,
4457 epoch: u64,
4458 generation: u64,
4459 transaction_id: Option<uuid::Uuid>,
4460}
4461
4462impl PoolMetaRevision {
4463 fn legacy(version: u16) -> Self {
4464 Self {
4465 version,
4466 cluster_id: None,
4467 epoch: 0,
4468 generation: 0,
4469 transaction_id: None,
4470 }
4471 }
4472
4473 fn is_generation_protocol(self) -> bool {
4474 self.version == POOL_META_GENERATION_VERSION
4475 }
4476}
4477
4478#[derive(Debug, Clone)]
4479struct PoolMetaCommittedCandidate {
4480 canonical: Vec<u8>,
4481 meta: PoolMeta,
4482 revision: PoolMetaRevision,
4483}
4484
4485#[derive(Debug)]
4486enum PoolMetaReplica {
4487 Missing,
4488 Valid {
4489 raw: Vec<u8>,
4490 canonical: Vec<u8>,
4491 meta: PoolMeta,
4492 revision: PoolMetaRevision,
4493 committed: bool,
4494 previous: Option<Box<PoolMetaCommittedCandidate>>,
4495 },
4496 Corrupt(String),
4497 Incompatible(String),
4498 Unreadable(Arc<Error>),
4499}
4500
4501#[derive(Debug, Clone)]
4502enum PoolMetaCasToken {
4503 Missing,
4504 Existing(String),
4505 Unsafe,
4506}
4507
4508#[derive(Debug)]
4509struct PoolMetaReplicaRead {
4510 replica: PoolMetaReplica,
4511 cas: PoolMetaCasToken,
4512}
4513
4514impl From<PoolMetaReplica> for PoolMetaReplicaRead {
4515 fn from(replica: PoolMetaReplica) -> Self {
4516 Self {
4517 replica,
4518 cas: PoolMetaCasToken::Unsafe,
4519 }
4520 }
4521}
4522
4523#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4524pub(crate) struct PoolMetaReplicaState {
4525 pub(crate) needs_repair: bool,
4526 pub(crate) repair_write_safe: bool,
4527}
4528
4529impl PoolMetaReplicaState {
4530 pub(crate) fn ensure_write_safe(self, operation: &str) -> Result<()> {
4531 if self.repair_write_safe {
4532 return Ok(());
4533 }
4534 Err(Error::other(format!(
4535 "{operation}: pool metadata update cannot overwrite an unreadable replica"
4536 )))
4537 }
4538}
4539
4540#[derive(Debug, Clone, Default)]
4541pub(crate) struct PoolMetaWriteState {
4542 write_blocked: bool,
4543 aborted_transaction: Arc<AtomicBool>,
4544 block_context: Option<crate::error::PoolMetadataError>,
4545 transaction_failure: Arc<std::sync::Mutex<Option<crate::error::PoolMetadataError>>>,
4546 active_transactions: Arc<AtomicUsize>,
4547 recovery_failure: Option<Arc<Error>>,
4548 block_started_at: Arc<std::sync::Mutex<Option<OffsetDateTime>>>,
4549 expected_cluster_id: Option<uuid::Uuid>,
4550 cluster_epoch: Option<u64>,
4551 pool_meta_absent: bool,
4552 bootstrap_authority: PoolMetaBootstrapAuthority,
4553 pool_bootstrap_authorities: Vec<PoolMetaBootstrapAuthority>,
4558 elected_bootstrap_writer: Option<bool>,
4563 identity_initialized: Option<bool>,
4564 identity_fresh_bootstrap_nonce: Option<uuid::Uuid>,
4565 identity_needs_repair: bool,
4566 identity_replicas_missing: bool,
4568 identity_replicas_invalid: bool,
4570}
4571
4572enum MissingMetadataRejection {
4574 BootstrapPending(Error),
4577 RecoveryRequired(Error),
4580}
4581
4582impl MissingMetadataRejection {
4583 fn into_error(self) -> Error {
4584 match self {
4585 Self::BootstrapPending(err) | Self::RecoveryRequired(err) => err,
4586 }
4587 }
4588}
4589
4590#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
4591pub(crate) enum PoolMetaBootstrapAuthority {
4592 #[default]
4593 None,
4594 Fresh,
4595 LegacyAdoption,
4596}
4597
4598impl PoolMetaBootstrapAuthority {
4599 pub(crate) fn combine_across_pools(self, other: Self) -> Self {
4600 if self == other { self } else { Self::None }
4601 }
4602
4603 fn is_proven(self) -> bool {
4604 !matches!(self, Self::None)
4605 }
4606}
4607
4608impl PoolMetaWriteState {
4609 #[cfg(test)]
4610 pub(crate) fn for_startup(cluster_id: uuid::Uuid, fresh_bootstrap_proven: bool) -> Self {
4611 let bootstrap_authority = if fresh_bootstrap_proven {
4612 PoolMetaBootstrapAuthority::Fresh
4613 } else {
4614 PoolMetaBootstrapAuthority::None
4615 };
4616 Self::for_startup_with_bootstrap_authority(cluster_id, bootstrap_authority)
4617 }
4618
4619 #[cfg(test)]
4620 pub(crate) fn for_startup_with_bootstrap_authority(
4621 cluster_id: uuid::Uuid,
4622 bootstrap_authority: PoolMetaBootstrapAuthority,
4623 ) -> Self {
4624 Self {
4625 expected_cluster_id: Some(cluster_id),
4626 bootstrap_authority,
4627 ..Default::default()
4628 }
4629 }
4630
4631 pub(crate) fn for_startup_with_pool_bootstrap_authorities(
4636 cluster_id: uuid::Uuid,
4637 pool_bootstrap_authorities: Vec<PoolMetaBootstrapAuthority>,
4638 elected_bootstrap_writer: bool,
4639 ) -> Self {
4640 let bootstrap_authority = pool_bootstrap_authorities
4641 .iter()
4642 .copied()
4643 .reduce(PoolMetaBootstrapAuthority::combine_across_pools)
4644 .unwrap_or_default();
4645 Self {
4646 expected_cluster_id: Some(cluster_id),
4647 bootstrap_authority,
4648 pool_bootstrap_authorities,
4649 elected_bootstrap_writer: Some(elected_bootstrap_writer),
4650 ..Default::default()
4651 }
4652 }
4653
4654 pub(crate) fn bootstrap_identity_proven(&self) -> bool {
4655 self.bootstrap_authority.is_proven()
4656 }
4657
4658 fn pool_bootstrap_authority_proven(&self, pool_idx: usize) -> bool {
4659 self.pool_bootstrap_authorities
4660 .get(pool_idx)
4661 .is_some_and(|authority| authority.is_proven())
4662 }
4663
4664 pub(crate) fn attested_pool_indices(&self) -> Vec<usize> {
4666 self.pool_bootstrap_authorities
4667 .iter()
4668 .enumerate()
4669 .filter(|(_, authority)| authority.is_proven())
4670 .map(|(pool_idx, _)| pool_idx)
4671 .collect()
4672 }
4673
4674 fn pending_identity_attested_by_every_pool(&self) -> bool {
4683 self.elected_bootstrap_writer == Some(true)
4684 && self.pool_bootstrap_authority_proven(0)
4685 && self.identity_initialized == Some(false)
4686 && !self.identity_needs_repair
4687 && self.identity_fresh_bootstrap_nonce.is_some()
4688 }
4689
4690 fn awaiting_creator_attestation(&self) -> bool {
4694 self.elected_bootstrap_writer == Some(true)
4695 && self.pool_bootstrap_authority_proven(0)
4696 && self.identity_initialized == Some(false)
4697 && self.identity_fresh_bootstrap_nonce.is_some()
4698 && self.identity_needs_repair
4699 && self.identity_replicas_missing
4700 && !self.identity_replicas_invalid
4701 }
4702
4703 fn is_non_elected_bootstrap_observer(&self) -> bool {
4704 self.elected_bootstrap_writer == Some(false)
4705 }
4706
4707 pub(crate) fn identity_is_pending(&self) -> bool {
4708 self.identity_initialized == Some(false)
4709 }
4710
4711 #[cfg(test)]
4712 pub(crate) fn aborted_transaction_latch_for_test(&self) -> Arc<AtomicBool> {
4713 Arc::clone(&self.aborted_transaction)
4714 }
4715
4716 #[cfg(test)]
4717 pub(crate) fn independent_clone_for_test(&self) -> Self {
4718 let mut cloned = self.clone();
4719 cloned.aborted_transaction = Arc::new(AtomicBool::new(self.aborted_transaction.load(Ordering::Acquire)));
4720 cloned.transaction_failure = Arc::new(std::sync::Mutex::new(
4721 self.transaction_failure
4722 .lock()
4723 .unwrap_or_else(std::sync::PoisonError::into_inner)
4724 .clone(),
4725 ));
4726 cloned.active_transactions = Arc::new(AtomicUsize::new(0));
4727 cloned.block_started_at = Arc::new(std::sync::Mutex::new(
4728 *self
4729 .block_started_at
4730 .lock()
4731 .unwrap_or_else(std::sync::PoisonError::into_inner),
4732 ));
4733 cloned
4734 }
4735
4736 #[cfg(any(test, feature = "test-util"))]
4737 fn for_test_bootstrap() -> Self {
4738 Self {
4739 bootstrap_authority: PoolMetaBootstrapAuthority::Fresh,
4740 identity_initialized: Some(false),
4741 identity_fresh_bootstrap_nonce: Some(uuid::Uuid::new_v4()),
4742 ..Default::default()
4743 }
4744 }
4745
4746 pub(crate) fn observe_replicas(&mut self, replica_state: PoolMetaReplicaState) {
4747 if !replica_state.repair_write_safe {
4748 self.block_writes();
4749 }
4750 }
4751
4752 fn block_writes(&mut self) {
4753 self.block_with_context(pool_metadata_error(
4754 crate::error::PoolMetadataFailure::RecoveryRequired,
4755 "validation",
4756 None,
4757 ));
4758 }
4759
4760 fn block_with_context(&mut self, mut context: crate::error::PoolMetadataError) {
4761 if !self.write_blocked {
4762 record_pool_meta_block_once(&self.block_started_at, &mut context);
4763 self.block_context = Some(context);
4764 } else if let Some(previous) = &self.block_context
4765 && previous.source.is_none()
4766 && context.source.is_some()
4767 {
4768 context.since = previous.since;
4769 self.block_context = Some(context);
4770 }
4771 self.write_blocked = true;
4772 }
4773
4774 pub(crate) fn block_writes_after_fence_loss(&mut self) {
4775 self.block_with_context(pool_metadata_error(crate::error::PoolMetadataFailure::FenceLost, "format_heal", None));
4776 }
4777
4778 fn arm_transaction(&self) -> PoolMetaTransactionArm {
4779 self.active_transactions.fetch_add(1, Ordering::SeqCst);
4780 PoolMetaTransactionArm {
4781 aborted_transaction: Arc::clone(&self.aborted_transaction),
4782 failure: Arc::clone(&self.transaction_failure),
4783 active_transactions: Arc::clone(&self.active_transactions),
4784 block_started_at: Arc::clone(&self.block_started_at),
4785 phase: None,
4786 source: None,
4787 }
4788 }
4789
4790 fn observe_selection(&mut self, selection: &PoolMetaSelection) -> Result<()> {
4791 self.pool_meta_absent = selection.absent;
4792 self.validate_selection(selection)?;
4793 if self.cluster_epoch.is_none()
4794 && let Some((_, metadata_epoch)) = selection.generation_identity
4795 {
4796 self.cluster_epoch = Some(metadata_epoch);
4797 }
4798 Ok(())
4799 }
4800
4801 fn validate_selection(&self, selection: &PoolMetaSelection) -> Result<()> {
4802 if let Some(expected_cluster_id) = self.expected_cluster_id
4803 && let Some((cluster_id, _)) = selection.generation_identity
4804 && cluster_id != expected_cluster_id
4805 {
4806 return Err(Error::other(format!(
4807 "pool metadata incompatible: cluster identity {cluster_id} does not match deployment {expected_cluster_id}"
4808 )));
4809 }
4810 if let Some(identity_epoch) = self.cluster_epoch
4811 && let Some((_, metadata_epoch)) = selection.generation_identity
4812 && metadata_epoch != identity_epoch
4813 {
4814 return Err(Error::other(format!(
4815 "pool metadata recovery required: committed epoch {} does not match cluster identity epoch {identity_epoch}",
4816 metadata_epoch
4817 )));
4818 }
4819 Ok(())
4820 }
4821
4822 fn observe_identity(&mut self, selection: &PoolMetaIdentitySelection) -> Result<()> {
4823 self.identity_needs_repair = selection.needs_repair;
4824 self.identity_initialized = selection.identity.map(|identity| identity.initialized);
4825 self.identity_fresh_bootstrap_nonce = selection.identity.and_then(|identity| identity.fresh_bootstrap_nonce);
4826 self.identity_replicas_missing = selection
4827 .cas_tokens
4828 .iter()
4829 .any(|token| matches!(token, PoolMetaCasToken::Missing));
4830 self.identity_replicas_invalid = selection
4831 .valid_replicas
4832 .iter()
4833 .zip(&selection.cas_tokens)
4834 .any(|(valid, token)| !valid && !matches!(token, PoolMetaCasToken::Missing));
4835 if let Some(identity) = selection.identity {
4836 if identity.initialized {
4837 self.bootstrap_authority = PoolMetaBootstrapAuthority::None;
4838 self.pool_bootstrap_authorities.clear();
4839 }
4840 if let Some(metadata_epoch) = self.cluster_epoch
4841 && metadata_epoch != identity.epoch
4842 {
4843 self.block_writes();
4844 return Err(Error::other(format!(
4845 "pool metadata recovery required: metadata epoch {metadata_epoch} does not match cluster identity epoch {}",
4846 identity.epoch
4847 )));
4848 }
4849 self.cluster_epoch = Some(identity.epoch);
4850 }
4851 if !selection.repair_write_safe {
4852 self.block_writes();
4853 }
4854 Ok(())
4855 }
4856
4857 pub(crate) fn ensure_missing_metadata_can_initialize(&mut self) -> Result<()> {
4858 if !self.pool_meta_absent {
4859 return Ok(());
4860 }
4861 match self.validate_missing_metadata_can_initialize() {
4862 Ok(()) => Ok(()),
4863 Err(MissingMetadataRejection::BootstrapPending(err)) => Err(err),
4864 Err(MissingMetadataRejection::RecoveryRequired(err)) => {
4865 Err(block_pool_meta_validation(self, err, "metadata_absence"))
4866 }
4867 }
4868 }
4869
4870 fn validate_missing_metadata_can_initialize(&self) -> std::result::Result<(), MissingMetadataRejection> {
4871 use MissingMetadataRejection::{BootstrapPending, RecoveryRequired};
4872 match self.identity_initialized {
4873 Some(false)
4874 if self.identity_fresh_bootstrap_nonce.is_some()
4875 && (self.bootstrap_identity_proven() || self.pending_identity_attested_by_every_pool()) =>
4876 {
4877 Ok(())
4878 }
4879 Some(false) if self.awaiting_creator_attestation() => Err(BootstrapPending(Error::other(
4880 "pool metadata bootstrap pending: waiting for every pool creator to attest the pending cluster identity",
4881 ))),
4882 Some(false) if self.is_non_elected_bootstrap_observer() && self.identity_fresh_bootstrap_nonce.is_some() => {
4883 Err(BootstrapPending(Error::other(
4884 "pool metadata bootstrap pending: waiting for the elected writer to publish the initial pool.bin",
4885 )))
4886 }
4887 Some(false) => Err(RecoveryRequired(Error::other(
4888 "pool metadata recovery required: pending cluster identity exists but this startup has no verified fresh-bootstrap proof, legacy-adoption proof, or complete per-pool creator attestation",
4889 ))),
4890 Some(true) => Err(RecoveryRequired(Error::other(
4891 "pool metadata recovery required: initialized cluster identity exists but every pool.bin replica is missing",
4892 ))),
4893 None if self.is_non_elected_bootstrap_observer() => Err(BootstrapPending(Error::other(
4894 "pool metadata bootstrap pending: waiting for the elected writer to establish the cluster identity",
4895 ))),
4896 None => Err(RecoveryRequired(Error::other(
4897 "pool metadata recovery required: no durable bootstrap identity or pool.bin replica is available",
4898 ))),
4899 }
4900 }
4901
4902 pub(crate) fn identity_requires_repair(&self) -> bool {
4903 self.expected_cluster_id.is_some() && (self.identity_needs_repair || self.identity_initialized != Some(true))
4904 }
4905
4906 pub(crate) fn ensure_write_safe(&self, operation: &str) -> Result<()> {
4907 if !self.write_blocked && !self.aborted_transaction.load(Ordering::SeqCst) {
4908 return Ok(());
4909 }
4910 let mut context = self
4911 .block_context
4912 .clone()
4913 .or_else(|| {
4914 self.transaction_failure
4915 .lock()
4916 .unwrap_or_else(std::sync::PoisonError::into_inner)
4917 .clone()
4918 })
4919 .unwrap_or_else(|| pool_metadata_error(crate::error::PoolMetadataFailure::FenceLost, "external_fence", None));
4920 context.operation = operation.to_owned();
4921 Err(Error::other(context))
4922 }
4923}
4924
4925fn pool_metadata_error(
4926 kind: crate::error::PoolMetadataFailure,
4927 phase: &'static str,
4928 source: Option<Arc<Error>>,
4929) -> crate::error::PoolMetadataError {
4930 crate::error::PoolMetadataError {
4931 kind,
4932 operation: phase.to_owned(),
4933 phase,
4934 since: OffsetDateTime::now_utc(),
4935 source,
4936 }
4937}
4938
4939fn record_pool_meta_block_once(
4940 started_at: &std::sync::Mutex<Option<OffsetDateTime>>,
4941 context: &mut crate::error::PoolMetadataError,
4942) {
4943 let mut started_at = started_at.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
4944 if let Some(since) = *started_at {
4945 context.since = since;
4946 return;
4947 }
4948 *started_at = Some(context.since);
4949 error!(
4950 event = EVENT_DECOMMISSION_STATE,
4951 component = LOG_COMPONENT_ECSTORE,
4952 subsystem = LOG_SUBSYSTEM_POOLS,
4953 state = "pool_metadata_blocked",
4954 reason = context.kind.as_str(),
4955 phase = context.phase,
4956 blocked_since = %context.since,
4957 "Pool metadata writes blocked"
4958 );
4959 metrics::counter!("rustfs_pool_metadata_blocks_total", "reason" => context.kind.as_str()).increment(1);
4960}
4961
4962#[derive(Debug)]
4963struct PoolMetaTransactionArm {
4964 aborted_transaction: Arc<AtomicBool>,
4965 failure: Arc<std::sync::Mutex<Option<crate::error::PoolMetadataError>>>,
4966 active_transactions: Arc<AtomicUsize>,
4967 block_started_at: Arc<std::sync::Mutex<Option<OffsetDateTime>>>,
4968 phase: Option<&'static str>,
4969 source: Option<Arc<Error>>,
4970}
4971
4972impl PoolMetaTransactionArm {
4973 fn failure(&mut self, err: Error) -> Error {
4974 let Some(phase) = self.phase else {
4975 return err;
4976 };
4977 let source = Arc::new(err);
4978 self.source = Some(Arc::clone(&source));
4979 Error::other(pool_metadata_error(
4980 crate::error::PoolMetadataFailure::TransactionUnknown,
4981 phase,
4982 Some(source),
4983 ))
4984 }
4985
4986 fn disarm(&mut self) {
4987 self.phase = None;
4988 }
4989}
4990
4991impl Drop for PoolMetaTransactionArm {
4992 fn drop(&mut self) {
4993 if let Some(phase) = self.phase {
4994 let mut context =
4995 pool_metadata_error(crate::error::PoolMetadataFailure::TransactionUnknown, phase, self.source.take());
4996 let mut failure = self.failure.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
4997 if !self.aborted_transaction.swap(true, Ordering::SeqCst) {
4998 record_pool_meta_block_once(&self.block_started_at, &mut context);
4999 *failure = Some(context);
5000 }
5001 }
5002 self.active_transactions.fetch_sub(1, Ordering::SeqCst);
5003 }
5004}
5005
5006#[derive(Debug)]
5007struct PoolMetaSelection {
5008 meta: PoolMeta,
5009 revision: PoolMetaRevision,
5010 canonical: Option<Vec<u8>>,
5011 replica_state: PoolMetaReplicaState,
5012 cas_tokens: Vec<PoolMetaCasToken>,
5013 absent: bool,
5014 generation_protocol_observed: bool,
5015 generation_identity: Option<(uuid::Uuid, u64)>,
5016}
5017
5018fn classify_pool_meta_tuple_decode_error(kind: &str, err: rmp_serde::decode::Error) -> PoolMetaReplica {
5019 let truncated = matches!(
5020 &err,
5021 rmp_serde::decode::Error::InvalidMarkerRead(source)
5022 | rmp_serde::decode::Error::InvalidDataRead(source)
5023 if source.kind() == std::io::ErrorKind::UnexpectedEof
5024 );
5025 if truncated {
5026 PoolMetaReplica::Corrupt(format!("{kind} tuple payload is truncated: {err}"))
5027 } else {
5028 PoolMetaReplica::Incompatible(format!("{kind} tuple payload is not decodable: {err}"))
5029 }
5030}
5031
5032fn parse_pool_meta_uuid(value: &str, field: &str) -> Result<uuid::Uuid> {
5033 let parsed = uuid::Uuid::parse_str(value)
5034 .map_err(|err| Error::other(format!("pool metadata corrupt: invalid {field} `{value}`: {err}")))?;
5035 if parsed.is_nil() || parsed == uuid::Uuid::max() {
5036 return Err(Error::other(format!("pool metadata corrupt: {field} must be a non-reserved UUID")));
5037 }
5038 Ok(parsed)
5039}
5040
5041fn pool_meta_generation_revision(
5042 cluster_id: &str,
5043 epoch: u64,
5044 generation: u64,
5045 transaction_id: &str,
5046) -> Result<PoolMetaRevision> {
5047 if epoch == 0 || generation == 0 {
5048 return Err(Error::other(
5049 "pool metadata corrupt: version 3 epoch and generation must both be non-zero",
5050 ));
5051 }
5052 Ok(PoolMetaRevision {
5053 version: POOL_META_GENERATION_VERSION,
5054 cluster_id: Some(parse_pool_meta_uuid(cluster_id, "cluster identity")?),
5055 epoch,
5056 generation,
5057 transaction_id: Some(parse_pool_meta_uuid(transaction_id, "transaction id")?),
5058 })
5059}
5060
5061fn pool_meta_from_v3_statuses(version: u16, pools: Vec<PersistedPoolStatus>) -> Result<PoolMeta> {
5062 if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) {
5063 return Err(Error::other(format!(
5064 "pool metadata corrupt: version 3 previous snapshot has unsupported source version {version}"
5065 )));
5066 }
5067 let meta = PoolMeta {
5068 version,
5069 pools: pools.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?,
5070 dont_save: false,
5071 };
5072 validate_decommission_capacity_model_cohort(&meta)?;
5073 Ok(meta)
5074}
5075
5076fn pool_meta_previous_candidate(value: PersistedPoolMetaV3Previous) -> Result<PoolMetaCommittedCandidate> {
5077 let revision = match value.version {
5078 POOL_META_V1_VERSION | POOL_META_VERSION => {
5079 if value.cluster_id.is_some() || value.epoch != 0 || value.generation != 0 || value.transaction_id.is_some() {
5080 return Err(Error::other(
5081 "pool metadata corrupt: legacy previous snapshot carries version 3 revision fields",
5082 ));
5083 }
5084 PoolMetaRevision::legacy(value.version)
5085 }
5086 POOL_META_GENERATION_VERSION => pool_meta_generation_revision(
5087 value
5088 .cluster_id
5089 .as_deref()
5090 .ok_or_else(|| Error::other("pool metadata corrupt: version 3 previous snapshot has no cluster identity"))?,
5091 value.epoch,
5092 value.generation,
5093 value
5094 .transaction_id
5095 .as_deref()
5096 .ok_or_else(|| Error::other("pool metadata corrupt: version 3 previous snapshot has no transaction id"))?,
5097 )?,
5098 version => {
5099 return Err(Error::other(format!(
5100 "pool metadata corrupt: previous snapshot has unsupported version {version}"
5101 )));
5102 }
5103 };
5104 let meta = pool_meta_from_v3_statuses(value.version, value.pools)?;
5105 let canonical = if revision.is_generation_protocol() {
5106 encode_pool_meta_v3_envelope(&meta, revision, true, None)?
5107 } else {
5108 meta.encode_config_data_for_v2_gate(true)?
5109 };
5110 Ok(PoolMetaCommittedCandidate {
5111 canonical,
5112 meta,
5113 revision,
5114 })
5115}
5116
5117fn decode_pool_meta_v3(data: Vec<u8>) -> PoolMetaReplica {
5118 let persisted = match rmp_serde::from_slice::<PersistedPoolMetaV3>(&data[4..]) {
5119 Ok(persisted) => persisted,
5120 Err(err) => return classify_pool_meta_tuple_decode_error("v3", err),
5121 };
5122 if persisted.version != POOL_META_GENERATION_VERSION {
5123 return PoolMetaReplica::Corrupt(format!(
5124 "v3 payload has version {}, expected {POOL_META_GENERATION_VERSION}",
5125 persisted.version
5126 ));
5127 }
5128 let revision = match pool_meta_generation_revision(
5129 &persisted.cluster_id,
5130 persisted.epoch,
5131 persisted.generation,
5132 &persisted.transaction_id,
5133 ) {
5134 Ok(revision) => revision,
5135 Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5136 };
5137 let meta = match pool_meta_from_v3_statuses(POOL_META_GENERATION_VERSION, persisted.pools) {
5138 Ok(meta) => meta,
5139 Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5140 };
5141 let previous = match persisted.previous.map(pool_meta_previous_candidate).transpose() {
5142 Ok(previous) => previous,
5143 Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5144 };
5145 if persisted.committed && previous.is_some() {
5146 return PoolMetaReplica::Corrupt("committed v3 payload unexpectedly retains a previous snapshot".to_string());
5147 }
5148 if !persisted.committed {
5149 match previous.as_ref() {
5150 Some(previous) if previous.revision.is_generation_protocol() => {
5151 if previous.revision.cluster_id != revision.cluster_id
5152 || previous.revision.epoch != revision.epoch
5153 || previous.revision.generation.checked_add(1) != Some(revision.generation)
5154 {
5155 return PoolMetaReplica::Corrupt(
5156 "pending v3 payload does not advance exactly one generation from its previous snapshot".to_string(),
5157 );
5158 }
5159 }
5160 Some(_) if revision.generation != 1 => {
5161 return PoolMetaReplica::Corrupt(
5162 "first v3 generation must be generation 1 when migrating a legacy snapshot".to_string(),
5163 );
5164 }
5165 None if revision.generation != 1 => {
5166 return PoolMetaReplica::Corrupt(
5167 "pending v3 payload without a previous snapshot must be the initial generation".to_string(),
5168 );
5169 }
5170 _ => {}
5171 }
5172 }
5173 let canonical = match encode_pool_meta_v3_envelope(&meta, revision, true, None) {
5174 Ok(canonical) => canonical,
5175 Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5176 };
5177 PoolMetaReplica::Valid {
5178 raw: data,
5179 canonical,
5180 meta,
5181 revision,
5182 committed: persisted.committed,
5183 previous: previous.map(Box::new),
5184 }
5185}
5186
5187fn decode_pool_meta_replica(data: Vec<u8>) -> PoolMetaReplica {
5188 if data.len() <= 4 {
5189 return PoolMetaReplica::Corrupt("metadata payload is empty or truncated".to_string());
5190 }
5191
5192 let format = LittleEndian::read_u16(&data[0..2]);
5193 if format != POOL_META_FORMAT {
5194 return PoolMetaReplica::Incompatible(format!("unsupported format {format}"));
5195 }
5196 let version = LittleEndian::read_u16(&data[2..4]);
5197 if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) {
5198 return PoolMetaReplica::Incompatible(format!("unsupported version {version}"));
5199 }
5200 if version == POOL_META_GENERATION_VERSION {
5201 return decode_pool_meta_v3(data);
5202 }
5203
5204 let payload = &data[4..];
5205 let meta = match (version, rmp::decode::read_array_len(&mut &payload[..])) {
5206 (POOL_META_VERSION, Ok(2)) => match rmp_serde::from_slice::<PersistedPoolMeta>(payload) {
5207 Ok(meta) => match PoolMeta::try_from(meta) {
5208 Ok(meta) => meta,
5209 Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5210 },
5211 Err(err) => return classify_pool_meta_tuple_decode_error("current", err),
5212 },
5213 (POOL_META_V1_VERSION, Ok(2)) => match rmp_serde::from_slice::<PersistedPoolMetaV1>(payload) {
5214 Ok(meta) => match PoolMeta::try_from(meta) {
5215 Ok(meta) => meta,
5216 Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5217 },
5218 Err(err) => return classify_pool_meta_tuple_decode_error("v1", err),
5219 },
5220 (POOL_META_V1_VERSION, Ok(3)) => match rmp_serde::from_slice::<LegacyPoolMeta>(payload) {
5222 Ok(meta) => match PoolMeta::try_from(meta) {
5223 Ok(meta) => meta,
5224 Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5225 },
5226 Err(err) => return classify_pool_meta_tuple_decode_error("legacy v1", err),
5227 },
5228 (_, Ok(field_count)) if field_count < 2 => {
5229 return PoolMetaReplica::Corrupt(format!("pool metadata tuple has only {field_count} fields"));
5230 }
5231 (_, Ok(field_count)) => {
5232 return PoolMetaReplica::Incompatible(format!(
5233 "pool metadata version {version} tuple has unsupported field count {field_count}"
5234 ));
5235 }
5236 (_, Err(_)) => {
5237 let mut meta = PoolMeta::default();
5238 if let Err(err) = meta.load_from_config_data(data.clone()) {
5239 let reason = err.to_string();
5240 if reason.contains("unknown field") {
5241 return PoolMetaReplica::Incompatible(format!("current-version payload uses unsupported fields: {reason}"));
5242 }
5243 return PoolMetaReplica::Corrupt(reason);
5244 }
5245 meta
5246 }
5247 };
5248
5249 match meta.encode_config_data_for_v2_gate(true) {
5250 Ok(canonical) => PoolMetaReplica::Valid {
5251 raw: data,
5252 canonical,
5253 meta,
5254 revision: PoolMetaRevision::legacy(version),
5255 committed: true,
5256 previous: None,
5257 },
5258 Err(err) => PoolMetaReplica::Corrupt(err.to_string()),
5259 }
5260}
5261
5262#[cfg(test)]
5263pub(crate) fn pool_meta_v3_commit_state_for_test(data: Vec<u8>) -> Result<(u64, bool)> {
5264 match decode_pool_meta_replica(data) {
5265 PoolMetaReplica::Valid { revision, committed, .. } if revision.is_generation_protocol() => {
5266 Ok((revision.generation, committed))
5267 }
5268 _ => Err(Error::other("test pool metadata is not a valid V3 replica")),
5269 }
5270}
5271
5272async fn read_pool_meta_replica<S>(pool: Arc<S>, no_lock: bool) -> PoolMetaReplicaRead
5273where
5274 S: EcstoreObjectIO,
5275{
5276 let result = if no_lock {
5277 read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME)
5278 .await
5279 .map(|(data, object_info)| (data, object_info.etag))
5280 } else {
5281 read_config_preserve_empty(pool, POOL_META_NAME)
5282 .await
5283 .map(|data| (data, None))
5284 };
5285 match result {
5286 Ok((data, etag)) => PoolMetaReplicaRead {
5287 replica: decode_pool_meta_replica(data),
5288 cas: etag
5289 .filter(|etag| !etag.trim().is_empty())
5290 .map(PoolMetaCasToken::Existing)
5291 .unwrap_or(PoolMetaCasToken::Unsafe),
5292 },
5293 Err(Error::ConfigNotFound) => PoolMetaReplicaRead {
5294 replica: PoolMetaReplica::Missing,
5295 cas: PoolMetaCasToken::Missing,
5296 },
5297 Err(err) => PoolMetaReplicaRead {
5298 replica: PoolMetaReplica::Unreadable(Arc::new(err)),
5299 cas: PoolMetaCasToken::Unsafe,
5300 },
5301 }
5302}
5303
5304fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<PoolMetaSelection> {
5305 if reads.is_empty() {
5306 return Err(Error::other("pool metadata recovery required: no storage pools available"));
5307 }
5308
5309 let cas_tokens = reads.iter().map(|read| read.cas.clone()).collect();
5310 let mut committed = Vec::<(usize, Vec<u8>, PoolMetaCommittedCandidate)>::new();
5311 let mut needs_repair = false;
5312 let mut repair_write_safe = true;
5313 let mut missing = 0usize;
5314 let mut unusable = Vec::new();
5315 let mut observed_version = POOL_META_V1_VERSION;
5316 let mut generation_protocol_observed = false;
5317 let mut generation_identity = None;
5318
5319 for (idx, read) in reads.into_iter().enumerate() {
5320 match read.replica {
5321 PoolMetaReplica::Missing => {
5322 missing += 1;
5323 needs_repair = true;
5324 }
5325 PoolMetaReplica::Corrupt(reason) => {
5326 needs_repair = true;
5327 unusable.push(format!("pool {idx} is corrupt: {reason}"));
5328 }
5329 PoolMetaReplica::Unreadable(reason) => {
5330 needs_repair = true;
5331 repair_write_safe = false;
5332 unusable.push(format!("pool {idx} is unreadable: {reason}"));
5333 }
5334 PoolMetaReplica::Incompatible(reason) => {
5335 return Err(Error::other(format!(
5336 "pool metadata recovery required: pool {idx} is incompatible ({reason}); upgrade or restore a compatible replica without overwriting it"
5337 )));
5338 }
5339 PoolMetaReplica::Valid {
5340 raw,
5341 canonical,
5342 meta,
5343 revision,
5344 committed: is_committed,
5345 previous,
5346 } => {
5347 generation_protocol_observed |= revision.is_generation_protocol();
5348 if revision.is_generation_protocol() {
5349 let identity = (
5350 revision
5351 .cluster_id
5352 .expect("validated V3 revision should carry a cluster identity"),
5353 revision.epoch,
5354 );
5355 if generation_identity.is_some_and(|current| current != identity) {
5356 return Err(Error::other(
5357 "pool metadata recovery required: V3 replicas disagree on cluster identity or epoch",
5358 ));
5359 }
5360 generation_identity = Some(identity);
5361 }
5362 if is_committed {
5363 observed_version = observed_version.max(meta.version);
5364 committed.push((
5365 idx,
5366 raw,
5367 PoolMetaCommittedCandidate {
5368 canonical,
5369 meta,
5370 revision,
5371 },
5372 ));
5373 } else if let Some(previous) = previous {
5374 observed_version = observed_version.max(previous.meta.version);
5375 needs_repair = true;
5376 committed.push((idx, raw, *previous));
5377 } else {
5378 needs_repair = true;
5379 unusable.push(format!("pool {idx} contains an uncommitted initial generation"));
5380 }
5381 }
5382 }
5383 }
5384
5385 if committed.is_empty() && missing > 0 && unusable.is_empty() {
5386 return Ok(PoolMetaSelection {
5387 meta: PoolMeta::default(),
5388 revision: PoolMetaRevision::legacy(0),
5389 canonical: None,
5390 replica_state: PoolMetaReplicaState {
5391 needs_repair: false,
5392 repair_write_safe: true,
5393 },
5394 cas_tokens,
5395 absent: true,
5396 generation_protocol_observed,
5397 generation_identity,
5398 });
5399 }
5400 if committed.is_empty() {
5401 return Err(Error::other(format!(
5402 "pool metadata recovery required: no valid committed replica is available ({})",
5403 unusable.join("; ")
5404 )));
5405 }
5406
5407 if committed
5408 .iter()
5409 .any(|(_, _, candidate)| candidate.revision.is_generation_protocol())
5410 {
5411 let highest = committed
5412 .iter()
5413 .filter(|(_, _, candidate)| candidate.revision.is_generation_protocol())
5414 .map(|(_, _, candidate)| candidate.revision.generation)
5415 .max()
5416 .ok_or_else(|| Error::other("pool metadata recovery required: no committed V3 generation is available"))?;
5417 let mut selected: Option<(usize, &PoolMetaCommittedCandidate)> = None;
5418 for (idx, _, candidate) in &committed {
5419 if !candidate.revision.is_generation_protocol() || candidate.revision.generation != highest {
5420 needs_repair = true;
5421 continue;
5422 }
5423 if let Some((selected_idx, selected_candidate)) = selected {
5424 if selected_candidate.canonical != candidate.canonical
5425 || selected_candidate.revision.transaction_id != candidate.revision.transaction_id
5426 {
5427 return Err(Error::other(format!(
5428 "pool metadata recovery required: committed generation {highest} diverges between pools {selected_idx} and {idx}"
5429 )));
5430 }
5431 } else {
5432 selected = Some((*idx, candidate));
5433 }
5434 }
5435 let (_, selected) = selected
5436 .ok_or_else(|| Error::other("pool metadata recovery required: highest V3 generation has no valid snapshot"))?;
5437 for (_, raw, candidate) in &committed {
5438 needs_repair |= candidate.canonical != selected.canonical || raw != &selected.canonical;
5439 }
5440 return Ok(PoolMetaSelection {
5441 meta: selected.meta.clone(),
5442 revision: selected.revision,
5443 canonical: Some(selected.canonical.clone()),
5444 replica_state: PoolMetaReplicaState {
5445 needs_repair,
5446 repair_write_safe,
5447 },
5448 cas_tokens,
5449 absent: false,
5450 generation_protocol_observed,
5451 generation_identity,
5452 });
5453 }
5454
5455 let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
5458 for (idx, raw, candidate) in committed {
5459 if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
5460 if selected_candidate.canonical != candidate.canonical {
5461 if selected_candidate.meta.version == candidate.meta.version && *selected_idx == 0 {
5462 needs_repair = true;
5463 } else {
5464 return Err(Error::other(format!(
5465 "pool metadata recovery required: valid replicas in pools {selected_idx} and {idx} diverge; restore one matching pool.bin snapshot before restart"
5466 )));
5467 }
5468 } else {
5469 needs_repair |= selected_raw != &raw;
5470 }
5471 } else {
5472 selected = Some((idx, raw, candidate));
5473 }
5474 }
5475 let (_, _, mut selected) =
5476 selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
5477 selected.meta.version = observed_version;
5478 selected.revision.version = observed_version;
5479 Ok(PoolMetaSelection {
5480 meta: selected.meta,
5481 revision: selected.revision,
5482 canonical: Some(selected.canonical),
5483 replica_state: PoolMetaReplicaState {
5484 needs_repair,
5485 repair_write_safe,
5486 },
5487 cas_tokens,
5488 absent: false,
5489 generation_protocol_observed,
5490 generation_identity,
5491 })
5492}
5493
5494#[cfg(test)]
5495fn select_pool_meta_replica(replicas: Vec<PoolMetaReplica>) -> Result<PoolMetaSelection> {
5496 select_pool_meta_replica_reads(
5497 replicas
5498 .into_iter()
5499 .map(|replica| PoolMetaReplicaRead {
5500 replica,
5501 cas: PoolMetaCasToken::Unsafe,
5502 })
5503 .collect(),
5504 )
5505}
5506
5507async fn read_pool_meta_replicas<S>(pools: Vec<Arc<S>>, no_lock: bool) -> Vec<PoolMetaReplicaRead>
5508where
5509 S: EcstoreObjectIO,
5510{
5511 let reads = join_all(pools.into_iter().map(|pool| read_pool_meta_replica(pool, no_lock))).await;
5512 #[cfg(feature = "e2e-test-hooks")]
5513 if STARTUP_CAS_OBSERVATION.try_with(|_| ()).is_ok() {
5514 let batch = uuid::Uuid::new_v4();
5515 for (pool, read) in reads.iter().enumerate() {
5516 let mut observation = serde_json::json!({
5517 "kind": "replica-read", "object": POOL_META_NAME, "batch": batch, "pool": pool,
5518 "cas": match &read.cas {
5519 PoolMetaCasToken::Missing => "missing",
5520 PoolMetaCasToken::Existing(_) => "existing",
5521 PoolMetaCasToken::Unsafe => "unsafe",
5522 },
5523 "etag": match &read.cas { PoolMetaCasToken::Existing(etag) => Some(etag), _ => None },
5524 });
5525 match &read.replica {
5526 PoolMetaReplica::Valid {
5527 raw,
5528 canonical,
5529 meta,
5530 revision,
5531 committed,
5532 previous,
5533 } => {
5534 observation["state"] = serde_json::json!("valid");
5535 observation["committed"] = serde_json::json!(committed);
5536 observation["version"] = serde_json::json!(revision.version);
5537 observation["cluster_id"] = serde_json::json!(revision.cluster_id);
5538 observation["epoch"] = serde_json::json!(revision.epoch);
5539 observation["generation"] = serde_json::json!(revision.generation);
5540 observation["transaction_id"] = serde_json::json!(revision.transaction_id);
5541 observation["pool_count"] = serde_json::json!(meta.pools.len());
5542 observation["payload_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(canonical)));
5543 observation["raw_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(raw)));
5544 match pool_meta_test_pools_sha256(meta) {
5545 Ok(digest) => observation["persisted_pools_sha256"] = serde_json::json!(digest),
5546 Err(err) => observation["observation_error"] = serde_json::json!(err.to_string()),
5547 }
5548 if let Some(previous) = previous {
5549 match pool_meta_test_snapshot(&previous.meta, previous.revision, &previous.canonical) {
5550 Ok(summary) => observation["previous"] = summary,
5551 Err(err) => observation["observation_error"] = serde_json::json!(err.to_string()),
5552 }
5553 }
5554 }
5555 PoolMetaReplica::Missing => observation["state"] = serde_json::json!("missing"),
5556 PoolMetaReplica::Corrupt(_) => observation["state"] = serde_json::json!("corrupt"),
5557 PoolMetaReplica::Incompatible(_) => observation["state"] = serde_json::json!("incompatible"),
5558 PoolMetaReplica::Unreadable(_) => observation["state"] = serde_json::json!("unreadable"),
5559 }
5560 startup_cas_test_observe(observation);
5561 }
5562 }
5563 reads
5564}
5565
5566fn select_pool_meta_replicas_observing<R>(write_state: &mut PoolMetaWriteState, replicas: Vec<R>) -> Result<PoolMetaSelection>
5567where
5568 R: Into<PoolMetaReplicaRead>,
5569{
5570 let replicas = replicas.into_iter().map(Into::into).collect::<Vec<_>>();
5571 if let Some(err) = pool_meta_replica_read_failure(&replicas, "pool_read") {
5572 return Err(err);
5573 }
5574 match select_pool_meta_replica_reads(replicas) {
5575 Ok(selection) => {
5576 if let Err(err) = write_state.observe_selection(&selection) {
5577 return Err(block_pool_meta_validation(write_state, err, "pool_selection"));
5578 }
5579 Ok(selection)
5580 }
5581 Err(err) => Err(block_pool_meta_validation(write_state, err, "pool_selection")),
5582 }
5583}
5584
5585fn block_pool_meta_validation(write_state: &mut PoolMetaWriteState, err: Error, phase: &'static str) -> Error {
5586 let context = pool_metadata_error(crate::error::PoolMetadataFailure::RecoveryRequired, phase, Some(Arc::new(err)));
5587 write_state.block_with_context(context.clone());
5588 Error::other(context)
5589}
5590
5591fn pool_meta_replica_read_failure(replicas: &[PoolMetaReplicaRead], operation: &str) -> Option<Error> {
5592 replicas.iter().find_map(|read| match &read.replica {
5593 PoolMetaReplica::Unreadable(source) => {
5594 let mut context =
5595 pool_metadata_error(crate::error::PoolMetadataFailure::ReadUnavailable, "pool_read", Some(Arc::clone(source)));
5596 context.operation = operation.to_owned();
5597 Some(Error::other(context))
5598 }
5599 _ => None,
5600 })
5601}
5602
5603fn select_pool_meta_replicas_for_read_probe<R>(
5604 write_state: &PoolMetaWriteState,
5605 replicas: Vec<R>,
5606 operation: &str,
5607) -> Result<PoolMetaSelection>
5608where
5609 R: Into<PoolMetaReplicaRead>,
5610{
5611 let replicas = replicas.into_iter().map(Into::into).collect::<Vec<_>>();
5612 if let Some(err) = pool_meta_replica_read_failure(&replicas, operation) {
5613 return Err(err);
5614 }
5615 let selection = select_pool_meta_replica_reads(replicas)?;
5616 write_state.validate_selection(&selection)?;
5617 selection.replica_state.ensure_write_safe(operation)?;
5618 if selection.absent && (write_state.expected_cluster_id.is_some() || write_state.identity_initialized.is_some()) {
5619 write_state
5620 .validate_missing_metadata_can_initialize()
5621 .map_err(MissingMetadataRejection::into_error)?;
5622 }
5623 Ok(selection)
5624}
5625
5626async fn load_pool_meta_replicas<S>(pools: Vec<Arc<S>>, no_lock: bool) -> Result<PoolMetaSelection>
5627where
5628 S: EcstoreObjectIO,
5629{
5630 select_pool_meta_replica_reads(read_pool_meta_replicas(pools, no_lock).await)
5631}
5632
5633async fn load_pool_meta_replicas_observing<S>(
5634 pools: Vec<Arc<S>>,
5635 no_lock: bool,
5636 write_state: &mut PoolMetaWriteState,
5637) -> Result<PoolMetaSelection>
5638where
5639 S: EcstoreObjectIO,
5640{
5641 let replicas = read_pool_meta_replicas(pools, no_lock).await;
5642 select_pool_meta_replicas_observing(write_state, replicas)
5643}
5644
5645async fn load_pool_meta_replicas_for_read_probe<S>(
5646 pools: Vec<Arc<S>>,
5647 no_lock: bool,
5648 write_state: &PoolMetaWriteState,
5649 operation: &str,
5650) -> Result<PoolMetaSelection>
5651where
5652 S: EcstoreObjectIO,
5653{
5654 let replicas = read_pool_meta_replicas(pools, no_lock).await;
5655 select_pool_meta_replicas_for_read_probe(write_state, replicas, operation).map_err(|err| {
5656 if err.pool_metadata_failure().is_some() {
5657 err
5658 } else {
5659 Error::other(pool_metadata_error(
5660 crate::error::PoolMetadataFailure::RecoveryRequired,
5661 "read_probe",
5662 Some(Arc::new(err)),
5663 ))
5664 }
5665 })
5666}
5667
5668#[derive(Debug, Clone, Serialize, Deserialize)]
5669#[serde(deny_unknown_fields)]
5670struct PersistedPoolMetaV3 {
5671 version: u16,
5672 cluster_id: String,
5673 epoch: u64,
5674 generation: u64,
5675 transaction_id: String,
5676 committed: bool,
5677 pools: Vec<PersistedPoolStatus>,
5678 previous: Option<PersistedPoolMetaV3Previous>,
5679}
5680
5681#[derive(Debug, Clone, Serialize, Deserialize)]
5682#[serde(deny_unknown_fields)]
5683struct PersistedPoolMetaV3Previous {
5684 version: u16,
5685 cluster_id: Option<String>,
5686 epoch: u64,
5687 generation: u64,
5688 transaction_id: Option<String>,
5689 pools: Vec<PersistedPoolStatus>,
5690}
5691
5692#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
5693#[serde(deny_unknown_fields)]
5694struct PersistedPoolMetaIdentity {
5695 version: u16,
5696 cluster_id: uuid::Uuid,
5697 epoch: u64,
5698 initialized: bool,
5699 fresh_bootstrap_nonce: Option<uuid::Uuid>,
5700}
5701
5702#[derive(Debug)]
5703enum PoolMetaIdentityReplica {
5704 Missing,
5705 Valid(PersistedPoolMetaIdentity),
5706 Corrupt(String),
5707 Incompatible(String),
5708 Unreadable(Arc<Error>),
5709}
5710
5711#[derive(Debug)]
5712struct PoolMetaIdentityRead {
5713 replica: PoolMetaIdentityReplica,
5714 cas: PoolMetaCasToken,
5715}
5716
5717#[derive(Debug)]
5718struct PoolMetaIdentitySelection {
5719 identity: Option<PersistedPoolMetaIdentity>,
5720 needs_repair: bool,
5721 repair_write_safe: bool,
5722 cas_tokens: Vec<PoolMetaCasToken>,
5723 valid_replicas: Vec<bool>,
5725}
5726
5727fn encode_pool_meta_identity(identity: PersistedPoolMetaIdentity) -> Result<Vec<u8>> {
5728 let mut data = Vec::new();
5729 data.write_u16::<LittleEndian>(POOL_META_IDENTITY_FORMAT)?;
5730 data.write_u16::<LittleEndian>(POOL_META_IDENTITY_VERSION)?;
5731 identity.serialize(&mut Serializer::new(&mut data))?;
5732 Ok(data)
5733}
5734
5735fn decode_pool_meta_identity(data: &[u8]) -> PoolMetaIdentityReplica {
5736 if data.len() <= 4 {
5737 return PoolMetaIdentityReplica::Corrupt("identity payload is empty or truncated".to_string());
5738 }
5739 let format = LittleEndian::read_u16(&data[0..2]);
5740 let version = LittleEndian::read_u16(&data[2..4]);
5741 if format != POOL_META_IDENTITY_FORMAT || version != POOL_META_IDENTITY_VERSION {
5742 return PoolMetaIdentityReplica::Incompatible(format!("unsupported identity format {format} version {version}"));
5743 }
5744 let identity = match rmp_serde::from_slice::<PersistedPoolMetaIdentity>(&data[4..]) {
5745 Ok(identity) => identity,
5746 Err(err) => return PoolMetaIdentityReplica::Corrupt(format!("identity payload is not decodable: {err}")),
5747 };
5748 let invalid_bootstrap_nonce = identity
5749 .fresh_bootstrap_nonce
5750 .is_some_and(|nonce| nonce.is_nil() || nonce == uuid::Uuid::max());
5751 if identity.version != POOL_META_IDENTITY_VERSION
5752 || identity.cluster_id.is_nil()
5753 || identity.cluster_id == uuid::Uuid::max()
5754 || identity.epoch == 0
5755 || invalid_bootstrap_nonce
5756 || identity.initialized == identity.fresh_bootstrap_nonce.is_some()
5757 {
5758 return PoolMetaIdentityReplica::Corrupt(
5759 "identity payload contains an invalid version, UUID, epoch, or fresh-bootstrap proof".to_string(),
5760 );
5761 }
5762 PoolMetaIdentityReplica::Valid(identity)
5763}
5764
5765#[cfg(test)]
5766pub(crate) fn pool_meta_identity_initialized_for_test(data: &[u8]) -> Result<bool> {
5767 match decode_pool_meta_identity(data) {
5768 PoolMetaIdentityReplica::Valid(identity) => Ok(identity.initialized),
5769 _ => Err(Error::other("test pool metadata identity is not valid")),
5770 }
5771}
5772
5773#[cfg(test)]
5774pub(crate) fn pending_pool_meta_identity_for_test(cluster_id: uuid::Uuid, epoch: u64, nonce: uuid::Uuid) -> Result<Vec<u8>> {
5775 encode_pool_meta_identity(PersistedPoolMetaIdentity {
5776 version: POOL_META_IDENTITY_VERSION,
5777 cluster_id,
5778 epoch,
5779 initialized: false,
5780 fresh_bootstrap_nonce: Some(nonce),
5781 })
5782}
5783
5784#[cfg(test)]
5785pub(crate) fn initialized_pool_meta_identity_for_test(cluster_id: uuid::Uuid, epoch: u64) -> Result<Vec<u8>> {
5786 encode_pool_meta_identity(PersistedPoolMetaIdentity {
5787 version: POOL_META_IDENTITY_VERSION,
5788 cluster_id,
5789 epoch,
5790 initialized: true,
5791 fresh_bootstrap_nonce: None,
5792 })
5793}
5794
5795async fn read_pool_meta_identity_replica<S>(pool: Arc<S>) -> PoolMetaIdentityRead
5796where
5797 S: EcstoreObjectIO,
5798{
5799 match read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_IDENTITY_NAME).await {
5800 Ok((data, object_info)) => PoolMetaIdentityRead {
5801 replica: decode_pool_meta_identity(&data),
5802 cas: object_info
5803 .etag
5804 .filter(|etag| !etag.trim().is_empty())
5805 .map(PoolMetaCasToken::Existing)
5806 .unwrap_or(PoolMetaCasToken::Unsafe),
5807 },
5808 Err(Error::ConfigNotFound) => PoolMetaIdentityRead {
5809 replica: PoolMetaIdentityReplica::Missing,
5810 cas: PoolMetaCasToken::Missing,
5811 },
5812 Err(err) => PoolMetaIdentityRead {
5813 replica: PoolMetaIdentityReplica::Unreadable(Arc::new(err)),
5814 cas: PoolMetaCasToken::Unsafe,
5815 },
5816 }
5817}
5818
5819fn select_pool_meta_identity(
5820 reads: Vec<PoolMetaIdentityRead>,
5821 expected_cluster_id: uuid::Uuid,
5822) -> Result<PoolMetaIdentitySelection> {
5823 let cas_tokens = reads.iter().map(|read| read.cas.clone()).collect();
5824 let valid_replicas = reads
5825 .iter()
5826 .map(|read| matches!(read.replica, PoolMetaIdentityReplica::Valid(_)))
5827 .collect();
5828 let mut selected: Option<PersistedPoolMetaIdentity> = None;
5829 let mut needs_repair = false;
5830 let mut repair_write_safe = true;
5831 let mut unusable = Vec::new();
5832 for (idx, read) in reads.into_iter().enumerate() {
5833 match read.replica {
5834 PoolMetaIdentityReplica::Missing => needs_repair = true,
5835 PoolMetaIdentityReplica::Corrupt(reason) => {
5836 needs_repair = true;
5837 unusable.push(format!("pool {idx} identity is corrupt: {reason}"));
5838 }
5839 PoolMetaIdentityReplica::Unreadable(reason) => {
5840 needs_repair = true;
5841 repair_write_safe = false;
5842 unusable.push(format!("pool {idx} identity is unreadable: {reason}"));
5843 }
5844 PoolMetaIdentityReplica::Incompatible(reason) => {
5845 return Err(Error::other(format!("pool metadata incompatible: pool {idx} identity uses {reason}")));
5846 }
5847 PoolMetaIdentityReplica::Valid(identity) => {
5848 if identity.cluster_id != expected_cluster_id {
5849 return Err(Error::other(format!(
5850 "pool metadata incompatible: pool {idx} identity {} does not match deployment {expected_cluster_id}",
5851 identity.cluster_id
5852 )));
5853 }
5854 if let Some(current) = selected {
5855 if current.cluster_id != identity.cluster_id || current.epoch != identity.epoch {
5856 return Err(Error::other(
5857 "pool metadata recovery required: identity replicas disagree on cluster identity or epoch",
5858 ));
5859 }
5860 if current.initialized != identity.initialized {
5861 needs_repair = true;
5862 selected = Some(if current.initialized { current } else { identity });
5863 } else if !current.initialized && current.fresh_bootstrap_nonce != identity.fresh_bootstrap_nonce {
5864 return Err(Error::other(
5865 "pool metadata recovery required: pending identity replicas disagree on fresh-bootstrap proof",
5866 ));
5867 }
5868 } else {
5869 selected = Some(identity);
5870 }
5871 }
5872 }
5873 }
5874 if selected.is_none() && !unusable.is_empty() {
5875 return Err(Error::other(format!(
5876 "pool metadata recovery required: no valid cluster identity replica is available ({})",
5877 unusable.join("; ")
5878 )));
5879 }
5880 Ok(PoolMetaIdentitySelection {
5881 identity: selected,
5882 needs_repair,
5883 repair_write_safe,
5884 cas_tokens,
5885 valid_replicas,
5886 })
5887}
5888
5889async fn load_pool_meta_identity<S>(pools: Vec<Arc<S>>, expected_cluster_id: uuid::Uuid) -> Result<PoolMetaIdentitySelection>
5890where
5891 S: EcstoreObjectIO,
5892{
5893 let reads = join_all(pools.into_iter().map(read_pool_meta_identity_replica)).await;
5894 if let Some(source) = reads.iter().find_map(|read| match &read.replica {
5895 PoolMetaIdentityReplica::Unreadable(source) => Some(Arc::clone(source)),
5896 _ => None,
5897 }) {
5898 return Err(Error::other(pool_metadata_error(
5899 crate::error::PoolMetadataFailure::ReadUnavailable,
5900 "identity_read",
5901 Some(source),
5902 )));
5903 }
5904 select_pool_meta_identity(reads, expected_cluster_id)
5905}
5906
5907async fn load_pool_meta_identity_selection_observing<S>(
5908 pools: Vec<Arc<S>>,
5909 write_state: &mut PoolMetaWriteState,
5910 expected_cluster_id: uuid::Uuid,
5911) -> Result<PoolMetaIdentitySelection>
5912where
5913 S: EcstoreObjectIO,
5914{
5915 let selection = match load_pool_meta_identity(pools, expected_cluster_id).await {
5916 Ok(selection) => selection,
5917 Err(err) => {
5918 if err
5919 .pool_metadata_failure()
5920 .is_some_and(|context| context.kind == crate::error::PoolMetadataFailure::ReadUnavailable)
5921 {
5922 return Err(err);
5923 }
5924 return Err(block_pool_meta_validation(write_state, err, "identity_selection"));
5925 }
5926 };
5927 if let Err(err) = write_state.observe_identity(&selection) {
5928 return Err(block_pool_meta_validation(write_state, err, "identity_selection"));
5929 }
5930 Ok(selection)
5931}
5932
5933fn persisted_pool_meta_v3_previous(candidate: &PoolMetaCommittedCandidate) -> PersistedPoolMetaV3Previous {
5934 PersistedPoolMetaV3Previous {
5935 version: candidate.revision.version,
5936 cluster_id: candidate.revision.cluster_id.map(|id| id.to_string()),
5937 epoch: candidate.revision.epoch,
5938 generation: candidate.revision.generation,
5939 transaction_id: candidate.revision.transaction_id.map(|id| id.to_string()),
5940 pools: candidate.meta.pools.iter().map(Into::into).collect(),
5941 }
5942}
5943
5944fn encode_pool_meta_v3_envelope(
5945 meta: &PoolMeta,
5946 revision: PoolMetaRevision,
5947 committed: bool,
5948 previous: Option<&PoolMetaCommittedCandidate>,
5949) -> Result<Vec<u8>> {
5950 if meta.dont_save {
5951 return Ok(Vec::new());
5952 }
5953 let cluster_id = revision
5954 .cluster_id
5955 .ok_or_else(|| Error::other("pool metadata V3 save failed: cluster identity is not initialized"))?;
5956 let transaction_id = revision
5957 .transaction_id
5958 .ok_or_else(|| Error::other("pool metadata V3 save failed: transaction id is not initialized"))?;
5959 if revision.version != POOL_META_GENERATION_VERSION || revision.epoch == 0 || revision.generation == 0 {
5960 return Err(Error::other("pool metadata V3 save failed: invalid durable revision"));
5961 }
5962 let persisted = PersistedPoolMetaV3 {
5963 version: POOL_META_GENERATION_VERSION,
5964 cluster_id: cluster_id.to_string(),
5965 epoch: revision.epoch,
5966 generation: revision.generation,
5967 transaction_id: transaction_id.to_string(),
5968 committed,
5969 pools: meta.pools.iter().map(Into::into).collect(),
5970 previous: previous.map(persisted_pool_meta_v3_previous),
5971 };
5972 let mut data = Vec::new();
5973 data.write_u16::<LittleEndian>(POOL_META_FORMAT)?;
5974 data.write_u16::<LittleEndian>(POOL_META_GENERATION_VERSION)?;
5975 persisted.serialize(&mut Serializer::new(&mut data))?;
5976 Ok(data)
5977}
5978
5979enum PoolMetaPersistenceFence<'a> {
5980 Distributed(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>),
5981 Activation(&'a PoolRebalanceActivationFence),
5982}
5983
5984impl PoolMetaPersistenceFence<'_> {
5985 fn ensure_held(&self) -> Result<()> {
5986 match self {
5987 Self::Distributed(Some(signal)) if signal.is_lost() => {
5988 Err(Error::other("pool metadata distributed fence was lost before a replica write"))
5989 }
5990 Self::Activation(fence) => fence.ensure_held(),
5991 _ => Ok(()),
5992 }
5993 }
5994
5995 fn add_to_options(&self, opts: &mut ObjectOptions) {
5996 match self {
5997 Self::Distributed(Some(signal)) => opts.add_namespace_lock_lost_signal(Arc::clone(signal)),
5998 Self::Activation(fence) => fence.add_namespace_lock_fence(opts),
5999 Self::Distributed(None) => {}
6000 }
6001 }
6002
6003 fn is_activation(&self) -> bool {
6004 matches!(self, Self::Activation(_))
6005 }
6006}
6007
6008fn pool_meta_cas_preconditions(token: &PoolMetaCasToken, object: &str) -> Result<HTTPPreconditions> {
6009 match token {
6010 PoolMetaCasToken::Missing => Ok(HTTPPreconditions {
6011 if_none_match: Some("*".to_string()),
6012 ..Default::default()
6013 }),
6014 PoolMetaCasToken::Existing(etag) => Ok(HTTPPreconditions {
6015 if_match: Some(etag.clone()),
6016 ..Default::default()
6017 }),
6018 PoolMetaCasToken::Unsafe => Err(Error::other(format!(
6019 "pool metadata recovery required: {object} replica has no safe conditional-write revision"
6020 ))),
6021 }
6022}
6023
6024#[cfg(feature = "e2e-test-hooks")]
6025struct StartupCasObservation {
6026 attempt: uuid::Uuid,
6027 phase: &'static str,
6028 pools: Vec<usize>,
6029}
6030
6031#[cfg(feature = "e2e-test-hooks")]
6032tokio::task_local! {
6033 static STARTUP_CAS_OBSERVATION: StartupCasObservation;
6034}
6035
6036#[cfg(feature = "e2e-test-hooks")]
6039pub(crate) async fn startup_cas_test_scope<S, F: std::future::Future>(
6040 attempt: uuid::Uuid,
6041 phase: &'static str,
6042 pools: &[Arc<S>],
6043 future: F,
6044) -> F::Output {
6045 STARTUP_CAS_OBSERVATION
6046 .scope(
6047 StartupCasObservation {
6048 attempt,
6049 phase,
6050 pools: pools.iter().map(|pool| Arc::as_ptr(pool) as usize).collect(),
6053 },
6054 future,
6055 )
6056 .await
6057}
6058
6059#[cfg(feature = "e2e-test-hooks")]
6061pub(crate) fn startup_cas_test_observe(mut observation: serde_json::Value) {
6062 let Some(nonce) = std::env::var("RUSTFS_E2E_STARTUP_CAS_NONCE")
6063 .ok()
6064 .and_then(|value| uuid::Uuid::parse_str(&value).ok())
6065 else {
6066 return;
6067 };
6068 observation["nonce"] = serde_json::json!(nonce);
6069 observation["pid"] = serde_json::json!(std::process::id());
6070 let _ = STARTUP_CAS_OBSERVATION.try_with(|scope| {
6071 observation["attempt"] = serde_json::json!(scope.attempt);
6072 observation["startup_phase"] = serde_json::json!(scope.phase);
6073 });
6074 let line = format!("RUSTFS_E2E_STARTUP_CAS {observation}\n");
6075 let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes());
6076}
6077
6078#[cfg(feature = "e2e-test-hooks")]
6079fn pool_meta_test_pools_sha256(meta: &PoolMeta) -> Result<String> {
6080 let pools = meta.pools.iter().map(PersistedPoolStatus::from).collect::<Vec<_>>();
6081 Ok(rustfs_utils::crypto::hex(Sha256::digest(serde_json::to_vec(&pools)?)))
6082}
6083
6084#[cfg(feature = "e2e-test-hooks")]
6085fn pool_meta_test_snapshot(meta: &PoolMeta, revision: PoolMetaRevision, canonical: &[u8]) -> Result<serde_json::Value> {
6086 Ok(serde_json::json!({
6087 "version": revision.version, "cluster_id": revision.cluster_id, "epoch": revision.epoch,
6088 "generation": revision.generation, "transaction_id": revision.transaction_id,
6089 "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(canonical)),
6090 "persisted_pools_sha256": pool_meta_test_pools_sha256(meta)?,
6091 }))
6092}
6093
6094#[cfg(feature = "e2e-test-hooks")]
6095#[derive(Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
6096#[serde(rename_all = "snake_case")]
6097enum PoolMetaPhaseBarrierCase {
6098 PrepareSubset,
6099 PreparedAll,
6100 CommitOne,
6101 BeforePublish,
6102}
6103
6104#[cfg(feature = "e2e-test-hooks")]
6105#[derive(Deserialize)]
6106#[serde(deny_unknown_fields)]
6107struct PoolMetaPhaseBarrierArm {
6108 nonce: uuid::Uuid,
6109 case: PoolMetaPhaseBarrierCase,
6110}
6111
6112#[cfg(feature = "e2e-test-hooks")]
6113struct PoolMetaPhaseBarrier {
6114 directory: std::path::PathBuf,
6115 arm: PoolMetaPhaseBarrierArm,
6116 transaction_id: uuid::Uuid,
6117 event_gate: tokio::sync::Mutex<()>,
6118 blocked_pools: AtomicUsize,
6119 completed_writes: AtomicUsize,
6120 ready_emitted: AtomicBool,
6121}
6122
6123#[cfg(feature = "e2e-test-hooks")]
6124impl PoolMetaPhaseBarrier {
6125 async fn bind(
6126 previous: &PoolMetaCommittedCandidate,
6127 candidate: &PoolMeta,
6128 revision: PoolMetaRevision,
6129 durable: &[u8],
6130 ) -> Result<Option<Self>> {
6131 let Some(directory) = std::env::var_os("RUSTFS_E2E_POOL_META_BARRIER_DIR").map(std::path::PathBuf::from) else {
6132 return Ok(None);
6133 };
6134 Self::bind_in_directory(directory, previous, candidate, revision, durable).await
6135 }
6136
6137 async fn bind_in_directory(
6138 directory: std::path::PathBuf,
6139 previous: &PoolMetaCommittedCandidate,
6140 candidate: &PoolMeta,
6141 revision: PoolMetaRevision,
6142 durable: &[u8],
6143 ) -> Result<Option<Self>> {
6144 let data = match tokio::fs::read(directory.join("arm.json")).await {
6145 Ok(data) => data,
6146 Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
6147 Err(err) => return Err(err.into()),
6148 };
6149 let arm: PoolMetaPhaseBarrierArm = serde_json::from_slice(&data)?;
6150 if arm.nonce.is_nil() || !directory.is_absolute() {
6151 return Err(Error::other(
6152 "pool metadata test barrier requires an absolute directory and non-nil nonce",
6153 ));
6154 }
6155 let transaction_id = revision
6156 .transaction_id
6157 .ok_or_else(|| Error::other("pool metadata test barrier requires a V3 transaction"))?;
6158 if pool_meta_test_pools_sha256(&previous.meta)? == pool_meta_test_pools_sha256(candidate)? {
6161 return Ok(None);
6162 }
6163 let claim = tokio::fs::OpenOptions::new()
6165 .write(true)
6166 .create_new(true)
6167 .open(directory.join("claimed"))
6168 .await;
6169 match claim {
6170 Ok(_) => {}
6171 Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => return Ok(None),
6172 Err(err) => return Err(err.into()),
6173 }
6174 let barrier = Self {
6175 directory,
6176 arm,
6177 transaction_id,
6178 event_gate: tokio::sync::Mutex::new(()),
6179 blocked_pools: AtomicUsize::new(0),
6180 completed_writes: AtomicUsize::new(0),
6181 ready_emitted: AtomicBool::new(false),
6182 };
6183 barrier
6184 .observe(serde_json::json!({
6185 "kind": "armed",
6186 "previous": pool_meta_test_snapshot(&previous.meta, previous.revision, &previous.canonical)?,
6187 "candidate": pool_meta_test_snapshot(candidate, revision, durable)?,
6188 }))
6189 .await?;
6190 Ok(Some(barrier))
6191 }
6192
6193 async fn observe(&self, mut event: serde_json::Value) -> Result<()> {
6194 use tokio::io::AsyncWriteExt as _;
6195 event["nonce"] = serde_json::json!(self.arm.nonce);
6196 event["pid"] = serde_json::json!(std::process::id());
6197 event["transaction_id"] = serde_json::json!(self.transaction_id);
6198 event["case"] = serde_json::json!(self.arm.case);
6199 let mut line = serde_json::to_vec(&event)?;
6200 line.push(b'\n');
6201 let _event_guard = self.event_gate.lock().await;
6202 let mut file = tokio::fs::OpenOptions::new()
6203 .create(true)
6204 .append(true)
6205 .open(self.directory.join("events.jsonl"))
6206 .await?;
6207 file.write_all(&line).await?;
6208 file.flush().await?;
6209 Ok(())
6210 }
6211
6212 async fn wait_for_release(&self) -> Result<()> {
6213 tokio::time::timeout(std::time::Duration::from_secs(120), async {
6214 loop {
6215 match tokio::fs::metadata(self.directory.join("release")).await {
6216 Ok(_) => return Ok(()),
6217 Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
6218 Err(err) => return Err(Error::from(err)),
6219 }
6220 tokio::time::sleep(std::time::Duration::from_millis(10)).await;
6221 }
6222 })
6223 .await
6224 .map_err(|_| Error::other("pool metadata test barrier release timed out"))?
6225 }
6226
6227 async fn ready(&self) -> Result<()> {
6228 self.observe(serde_json::json!({"kind": "ready", "cutpoint": self.arm.case}))
6229 .await?;
6230 self.wait_for_release().await
6231 }
6232
6233 async fn ready_after_all_participants(&self) -> Result<()> {
6234 let (blocked, completed) = match self.arm.case {
6235 PoolMetaPhaseBarrierCase::PrepareSubset => (0b0101, 0b1010),
6236 PoolMetaPhaseBarrierCase::CommitOne => (0b0111, 0b1000),
6237 _ => return Ok(()),
6238 };
6239 if self.blocked_pools.load(Ordering::SeqCst) == blocked
6240 && self.completed_writes.load(Ordering::SeqCst) == completed
6241 && !self.ready_emitted.swap(true, Ordering::SeqCst)
6242 {
6243 self.ready().await?;
6244 }
6245 Ok(())
6246 }
6247
6248 async fn before_write(&self, phase: &'static str, pool: usize, data: &[u8]) -> Result<()> {
6249 let blocked = match (self.arm.case, phase) {
6250 (PoolMetaPhaseBarrierCase::PrepareSubset, "prepare_cas") => ![1, 3].contains(&pool),
6251 (PoolMetaPhaseBarrierCase::CommitOne, "commit_cas") => pool != 3,
6252 _ => false,
6253 };
6254 if blocked {
6255 self.observe(serde_json::json!({
6256 "kind": "blocked", "phase": phase, "pool": pool,
6257 "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(data)),
6258 }))
6259 .await?;
6260 self.blocked_pools.fetch_or(1 << pool, Ordering::SeqCst);
6261 self.ready_after_all_participants().await?;
6262 self.wait_for_release().await?;
6263 }
6264 self.observe(serde_json::json!({
6265 "kind": "before-dispatch", "phase": phase, "pool": pool,
6266 "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(data)),
6267 }))
6268 .await
6269 }
6270
6271 async fn after_write(&self, phase: &'static str, pool: usize, data: &[u8], outcome: &PoolMetaCasWriteOutcome) -> Result<()> {
6272 self.observe(serde_json::json!({
6273 "kind": "after-cas", "phase": phase, "pool": pool,
6274 "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(data)),
6275 "ok": outcome.result.is_ok(), "tail_drained": outcome.result.is_ok(),
6276 "etag": outcome.result.as_ref().ok().and_then(|info| info.etag.as_deref()),
6277 "may_have_mutated": outcome.may_have_mutated,
6278 }))
6279 .await?;
6280 if outcome.result.is_ok() {
6281 match (self.arm.case, phase, pool) {
6282 (PoolMetaPhaseBarrierCase::PrepareSubset, "prepare_cas", 1 | 3)
6283 | (PoolMetaPhaseBarrierCase::CommitOne, "commit_cas", 3) => {
6284 self.completed_writes.fetch_or(1 << pool, Ordering::SeqCst);
6285 self.ready_after_all_participants().await?;
6286 }
6287 _ => {}
6288 }
6289 }
6290 Ok(())
6291 }
6292}
6293
6294fn pool_meta_cas_options(object: &str, token: &PoolMetaCasToken, fence: &PoolMetaPersistenceFence<'_>) -> Result<ObjectOptions> {
6295 fence.ensure_held()?;
6296 let mut opts = ObjectOptions {
6297 max_parity: true,
6298 write_completion: crate::object_api::WriteCompletion::TailDrained,
6299 no_lock: true,
6300 http_preconditions: Some(pool_meta_cas_preconditions(token, object)?),
6301 ..Default::default()
6302 };
6303 fence.add_to_options(&mut opts);
6304 Ok(opts)
6305}
6306
6307struct PoolMetaCasWriteOutcome {
6308 result: Result<crate::object_api::ObjectInfo>,
6309 source: Option<Arc<Error>>,
6310 may_have_mutated: bool,
6311}
6312
6313impl PoolMetaCasWriteOutcome {
6314 fn not_dispatched(err: Error) -> Self {
6315 Self {
6316 result: Err(err),
6317 source: None,
6318 may_have_mutated: false,
6319 }
6320 }
6321}
6322
6323async fn execute_pool_meta_object_cas<S>(
6324 pool: Arc<S>,
6325 object: &str,
6326 data: Vec<u8>,
6327 opts: ObjectOptions,
6328 fence: &PoolMetaPersistenceFence<'_>,
6329 phase: &'static str,
6330) -> PoolMetaCasWriteOutcome
6331where
6332 S: EcstoreObjectIO,
6333{
6334 if let Err(err) = fence.ensure_held() {
6336 return PoolMetaCasWriteOutcome::not_dispatched(err);
6337 }
6338 #[cfg(feature = "e2e-test-hooks")]
6339 let observation = std::env::var_os("RUSTFS_E2E_STARTUP_CAS_NONCE").map(|_| {
6340 serde_json::json!({
6341 "kind": "cas", "object": object, "phase": phase,
6342 "pool": STARTUP_CAS_OBSERVATION.try_with(|scope| {
6343 scope.pools.iter().position(|identity| *identity == Arc::as_ptr(&pool) as usize)
6344 }).ok().flatten(),
6345 "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&data)),
6346 "if_match": opts.http_preconditions.as_ref().and_then(|p| p.if_match.as_deref()),
6347 "if_none_match": opts.http_preconditions.as_ref().and_then(|p| p.if_none_match.as_deref()),
6348 "tail_drained": opts.write_completion == crate::object_api::WriteCompletion::TailDrained,
6349 "no_lock": opts.no_lock,
6350 })
6351 });
6352 let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await;
6353 let may_have_mutated = !matches!(&result, Err(Error::PreconditionFailed));
6354 if !may_have_mutated {
6355 record_pool_meta_stale_write_rejection(phase);
6356 }
6357 let mut source = None;
6358 let result = match result {
6359 Ok(object_info) => fence.ensure_held().map(|()| object_info),
6360 Err(err) => {
6361 let original = Arc::new(err);
6362 source = Some(Arc::clone(&original));
6363 if matches!(original.as_ref(), Error::PreconditionFailed) {
6364 Err(Error::PreconditionFailed)
6365 } else {
6366 Err(Error::other(pool_metadata_error(
6367 crate::error::PoolMetadataFailure::TransactionUnknown,
6368 phase,
6369 Some(original),
6370 )))
6371 }
6372 }
6373 };
6374 #[cfg(feature = "e2e-test-hooks")]
6375 if let Some(mut observation) = observation {
6376 observation["ok"] = serde_json::json!(result.is_ok());
6377 observation["etag"] = serde_json::json!(result.as_ref().ok().and_then(|info| info.etag.as_deref()));
6378 observation["mod_time"] = serde_json::json!(
6379 result
6380 .as_ref()
6381 .ok()
6382 .and_then(|info| info.mod_time)
6383 .map(|time| time.unix_timestamp_nanos().to_string())
6384 );
6385 observation["error"] = serde_json::json!(result.as_ref().err().map(ToString::to_string));
6386 startup_cas_test_observe(observation);
6387 }
6388 PoolMetaCasWriteOutcome {
6389 result,
6390 source,
6391 may_have_mutated,
6392 }
6393}
6394
6395async fn save_pool_meta_object_cas<S>(
6396 pool: Arc<S>,
6397 object: &str,
6398 data: Vec<u8>,
6399 token: &PoolMetaCasToken,
6400 fence: &PoolMetaPersistenceFence<'_>,
6401 phase: &'static str,
6402 transaction_arm: &mut PoolMetaTransactionArm,
6403) -> Result<crate::object_api::ObjectInfo>
6404where
6405 S: EcstoreObjectIO,
6406{
6407 let opts = pool_meta_cas_options(object, token, fence)?;
6408 let previous_phase = transaction_arm.phase;
6411 transaction_arm.phase = Some(phase);
6412 let outcome = execute_pool_meta_object_cas(pool, object, data, opts, fence, phase).await;
6413 if !outcome.may_have_mutated {
6414 transaction_arm.phase = previous_phase;
6415 }
6416 if let Some(source) = outcome.source
6417 && (!matches!(source.as_ref(), Error::PreconditionFailed)
6418 || transaction_arm
6419 .source
6420 .as_ref()
6421 .is_none_or(|previous| matches!(previous.as_ref(), Error::PreconditionFailed)))
6422 {
6423 transaction_arm.source = Some(source);
6424 }
6425 outcome.result
6426}
6427
6428async fn save_pool_meta_phase<S>(
6429 pools: &[Arc<S>],
6430 data: &[u8],
6431 tokens: &[PoolMetaCasToken],
6432 fence: &PoolMetaPersistenceFence<'_>,
6433 phase: &'static str,
6434 transaction_arm: &mut PoolMetaTransactionArm,
6435 #[cfg(feature = "e2e-test-hooks")] barrier: Option<&PoolMetaPhaseBarrier>,
6436) -> Vec<Result<crate::object_api::ObjectInfo>>
6437where
6438 S: EcstoreObjectIO,
6439{
6440 if pools.len() != tokens.len() {
6441 return vec![Err(Error::other("pool metadata phase has inconsistent replica revisions"))];
6442 }
6443 let options = tokens
6444 .iter()
6445 .map(|token| pool_meta_cas_options(POOL_META_NAME, token, fence))
6446 .collect::<Vec<_>>();
6447 let previous_phase = transaction_arm.phase;
6448 if options.iter().any(Result::is_ok) {
6449 transaction_arm.phase = Some(phase);
6450 }
6451 let writes = futures::stream::iter(pools.iter().cloned().zip(options).enumerate().map(
6454 |(pool_index, (pool, opts))| async move {
6455 #[cfg(feature = "e2e-test-hooks")]
6456 if opts.is_ok()
6457 && let Some(barrier) = barrier
6458 && let Err(err) = barrier.before_write(phase, pool_index, data).await
6459 {
6460 return (pool_index, PoolMetaCasWriteOutcome::not_dispatched(err));
6461 }
6462 let outcome = match opts {
6463 Ok(opts) => execute_pool_meta_object_cas(pool, POOL_META_NAME, data.to_vec(), opts, fence, phase).await,
6464 Err(err) => PoolMetaCasWriteOutcome::not_dispatched(err),
6465 };
6466 #[cfg(feature = "e2e-test-hooks")]
6467 let outcome = {
6468 let mut outcome = outcome;
6469 if let Some(barrier) = barrier
6470 && let Err(err) = barrier.after_write(phase, pool_index, data, &outcome).await
6471 && outcome.result.is_ok()
6472 {
6473 outcome.result = Err(err);
6474 }
6475 outcome
6476 };
6477 (pool_index, outcome)
6478 },
6479 ))
6480 .buffer_unordered(4);
6481 futures::pin_mut!(writes);
6482 let mut results = Vec::with_capacity(pools.len());
6483 let mut may_have_mutated = false;
6484 let mut source_pool_index: Option<(bool, usize)> = None;
6485 while let Some((pool_index, outcome)) = writes.next().await {
6486 may_have_mutated |= outcome.may_have_mutated;
6487 if let Some(source) = outcome.source
6488 && (outcome.may_have_mutated
6489 || transaction_arm
6490 .source
6491 .as_ref()
6492 .is_none_or(|previous| matches!(previous.as_ref(), Error::PreconditionFailed)))
6493 && source_pool_index.is_none_or(|(previous_mutated, previous_index)| {
6494 (outcome.may_have_mutated && !previous_mutated)
6495 || (outcome.may_have_mutated == previous_mutated && pool_index < previous_index)
6496 })
6497 {
6498 transaction_arm.source = Some(source);
6500 source_pool_index = Some((outcome.may_have_mutated, pool_index));
6501 }
6502 results.push((pool_index, outcome.result));
6503 }
6504 if !may_have_mutated {
6505 transaction_arm.phase = previous_phase;
6506 }
6507 results.sort_unstable_by_key(|(pool_index, _)| *pool_index);
6508 results.into_iter().map(|(_, result)| result).collect()
6509}
6510
6511#[derive(Debug, Clone, Copy)]
6513enum PoolMetaIdentityWriteScope<'a> {
6514 All,
6517 Pools(&'a [usize]),
6524}
6525
6526fn identity_write_satisfied(
6527 selection: &PoolMetaIdentitySelection,
6528 identity: PersistedPoolMetaIdentity,
6529 scope: PoolMetaIdentityWriteScope<'_>,
6530 targets: &[usize],
6531) -> bool {
6532 if selection.identity != Some(identity) {
6533 return false;
6534 }
6535 match scope {
6536 PoolMetaIdentityWriteScope::All => !selection.needs_repair,
6537 PoolMetaIdentityWriteScope::Pools(_) => targets
6538 .iter()
6539 .all(|pool_idx| selection.valid_replicas.get(*pool_idx).copied().unwrap_or(false)),
6540 }
6541}
6542
6543async fn persist_pool_meta_identity<S>(
6544 pools: Vec<Arc<S>>,
6545 write_state: &mut PoolMetaWriteState,
6546 initialized: bool,
6547 scope: PoolMetaIdentityWriteScope<'_>,
6548 fence: &PoolMetaPersistenceFence<'_>,
6549 transaction_arm: &mut PoolMetaTransactionArm,
6550) -> Result<()>
6551where
6552 S: EcstoreObjectIO,
6553{
6554 let Some(cluster_id) = write_state.expected_cluster_id else {
6555 return Ok(());
6556 };
6557 let targets: Vec<usize> = match scope {
6558 PoolMetaIdentityWriteScope::All => (0..pools.len()).collect(),
6559 PoolMetaIdentityWriteScope::Pools(indices) => {
6560 if initialized {
6561 return Err(Error::other("pool metadata identity commit must address every pool"));
6562 }
6563 if indices
6564 .iter()
6565 .any(|pool_idx| *pool_idx >= pools.len() || !write_state.pool_bootstrap_authority_proven(*pool_idx))
6566 {
6567 return Err(Error::other(
6568 "pool metadata recovery required: a pending cluster identity can only be attested for pools this startup formatted or adopted first-hand",
6569 ));
6570 }
6571 indices.to_vec()
6572 }
6573 };
6574 for attempt in 0..POOL_META_CAS_MAX_ATTEMPTS {
6575 let selection = load_pool_meta_identity_selection_observing(pools.clone(), write_state, cluster_id).await?;
6576 if !selection.repair_write_safe {
6577 write_state.block_writes();
6578 return Err(Error::other(
6579 "pool metadata recovery required: cluster identity has an unreadable replica",
6580 ));
6581 }
6582 let identity = match (selection.identity, scope) {
6583 (Some(identity), PoolMetaIdentityWriteScope::Pools(_)) if identity.initialized => return Ok(()),
6587 (Some(identity), _) if identity.initialized || initialized => PersistedPoolMetaIdentity {
6588 initialized: true,
6589 fresh_bootstrap_nonce: None,
6590 ..identity
6591 },
6592 (Some(identity), _) => identity,
6593 (None, PoolMetaIdentityWriteScope::Pools(indices)) => {
6597 if !indices.contains(&0) {
6598 return Ok(());
6599 }
6600 PersistedPoolMetaIdentity {
6601 version: POOL_META_IDENTITY_VERSION,
6602 cluster_id,
6603 epoch: write_state.cluster_epoch.unwrap_or(POOL_META_INITIAL_EPOCH),
6604 initialized: false,
6605 fresh_bootstrap_nonce: Some(uuid::Uuid::new_v4()),
6606 }
6607 }
6608 (None, PoolMetaIdentityWriteScope::All) if !initialized && !write_state.bootstrap_identity_proven() => {
6609 write_state.block_writes();
6610 return Err(Error::other(
6611 "pool metadata recovery required: cannot create a pending cluster identity without verified fresh-bootstrap proof or legacy-adoption proof",
6612 ));
6613 }
6614 (None, PoolMetaIdentityWriteScope::All) => PersistedPoolMetaIdentity {
6615 version: POOL_META_IDENTITY_VERSION,
6616 cluster_id,
6617 epoch: write_state.cluster_epoch.unwrap_or(POOL_META_INITIAL_EPOCH),
6618 initialized,
6619 fresh_bootstrap_nonce: (!initialized).then(uuid::Uuid::new_v4),
6620 },
6621 };
6622 if identity_write_satisfied(&selection, identity, scope, &targets) {
6623 return Ok(());
6624 }
6625 let data = encode_pool_meta_identity(identity)?;
6626 let mut conflict = false;
6627 for (pool_idx, (pool, token)) in pools.iter().cloned().zip(&selection.cas_tokens).enumerate() {
6628 if !targets.contains(&pool_idx) {
6629 continue;
6630 }
6631 match save_pool_meta_object_cas(
6632 pool,
6633 POOL_META_IDENTITY_NAME,
6634 data.clone(),
6635 token,
6636 fence,
6637 "identity_cas",
6638 transaction_arm,
6639 )
6640 .await
6641 {
6642 Ok(_) => {}
6643 Err(Error::PreconditionFailed) => {
6644 conflict = true;
6645 break;
6646 }
6647 Err(err) => return Err(err),
6648 }
6649 }
6650 if conflict {
6651 if attempt + 1 < POOL_META_CAS_MAX_ATTEMPTS {
6652 continue;
6653 }
6654 return Err(Error::PreconditionFailed);
6655 }
6656 let confirmed = load_pool_meta_identity_selection_observing(pools.clone(), write_state, cluster_id).await?;
6657 if identity_write_satisfied(&confirmed, identity, scope, &targets) {
6658 return Ok(());
6659 }
6660 }
6661 Err(Error::PreconditionFailed)
6662}
6663
6664pub(crate) async fn load_pool_meta_identity_observing<S>(pools: Vec<Arc<S>>, write_state: &mut PoolMetaWriteState) -> Result<()>
6665where
6666 S: EcstoreObjectIO,
6667{
6668 let Some(cluster_id) = write_state.expected_cluster_id else {
6669 return Ok(());
6670 };
6671 load_pool_meta_identity_selection_observing(pools, write_state, cluster_id)
6672 .await
6673 .map(|_| ())
6674}
6675
6676pub(crate) async fn persist_pool_meta_identity_for_startup<S>(
6677 pools: Vec<Arc<S>>,
6678 write_state: &mut PoolMetaWriteState,
6679 initialized: bool,
6680) -> Result<()>
6681where
6682 S: EcstoreObjectIO,
6683{
6684 let mut transaction_arm = write_state.arm_transaction();
6685 persist_pool_meta_identity(
6686 pools,
6687 write_state,
6688 initialized,
6689 PoolMetaIdentityWriteScope::All,
6690 &PoolMetaPersistenceFence::Distributed(None),
6691 &mut transaction_arm,
6692 )
6693 .await?;
6694 transaction_arm.disarm();
6695 Ok(())
6696}
6697
6698pub(crate) async fn persist_pool_meta_identity_for_attested_pools<S>(
6704 pools: Vec<Arc<S>>,
6705 write_state: &mut PoolMetaWriteState,
6706 pool_indices: &[usize],
6707) -> Result<()>
6708where
6709 S: EcstoreObjectIO,
6710{
6711 let mut transaction_arm = write_state.arm_transaction();
6712 persist_pool_meta_identity(
6713 pools,
6714 write_state,
6715 false,
6716 PoolMetaIdentityWriteScope::Pools(pool_indices),
6717 &PoolMetaPersistenceFence::Distributed(None),
6718 &mut transaction_arm,
6719 )
6720 .await?;
6721 transaction_arm.disarm();
6722 Ok(())
6723}
6724
6725#[derive(Debug, Clone, Serialize, Deserialize)]
6726#[serde(deny_unknown_fields)]
6727struct PersistedPoolMeta {
6728 pub version: u16,
6729 pub pools: Vec<PersistedPoolStatus>,
6730}
6731
6732#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
6733#[serde(deny_unknown_fields)]
6734struct PersistedPoolStatus {
6735 #[serde(rename = "id")]
6736 pub id: usize,
6737 #[serde(rename = "cmdline")]
6738 pub cmd_line: String,
6739 #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
6740 pub last_update: OffsetDateTime,
6741 #[serde(rename = "decommissionInfo")]
6742 pub decommission: Option<PersistedPoolDecommissionInfo>,
6743}
6744
6745#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
6746#[serde(deny_unknown_fields)]
6747struct PersistedPoolDecommissionInfo {
6748 #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
6749 pub start_time: Option<OffsetDateTime>,
6750 #[serde(rename = "startSize")]
6751 pub start_size: usize,
6752 #[serde(rename = "totalSize")]
6753 pub total_size: usize,
6754 #[serde(rename = "currentSize")]
6755 pub current_size: usize,
6756 #[serde(rename = "complete")]
6757 pub complete: bool,
6758 #[serde(rename = "failed")]
6759 pub failed: bool,
6760 #[serde(rename = "canceled")]
6761 pub canceled: bool,
6762 #[serde(rename = "queued", default)]
6763 pub queued: bool,
6764 #[serde(rename = "queuedBuckets", default)]
6765 pub queued_buckets: Vec<String>,
6766 #[serde(rename = "decommissionedBuckets", default)]
6767 pub decommissioned_buckets: Vec<String>,
6768 #[serde(rename = "bucket", default)]
6769 pub bucket: String,
6770 #[serde(rename = "prefix", default)]
6771 pub prefix: String,
6772 #[serde(rename = "object", default)]
6773 pub object: String,
6774 #[serde(rename = "objectsDecommissioned")]
6775 pub items_decommissioned: usize,
6776 #[serde(rename = "objectsDecommissionedFailed")]
6777 pub items_decommission_failed: usize,
6778 #[serde(rename = "bytesDecommissioned")]
6779 pub bytes_done: usize,
6780 #[serde(rename = "bytesDecommissionedFailed")]
6781 pub bytes_failed: usize,
6782 #[serde(rename = "terminalReloadAttemptAt", with = "time::serde::rfc3339::option", default)]
6783 pub terminal_reload_attempt_at: Option<OffsetDateTime>,
6784 #[serde(rename = "terminalReloadFailures", default)]
6785 pub terminal_reload_failures: Vec<String>,
6786 #[serde(rename = "unresolvedEntries", default)]
6787 pub unresolved_entries: Vec<DecommissionUnresolvedEntry>,
6788 #[serde(rename = "capacityReservation", default)]
6789 pub capacity_reservation: Option<DecommissionCapacityReservation>,
6790 #[serde(rename = "capacityBlockedReason", default)]
6791 pub capacity_blocked_reason: Option<String>,
6792}
6793
6794#[derive(Debug, Clone, Serialize, Deserialize)]
6795#[serde(deny_unknown_fields)]
6796struct PersistedPoolMetaV1 {
6797 pub version: u16,
6798 pub pools: Vec<PersistedPoolStatusV1>,
6799}
6800
6801#[derive(Debug, Clone, Serialize, Deserialize)]
6802#[serde(deny_unknown_fields)]
6803struct PersistedPoolStatusV1 {
6804 #[serde(rename = "id")]
6805 pub id: usize,
6806 #[serde(rename = "cmdline")]
6807 pub cmd_line: String,
6808 #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
6809 pub last_update: OffsetDateTime,
6810 #[serde(rename = "decommissionInfo")]
6811 pub decommission: Option<PersistedPoolDecommissionInfoV1>,
6812}
6813
6814#[derive(Debug, Clone, Default, Serialize, Deserialize)]
6815#[serde(deny_unknown_fields)]
6816struct PersistedPoolDecommissionInfoV1 {
6817 #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
6818 pub start_time: Option<OffsetDateTime>,
6819 #[serde(rename = "startSize")]
6820 pub start_size: usize,
6821 #[serde(rename = "totalSize")]
6822 pub total_size: usize,
6823 #[serde(rename = "currentSize")]
6824 pub current_size: usize,
6825 #[serde(rename = "complete")]
6826 pub complete: bool,
6827 #[serde(rename = "failed")]
6828 pub failed: bool,
6829 #[serde(rename = "canceled")]
6830 pub canceled: bool,
6831 #[serde(rename = "queued", default)]
6832 pub queued: bool,
6833 #[serde(rename = "queuedBuckets", default)]
6834 pub queued_buckets: Vec<String>,
6835 #[serde(rename = "decommissionedBuckets", default)]
6836 pub decommissioned_buckets: Vec<String>,
6837 #[serde(rename = "bucket", default)]
6838 pub bucket: String,
6839 #[serde(rename = "prefix", default)]
6840 pub prefix: String,
6841 #[serde(rename = "object", default)]
6842 pub object: String,
6843 #[serde(rename = "objectsDecommissioned")]
6844 pub items_decommissioned: usize,
6845 #[serde(rename = "objectsDecommissionedFailed")]
6846 pub items_decommission_failed: usize,
6847 #[serde(rename = "bytesDecommissioned")]
6848 pub bytes_done: usize,
6849 #[serde(rename = "bytesDecommissionedFailed")]
6850 pub bytes_failed: usize,
6851 #[serde(rename = "terminalReloadAttemptAt", with = "time::serde::rfc3339::option", default)]
6852 pub terminal_reload_attempt_at: Option<OffsetDateTime>,
6853 #[serde(rename = "terminalReloadFailures", default)]
6854 pub terminal_reload_failures: Vec<String>,
6855}
6856
6857#[derive(Debug, Clone, Serialize, Deserialize)]
6858#[serde(deny_unknown_fields)]
6859struct LegacyPoolMeta {
6860 pub version: u16,
6861 pub pools: Vec<LegacyPoolStatus>,
6862 pub dont_save: bool,
6863}
6864
6865#[derive(Debug, Clone, Serialize, Deserialize)]
6866#[serde(deny_unknown_fields)]
6867struct LegacyPoolStatus {
6868 #[serde(rename = "id")]
6869 pub id: usize,
6870 #[serde(rename = "cmdline")]
6871 pub cmd_line: String,
6872 #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
6873 pub last_update: OffsetDateTime,
6874 #[serde(rename = "decommissionInfo")]
6875 pub decommission: Option<LegacyPoolDecommissionInfo>,
6876}
6877
6878#[derive(Debug, Clone, Default, Serialize, Deserialize)]
6879#[serde(deny_unknown_fields)]
6880struct LegacyPoolDecommissionInfo {
6881 #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
6882 pub start_time: Option<OffsetDateTime>,
6883 #[serde(rename = "startSize")]
6884 pub start_size: usize,
6885 #[serde(rename = "totalSize")]
6886 pub total_size: usize,
6887 #[serde(rename = "currentSize")]
6888 pub current_size: usize,
6889 #[serde(rename = "complete")]
6890 pub complete: bool,
6891 #[serde(rename = "failed")]
6892 pub failed: bool,
6893 #[serde(rename = "canceled")]
6894 pub canceled: bool,
6895 #[serde(rename = "objectsDecommissioned")]
6896 pub items_decommissioned: usize,
6897 #[serde(rename = "objectsDecommissionedFailed")]
6898 pub items_decommission_failed: usize,
6899 #[serde(rename = "bytesDecommissioned")]
6900 pub bytes_done: usize,
6901 #[serde(rename = "bytesDecommissionedFailed")]
6902 pub bytes_failed: usize,
6903}
6904
6905fn ensure_pool_meta_payload_version(actual: u16, expected: u16, kind: &str) -> Result<()> {
6906 if actual == expected {
6907 return Ok(());
6908 }
6909 Err(Error::other(format!(
6910 "pool metadata {kind} payload has version {actual}, expected {expected}"
6911 )))
6912}
6913
6914impl TryFrom<PersistedPoolMeta> for PoolMeta {
6915 type Error = Error;
6916
6917 fn try_from(value: PersistedPoolMeta) -> Result<Self> {
6918 ensure_pool_meta_payload_version(value.version, POOL_META_VERSION, "current")?;
6919 let meta = Self {
6920 version: POOL_META_VERSION,
6921 pools: value.pools.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?,
6922 dont_save: false,
6923 };
6924 validate_decommission_capacity_model_cohort(&meta)?;
6925 Ok(meta)
6926 }
6927}
6928
6929impl TryFrom<PersistedPoolMetaV1> for PoolMeta {
6930 type Error = Error;
6931
6932 fn try_from(value: PersistedPoolMetaV1) -> Result<Self> {
6933 ensure_pool_meta_payload_version(value.version, POOL_META_V1_VERSION, "v1")?;
6934 Ok(Self {
6935 version: POOL_META_V1_VERSION,
6936 pools: value.pools.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?,
6937 dont_save: false,
6938 })
6939 }
6940}
6941
6942impl TryFrom<LegacyPoolMeta> for PoolMeta {
6943 type Error = Error;
6944
6945 fn try_from(value: LegacyPoolMeta) -> Result<Self> {
6946 let LegacyPoolMeta {
6947 version,
6948 pools,
6949 dont_save: _,
6950 } = value;
6951 ensure_pool_meta_payload_version(version, POOL_META_V1_VERSION, "legacy v1")?;
6952 Ok(Self {
6953 version: POOL_META_V1_VERSION,
6954 pools: pools.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?,
6955 dont_save: false,
6956 })
6957 }
6958}
6959
6960impl TryFrom<PersistedPoolStatus> for PoolStatus {
6961 type Error = Error;
6962
6963 fn try_from(value: PersistedPoolStatus) -> Result<Self> {
6964 let decommission = value.decommission.map(TryInto::try_into).transpose()?;
6965 if decommission
6966 .as_ref()
6967 .and_then(|info: &PoolDecommissionInfo| info.capacity_reservation.as_ref())
6968 .is_some_and(|reservation| reservation.source_pool_index != value.id)
6969 {
6970 return Err(Error::other(
6971 "pool metadata load failed: decommission capacity reservation source pool does not match its owner",
6972 ));
6973 }
6974 Ok(Self {
6975 id: value.id,
6976 cmd_line: value.cmd_line,
6977 last_update: value.last_update,
6978 decommission,
6979 })
6980 }
6981}
6982
6983impl TryFrom<PersistedPoolStatusV1> for PoolStatus {
6984 type Error = Error;
6985
6986 fn try_from(value: PersistedPoolStatusV1) -> Result<Self> {
6987 Ok(Self {
6988 id: value.id,
6989 cmd_line: value.cmd_line,
6990 last_update: value.last_update,
6991 decommission: value.decommission.map(TryInto::try_into).transpose()?,
6992 })
6993 }
6994}
6995
6996impl TryFrom<LegacyPoolStatus> for PoolStatus {
6997 type Error = Error;
6998
6999 fn try_from(value: LegacyPoolStatus) -> Result<Self> {
7000 Ok(Self {
7001 id: value.id,
7002 cmd_line: value.cmd_line,
7003 last_update: value.last_update,
7004 decommission: value.decommission.map(TryInto::try_into).transpose()?,
7005 })
7006 }
7007}
7008
7009impl TryFrom<PersistedPoolDecommissionInfo> for PoolDecommissionInfo {
7010 type Error = Error;
7011
7012 fn try_from(value: PersistedPoolDecommissionInfo) -> Result<Self> {
7013 validate_decommission_terminal_state(value.complete, value.failed, value.canceled)?;
7014 validate_decommission_capacity_reservation(value.capacity_reservation.as_ref())?;
7015 Ok(Self {
7016 start_time: value.start_time,
7017 start_size: value.start_size,
7018 total_size: value.total_size,
7019 current_size: value.current_size,
7020 complete: value.complete,
7021 failed: value.failed,
7022 canceled: value.canceled,
7023 queued: value.queued,
7024 queued_buckets: value.queued_buckets,
7025 decommissioned_buckets: value.decommissioned_buckets,
7026 bucket: value.bucket,
7027 prefix: value.prefix,
7028 object: value.object,
7029 stage: String::new(),
7030 items_decommissioned: value.items_decommissioned,
7031 items_decommission_failed: value.items_decommission_failed,
7032 bytes_done: value.bytes_done,
7033 bytes_failed: value.bytes_failed,
7034 terminal_reload_attempt_at: value.terminal_reload_attempt_at,
7035 terminal_reload_failures: value.terminal_reload_failures,
7036 capacity_reservation: value.capacity_reservation,
7037 capacity_blocked_reason: value.capacity_blocked_reason,
7038 unresolved_entries: value.unresolved_entries,
7039 progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed),
7040 progress_save_retry_after: None,
7041 progress_save_last_at: None,
7042 })
7043 }
7044}
7045
7046impl TryFrom<PersistedPoolDecommissionInfoV1> for PoolDecommissionInfo {
7047 type Error = Error;
7048
7049 fn try_from(value: PersistedPoolDecommissionInfoV1) -> Result<Self> {
7050 validate_decommission_terminal_state(value.complete, value.failed, value.canceled)?;
7051 Ok(Self {
7052 start_time: value.start_time,
7053 start_size: value.start_size,
7054 total_size: value.total_size,
7055 current_size: value.current_size,
7056 complete: value.complete,
7057 failed: value.failed,
7058 canceled: value.canceled,
7059 queued: value.queued,
7060 queued_buckets: value.queued_buckets,
7061 decommissioned_buckets: value.decommissioned_buckets,
7062 bucket: value.bucket,
7063 prefix: value.prefix,
7064 object: value.object,
7065 stage: String::new(),
7066 items_decommissioned: value.items_decommissioned,
7067 items_decommission_failed: value.items_decommission_failed,
7068 bytes_done: value.bytes_done,
7069 bytes_failed: value.bytes_failed,
7070 terminal_reload_attempt_at: value.terminal_reload_attempt_at,
7071 terminal_reload_failures: value.terminal_reload_failures,
7072 capacity_reservation: None,
7073 capacity_blocked_reason: None,
7074 unresolved_entries: Vec::new(),
7075 progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed),
7076 progress_save_retry_after: None,
7077 progress_save_last_at: None,
7078 })
7079 }
7080}
7081
7082impl TryFrom<LegacyPoolDecommissionInfo> for PoolDecommissionInfo {
7083 type Error = Error;
7084
7085 fn try_from(value: LegacyPoolDecommissionInfo) -> Result<Self> {
7086 validate_decommission_terminal_state(value.complete, value.failed, value.canceled)?;
7087 Ok(Self {
7088 start_time: value.start_time,
7089 start_size: value.start_size,
7090 total_size: value.total_size,
7091 current_size: value.current_size,
7092 complete: value.complete,
7093 failed: value.failed,
7094 canceled: value.canceled,
7095 queued: false,
7096 queued_buckets: Vec::new(),
7097 decommissioned_buckets: Vec::new(),
7098 bucket: String::new(),
7099 prefix: String::new(),
7100 object: String::new(),
7101 stage: String::new(),
7102 items_decommissioned: value.items_decommissioned,
7103 items_decommission_failed: value.items_decommission_failed,
7104 bytes_done: value.bytes_done,
7105 bytes_failed: value.bytes_failed,
7106 terminal_reload_attempt_at: None,
7107 terminal_reload_failures: Vec::new(),
7108 capacity_reservation: None,
7109 capacity_blocked_reason: None,
7110 unresolved_entries: Vec::new(),
7111 progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed),
7112 progress_save_retry_after: None,
7113 progress_save_last_at: None,
7114 })
7115 }
7116}
7117
7118impl From<&PoolMeta> for PersistedPoolMeta {
7119 fn from(value: &PoolMeta) -> Self {
7120 Self {
7121 version: POOL_META_VERSION,
7122 pools: value.pools.iter().map(Into::into).collect(),
7123 }
7124 }
7125}
7126
7127impl From<&PoolMeta> for PersistedPoolMetaV1 {
7128 fn from(value: &PoolMeta) -> Self {
7129 Self {
7130 version: POOL_META_V1_VERSION,
7131 pools: value.pools.iter().map(Into::into).collect(),
7132 }
7133 }
7134}
7135
7136impl From<&PoolStatus> for PersistedPoolStatus {
7137 fn from(value: &PoolStatus) -> Self {
7138 Self {
7139 id: value.id,
7140 cmd_line: value.cmd_line.clone(),
7141 last_update: value.last_update,
7142 decommission: value.decommission.as_ref().map(Into::into),
7143 }
7144 }
7145}
7146
7147impl From<&PoolStatus> for PersistedPoolStatusV1 {
7148 fn from(value: &PoolStatus) -> Self {
7149 Self {
7150 id: value.id,
7151 cmd_line: value.cmd_line.clone(),
7152 last_update: value.last_update,
7153 decommission: value.decommission.as_ref().map(Into::into),
7154 }
7155 }
7156}
7157
7158impl From<&PoolDecommissionInfo> for PersistedPoolDecommissionInfo {
7159 fn from(value: &PoolDecommissionInfo) -> Self {
7160 Self {
7161 start_time: value.start_time,
7162 start_size: value.start_size,
7163 total_size: value.total_size,
7164 current_size: value.current_size,
7165 complete: value.complete,
7166 failed: value.failed,
7167 canceled: value.canceled,
7168 queued: value.queued,
7169 queued_buckets: value.queued_buckets.clone(),
7170 decommissioned_buckets: value.decommissioned_buckets.clone(),
7171 bucket: value.bucket.clone(),
7172 prefix: value.prefix.clone(),
7173 object: value.object.clone(),
7174 items_decommissioned: value.items_decommissioned,
7175 items_decommission_failed: value.items_decommission_failed,
7176 bytes_done: value.bytes_done,
7177 bytes_failed: value.bytes_failed,
7178 terminal_reload_attempt_at: value.terminal_reload_attempt_at,
7179 terminal_reload_failures: value.terminal_reload_failures.clone(),
7180 capacity_reservation: value.capacity_reservation.clone(),
7181 capacity_blocked_reason: value.capacity_blocked_reason.clone(),
7182 unresolved_entries: value.unresolved_entries.clone(),
7183 }
7184 }
7185}
7186
7187impl From<&PoolDecommissionInfo> for PersistedPoolDecommissionInfoV1 {
7188 fn from(value: &PoolDecommissionInfo) -> Self {
7189 Self {
7190 start_time: value.start_time,
7191 start_size: value.start_size,
7192 total_size: value.total_size,
7193 current_size: value.current_size,
7194 complete: value.complete,
7195 failed: value.failed,
7196 canceled: value.canceled,
7197 queued: value.queued,
7198 queued_buckets: value.queued_buckets.clone(),
7199 decommissioned_buckets: value.decommissioned_buckets.clone(),
7200 bucket: value.bucket.clone(),
7201 prefix: value.prefix.clone(),
7202 object: value.object.clone(),
7203 items_decommissioned: value.items_decommissioned,
7204 items_decommission_failed: value.items_decommission_failed,
7205 bytes_done: value.bytes_done,
7206 bytes_failed: value.bytes_failed,
7207 terminal_reload_attempt_at: value.terminal_reload_attempt_at,
7208 terminal_reload_failures: value.terminal_reload_failures.clone(),
7209 }
7210 }
7211}
7212
7213#[derive(Debug)]
7214struct PoolMetaSaveOutcome {
7215 transaction_arm: PoolMetaTransactionArm,
7216 committed: PoolMeta,
7217}
7218
7219async fn load_pool_meta_for_transaction_recovery<S>(
7223 pools: Vec<Arc<S>>,
7224 write_state: &mut PoolMetaWriteState,
7225) -> Result<PoolMetaSelection>
7226where
7227 S: EcstoreObjectIO,
7228{
7229 let result = async {
7230 if let Some(cluster_id) = write_state.expected_cluster_id {
7231 let reads = join_all(pools.iter().cloned().map(read_pool_meta_identity_replica)).await;
7232 for read in &reads {
7233 match &read.replica {
7234 PoolMetaIdentityReplica::Unreadable(source) => {
7235 return Err(Error::other(pool_metadata_error(
7236 crate::error::PoolMetadataFailure::ReadUnavailable,
7237 "recovery_identity_read",
7238 Some(Arc::clone(source)),
7239 )));
7240 }
7241 PoolMetaIdentityReplica::Corrupt(_) | PoolMetaIdentityReplica::Incompatible(_) => {
7242 return Err(Error::other("pool metadata recovery requires operator repair of cluster identity"));
7243 }
7244 _ => {}
7245 }
7246 }
7247 let identity = select_pool_meta_identity(reads, cluster_id)?;
7248 if !identity.identity.is_some_and(|identity| identity.initialized) {
7249 return Err(Error::other("pool metadata recovery requires an initialized durable cluster identity"));
7250 }
7251 write_state.observe_identity(&identity)?;
7252 }
7253 let reads = read_pool_meta_replicas(pools, true).await;
7254 if let Some(err) = pool_meta_replica_read_failure(&reads, "pool metadata transaction recovery") {
7255 return Err(err);
7256 }
7257 if reads
7258 .iter()
7259 .any(|read| matches!(&read.replica, PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)))
7260 {
7261 return Err(Error::other(
7262 "pool metadata recovery requires operator repair of corrupt or incompatible replicas",
7263 ));
7264 }
7265 let selection = select_pool_meta_replica_reads(reads)?;
7266 write_state.observe_selection(&selection)?;
7267 if selection.absent || selection.meta.pools.is_empty() {
7268 return Err(Error::other("pool metadata recovery cannot initialize missing metadata"));
7269 }
7270 selection
7271 .replica_state
7272 .ensure_write_safe("pool metadata transaction recovery")?;
7273 Ok(selection)
7274 }
7275 .await;
7276 result.map_err(|err: Error| {
7277 if err.pool_metadata_failure().is_some() {
7278 err
7279 } else {
7280 Error::other(pool_metadata_error(
7281 crate::error::PoolMetadataFailure::RecoveryRequired,
7282 "recovery_validation",
7283 Some(Arc::new(err)),
7284 ))
7285 }
7286 })
7287}
7288
7289async fn repair_pool_meta_transaction<S>(
7290 pools: Vec<Arc<S>>,
7291 write_state: &mut PoolMetaWriteState,
7292 selection: PoolMetaSelection,
7293 fence: &PoolMetaPersistenceFence<'_>,
7294) -> Result<PoolMetaSaveOutcome>
7295where
7296 S: EcstoreObjectIO,
7297{
7298 let mut transaction_arm = write_state.arm_transaction();
7299 let mut expected_meta = selection.meta.clone();
7300 let mut expected_revision = selection.revision;
7301 if selection.generation_protocol_observed && !expected_revision.is_generation_protocol() {
7304 let (cluster_id, epoch) = selection
7305 .generation_identity
7306 .ok_or_else(|| Error::other("pool metadata recovery has no V3 cluster identity"))?;
7307 expected_meta.version = POOL_META_GENERATION_VERSION;
7308 expected_revision = PoolMetaRevision {
7309 version: POOL_META_GENERATION_VERSION,
7310 cluster_id: Some(cluster_id),
7311 epoch,
7312 generation: 1,
7313 transaction_id: Some(uuid::Uuid::new_v4()),
7314 };
7315 }
7316 let expected_canonical = if expected_revision.is_generation_protocol() {
7317 encode_pool_meta_v3_envelope(&expected_meta, expected_revision, true, None)?
7318 } else {
7319 expected_meta.encode_config_data_for_v2_gate(true)?
7320 };
7321 if selection.replica_state.needs_repair {
7322 let data = if expected_revision.is_generation_protocol() {
7323 expected_canonical.clone()
7324 } else {
7325 expected_meta.encode_config_data_for_v2_gate(expected_revision.version == POOL_META_VERSION)?
7326 };
7327 for (pool, token) in pools.iter().cloned().zip(&selection.cas_tokens) {
7328 save_pool_meta_object_cas(pool, POOL_META_NAME, data.clone(), token, fence, "recovery_cas", &mut transaction_arm)
7329 .await?;
7330 }
7331 }
7332 persist_pool_meta_identity(
7333 pools.clone(),
7334 write_state,
7335 true,
7336 PoolMetaIdentityWriteScope::All,
7337 fence,
7338 &mut transaction_arm,
7339 )
7340 .await?;
7341 let confirmed = load_pool_meta_for_transaction_recovery(pools, write_state).await?;
7342 if confirmed.revision != expected_revision
7343 || confirmed.canonical.as_ref() != Some(&expected_canonical)
7344 || confirmed.replica_state.needs_repair
7345 || write_state.identity_requires_repair()
7346 {
7347 return Err(Error::PreconditionFailed);
7348 }
7349 fence.ensure_held()?;
7350 Ok(PoolMetaSaveOutcome {
7351 transaction_arm,
7352 committed: confirmed.meta,
7353 })
7354}
7355
7356impl PoolMetaSaveOutcome {
7357 fn disarm(mut self) {
7358 self.transaction_arm.disarm();
7359 }
7360
7361 fn into_committed(mut self) -> PoolMeta {
7362 self.transaction_arm.disarm();
7363 self.committed
7364 }
7365}
7366
7367impl PoolMeta {
7368 fn current_decommission_movement_update(&self) -> Option<OffsetDateTime> {
7369 self.pools
7370 .iter()
7371 .filter(|pool| pool.decommission.is_some())
7372 .map(|pool| pool.last_update)
7373 .max()
7374 }
7375
7376 fn current_rebalance_movement_update(rebalance_meta: Option<&RebalanceMeta>) -> Option<OffsetDateTime> {
7377 rebalance_meta
7378 .into_iter()
7379 .flat_map(|meta| {
7380 meta.stopped_at.into_iter().chain(
7381 meta.pool_stats
7382 .iter()
7383 .flat_map(|pool| [pool.info.start_time, pool.info.end_time])
7384 .flatten(),
7385 )
7386 })
7387 .max()
7388 }
7389
7390 pub(crate) fn next_scanner_data_movement_update(
7391 &self,
7392 now: OffsetDateTime,
7393 rebalance_meta: Option<&RebalanceMeta>,
7394 ) -> OffsetDateTime {
7395 let max_seen = self
7396 .current_decommission_movement_update()
7397 .max(Self::current_rebalance_movement_update(rebalance_meta));
7398 match max_seen {
7399 Some(max_seen) => max_seen
7400 .checked_add(Duration::nanoseconds(1))
7401 .map(|next| now.max(next))
7402 .unwrap_or(max_seen),
7403 None => now,
7404 }
7405 }
7406
7407 fn decode_pool_meta_payload(version: u16, payload: &[u8]) -> Result<Self> {
7408 match version {
7409 POOL_META_VERSION => rmp_serde::from_slice::<PersistedPoolMeta>(payload)
7410 .map_err(|err| Error::other(format!("PoolMeta v{POOL_META_VERSION} decode failed: {err}")))?
7411 .try_into(),
7412 POOL_META_V1_VERSION => match rmp_serde::from_slice::<PersistedPoolMetaV1>(payload) {
7413 Ok(meta) => meta.try_into(),
7414 Err(persisted_err) => {
7415 let legacy: LegacyPoolMeta = rmp_serde::from_slice(payload).map_err(|legacy_err| {
7416 Error::other(format!(
7417 "PoolMeta v1 decode failed for both persisted and legacy formats: persisted={persisted_err}; legacy={legacy_err}"
7418 ))
7419 })?;
7420 legacy.try_into()
7421 }
7422 },
7423 _ => Err(Error::other(format!("pool metadata load failed: unknown version {version}"))),
7424 }
7425 }
7426
7427 pub fn new(pools: &[Arc<Sets>], prev_meta: &PoolMeta) -> Self {
7428 let mut new_meta = Self {
7429 version: if prev_meta.version == POOL_META_GENERATION_VERSION || pool_meta_v3_writer_enabled() {
7430 POOL_META_GENERATION_VERSION
7431 } else if prev_meta.version == POOL_META_VERSION || pool_meta_v2_writer_enabled() {
7432 POOL_META_VERSION
7433 } else {
7434 POOL_META_V1_VERSION
7435 },
7436 pools: Vec::new(),
7437 ..Default::default()
7438 };
7439
7440 for (idx, pool) in pools.iter().enumerate() {
7441 let mut skip = false;
7442
7443 for current_pool in prev_meta.pools.iter() {
7444 if current_pool.cmd_line == pool.endpoints.cmd_line {
7445 new_meta.pools.push(current_pool.clone());
7446 skip = true;
7447 break;
7448 }
7449 }
7450
7451 if skip {
7452 continue;
7453 }
7454
7455 new_meta.pools.push(PoolStatus {
7456 cmd_line: pool.endpoints.cmd_line.clone(),
7457 id: idx,
7458 last_update: OffsetDateTime::now_utc(),
7459 decommission: None,
7460 });
7461 }
7462
7463 new_meta
7464 }
7465
7466 pub fn is_suspended(&self, idx: usize) -> bool {
7467 self.pools
7468 .get(idx)
7469 .and_then(|pool| pool.decommission.as_ref())
7470 .is_some_and(is_decommission_suspended)
7471 }
7472
7473 pub(crate) fn has_active_decommission_capacity_reservation(&self, idx: usize) -> bool {
7474 self.pools
7475 .get(idx)
7476 .and_then(|pool| pool.decommission.as_ref())
7477 .is_some_and(|info| {
7478 info.has_decommission_state()
7479 && is_decommission_active(info.complete, info.failed, info.canceled)
7480 && info
7481 .capacity_reservation
7482 .as_ref()
7483 .is_some_and(DecommissionCapacityReservation::active)
7484 })
7485 }
7486
7487 pub(crate) fn scanner_pause_backlog_pool_writable(&self, idx: usize) -> bool {
7488 self.pools.get(idx).is_some_and(|pool| {
7489 !pool
7490 .decommission
7491 .as_ref()
7492 .is_some_and(|info| info.has_decommission_state() && !info.failed && !info.canceled)
7493 })
7494 }
7495
7496 fn mark_decommission_progress_saved(&mut self) {
7497 for pool in &mut self.pools {
7498 if let Some(info) = pool.decommission.as_mut() {
7499 info.mark_progress_saved();
7500 info.progress_save_last_at = Some(pool.last_update);
7501 }
7502 }
7503 }
7504
7505 fn decommission_progress_checkpoint(
7506 &self,
7507 idx: usize,
7508 duration: Duration,
7509 now: OffsetDateTime,
7510 rebalance_meta: Option<&RebalanceMeta>,
7511 ) -> Result<Option<DecommissionProgressCheckpoint>> {
7512 let pool_count = self.pools.len();
7513 ensure_valid_decommission_pool_index(pool_count, idx)?;
7514
7515 let Some(pool) = self.pools.get(idx) else {
7516 return Err(invalid_decommission_pool_index_error(pool_count, idx));
7517 };
7518 let Some(info) = pool.decommission.as_ref() else {
7519 return Err(decommission_metadata_not_initialized_error("update decommission metadata timestamp"));
7520 };
7521
7522 if info.progress_save_retry_after.is_some_and(|retry_after| now < retry_after) {
7523 return Ok(None);
7524 }
7525
7526 let last_progress_save = info.progress_save_last_at.unwrap_or(pool.last_update);
7527 let time_threshold_reached = now.unix_timestamp() - last_progress_save.unix_timestamp() >= duration.whole_seconds();
7528 let item_threshold_reached = info.items_since_last_progress_save() >= DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD;
7529 if !time_threshold_reached && !item_threshold_reached {
7530 return Ok(None);
7531 }
7532
7533 Ok(Some(DecommissionProgressCheckpoint {
7534 start_time: info.start_time,
7535 queued: info.queued,
7536 counted_items: info.counted_items(),
7537 checkpoint_at: self.next_scanner_data_movement_update(now, rebalance_meta),
7538 capacity_operation_id: info
7539 .capacity_reservation
7540 .as_ref()
7541 .filter(|reservation| reservation.lease_active_at(now))
7542 .map(|reservation| reservation.operation_id),
7543 capacity_owner_nonce: info
7544 .capacity_reservation
7545 .as_ref()
7546 .filter(|reservation| reservation.lease_active_at(now))
7547 .map(|reservation| reservation.owner_nonce),
7548 capacity_lease_expires_at: info
7549 .capacity_reservation
7550 .as_ref()
7551 .filter(|reservation| reservation.lease_active_at(now))
7552 .map(|_| now + DECOMMISSION_CAPACITY_RESERVATION_TTL),
7553 }))
7554 }
7555
7556 fn commit_decommission_progress_checkpoint(&mut self, idx: usize, checkpoint: DecommissionProgressCheckpoint) -> bool {
7557 let Some(pool) = self.pools.get_mut(idx) else {
7558 return false;
7559 };
7560 let Some(info) = pool.decommission.as_mut() else {
7561 return false;
7562 };
7563
7564 if info.start_time != checkpoint.start_time
7565 || info.queued != checkpoint.queued
7566 || !is_decommission_active(info.complete, info.failed, info.canceled)
7567 {
7568 return false;
7569 }
7570
7571 info.progress_save_item_baseline = info.progress_save_item_baseline.max(checkpoint.counted_items);
7572 info.progress_save_retry_after = None;
7573 info.progress_save_last_at = Some(
7574 info.progress_save_last_at
7575 .map_or(checkpoint.checkpoint_at, |at| at.max(checkpoint.checkpoint_at)),
7576 );
7577 if let (Some(operation_id), Some(owner_nonce), Some(expires_at), Some(reservation)) = (
7578 checkpoint.capacity_operation_id,
7579 checkpoint.capacity_owner_nonce,
7580 checkpoint.capacity_lease_expires_at,
7581 info.capacity_reservation.as_mut(),
7582 ) && reservation.operation_id == operation_id
7583 && reservation.owner_nonce == owner_nonce
7584 && reservation.active()
7585 {
7586 reservation.renewed_at = checkpoint.checkpoint_at;
7587 reservation.expires_at = expires_at;
7588 }
7589 pool.last_update = pool.last_update.max(checkpoint.checkpoint_at);
7590 true
7591 }
7592
7593 fn defer_decommission_progress_checkpoint(
7594 &mut self,
7595 idx: usize,
7596 checkpoint: DecommissionProgressCheckpoint,
7597 retry_after: OffsetDateTime,
7598 ) {
7599 let Some(pool) = self.pools.get_mut(idx) else {
7600 return;
7601 };
7602 let Some(info) = pool.decommission.as_mut() else {
7603 return;
7604 };
7605
7606 if info.start_time == checkpoint.start_time
7607 && info.queued == checkpoint.queued
7608 && is_decommission_active(info.complete, info.failed, info.canceled)
7609 {
7610 info.progress_save_retry_after = Some(retry_after);
7611 }
7612 }
7613
7614 fn load_from_config_data(&mut self, data: Vec<u8>) -> Result<()> {
7615 if data.is_empty() {
7616 return Ok(());
7617 } else if data.len() <= 4 {
7618 return Err(Error::other("pool metadata load failed: metadata payload is too short"));
7619 }
7620
7621 let format = LittleEndian::read_u16(&data[0..2]);
7622 if format != POOL_META_FORMAT {
7623 return Err(Error::other(format!("pool metadata load failed: unknown format {format}")));
7624 }
7625 let version = LittleEndian::read_u16(&data[2..4]);
7626 if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) {
7627 return Err(Error::other(format!("pool metadata load failed: unknown version {version}")));
7628 }
7629
7630 if version == POOL_META_GENERATION_VERSION {
7631 let PoolMetaReplica::Valid {
7632 meta, committed: true, ..
7633 } = decode_pool_meta_replica(data)
7634 else {
7635 return Err(Error::other(
7636 "pool metadata load failed: V3 payload is corrupt, incompatible, or not committed",
7637 ));
7638 };
7639 *self = meta;
7640 return Ok(());
7641 }
7642
7643 *self = Self::decode_pool_meta_payload(version, &data[4..])?;
7644
7645 if !matches!(self.version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) {
7646 return Err(Error::other(format!(
7647 "pool metadata load failed: unexpected decoded version {}",
7648 self.version
7649 )));
7650 }
7651 Ok(())
7652 }
7653
7654 pub async fn load(&mut self, pool: Arc<Sets>, pools: Vec<Arc<Sets>>) -> Result<()> {
7655 let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
7656 let _pool_meta_guard = pool_meta_lock.get_read_lock(get_lock_acquire_timeout()).await?;
7657 let replica_state = self.load_no_lock_from_replicas(pools).await?;
7658 replica_state.ensure_write_safe("pool metadata load failed")?;
7659 Ok(())
7660 }
7661
7662 pub(crate) async fn load_no_lock_from_replicas<S>(&mut self, pools: Vec<Arc<S>>) -> Result<PoolMetaReplicaState>
7665 where
7666 S: EcstoreObjectIO,
7667 {
7668 let selection = load_pool_meta_replicas(pools, true).await?;
7669 *self = selection.meta;
7670 Ok(selection.replica_state)
7671 }
7672
7673 pub(crate) async fn load_no_lock_from_replicas_observing<S>(
7674 &mut self,
7675 pools: Vec<Arc<S>>,
7676 write_state: &mut PoolMetaWriteState,
7677 ) -> Result<PoolMetaReplicaState>
7678 where
7679 S: EcstoreObjectIO,
7680 {
7681 let selection = load_pool_meta_replicas_observing(pools, true, write_state).await?;
7682 *self = selection.meta;
7683 Ok(selection.replica_state)
7684 }
7685
7686 pub(crate) async fn load_for_startup_observing<S>(
7687 &mut self,
7688 pools: Vec<Arc<S>>,
7689 write_state: &mut PoolMetaWriteState,
7690 ) -> Result<PoolMetaReplicaState>
7691 where
7692 S: EcstoreObjectIO,
7693 {
7694 if let Some(cluster_id) = write_state.expected_cluster_id {
7697 let reads = join_all(pools.iter().cloned().map(read_pool_meta_identity_replica)).await;
7698 if let Some(source) = reads.iter().find_map(|read| match &read.replica {
7699 PoolMetaIdentityReplica::Unreadable(source) => Some(Arc::clone(source)),
7700 _ => None,
7701 }) {
7702 write_state.block_with_context(pool_metadata_error(
7703 crate::error::PoolMetadataFailure::RecoveryRequired,
7704 "startup_identity_read",
7705 Some(source),
7706 ));
7707 }
7708 let identity = select_pool_meta_identity(reads, cluster_id)
7709 .map_err(|err| block_pool_meta_validation(write_state, err, "startup_identity"))?;
7710 write_state.observe_identity(&identity)?;
7711 }
7712 let reads = read_pool_meta_replicas(pools, true).await;
7713 if let Some(err) = pool_meta_replica_read_failure(&reads, "pool metadata load") {
7714 write_state.block_with_context(pool_metadata_error(
7715 crate::error::PoolMetadataFailure::RecoveryRequired,
7716 "startup_read",
7717 Some(Arc::new(err)),
7718 ));
7719 }
7720 let selection = select_pool_meta_replica_reads(reads)
7721 .map_err(|err| block_pool_meta_validation(write_state, err, "startup_selection"))?;
7722 write_state
7723 .observe_selection(&selection)
7724 .map_err(|err| block_pool_meta_validation(write_state, err, "startup_selection"))?;
7725 *self = selection.meta;
7726 Ok(selection.replica_state)
7727 }
7728
7729 #[cfg(test)]
7730 fn encode_config_data(&self) -> Result<Vec<u8>> {
7731 self.encode_config_data_for_v2_gate(pool_meta_v2_writer_enabled())
7732 }
7733
7734 fn encode_config_data_for_v2_gate(&self, v2_enabled: bool) -> Result<Vec<u8>> {
7735 if self.dont_save {
7736 return Ok(Vec::new());
7737 }
7738 if self.version == POOL_META_GENERATION_VERSION {
7739 return Err(Error::other(
7740 "pool metadata V3 save requires cluster identity and generation transaction context",
7741 ));
7742 }
7743 if !matches!(self.version, 0 | POOL_META_V1_VERSION | POOL_META_VERSION) {
7744 return Err(Error::other(format!(
7745 "pool metadata save failed: unexpected runtime version {}",
7746 self.version
7747 )));
7748 }
7749 let version = if self.version == POOL_META_VERSION || v2_enabled {
7750 POOL_META_VERSION
7751 } else {
7752 POOL_META_V1_VERSION
7753 };
7754 if version == POOL_META_V1_VERSION {
7755 let mut decommission_infos = self.pools.iter().filter_map(|pool| pool.decommission.as_ref());
7756 if decommission_infos.clone().any(|info| !info.unresolved_entries.is_empty()) {
7757 return Err(Error::other(format!(
7758 "pool metadata V2 is required to persist unresolved decommission entries; enable both {} and {} only after every reader and writer supports V2",
7759 rustfs_config::ENV_POOL_META_V2_WRITE,
7760 rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
7761 )));
7762 }
7763 if decommission_infos.any(|info| info.capacity_reservation.is_some()) {
7764 return Err(Error::DecommissionCapacity(format!(
7765 "pool metadata V2 is required to persist decommission capacity reservations; enable both {} and {} only after every reader and writer supports V2",
7766 rustfs_config::ENV_POOL_META_V2_WRITE,
7767 rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
7768 )));
7769 }
7770 }
7771 let mut data = Vec::new();
7772 data.write_u16::<LittleEndian>(POOL_META_FORMAT)?;
7773 data.write_u16::<LittleEndian>(version)?;
7774 let mut buf = Vec::new();
7775 match version {
7776 POOL_META_V1_VERSION => PersistedPoolMetaV1::from(self).serialize(&mut Serializer::new(&mut buf))?,
7777 POOL_META_VERSION => PersistedPoolMeta::from(self).serialize(&mut Serializer::new(&mut buf))?,
7778 _ => unreachable!("pool metadata writer selected an unsupported version"),
7779 }
7780 data.write_all(&buf)?;
7781 Ok(data)
7782 }
7783
7784 #[cfg(test)]
7785 pub(crate) fn encode_config_data_for_test(&self) -> Result<Vec<u8>> {
7786 self.encode_config_data_for_v2_gate(true)
7787 }
7788
7789 pub async fn save(&self, pools: Vec<Arc<Sets>>) -> Result<()> {
7790 let pool = pools
7791 .first()
7792 .cloned()
7793 .ok_or_else(|| Error::other("pool metadata save failed: no storage pools available"))?;
7794 let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
7795 let pool_meta_guard = pool_meta_lock.get_write_lock(get_lock_acquire_timeout()).await?;
7796 let mut write_state = PoolMetaWriteState::default();
7797 let indices = (0..self.pools.len()).collect::<Vec<_>>();
7798 let outcome = self
7799 .save_no_lock_armed_scoped(pools, &mut write_state, pool_meta_guard.lock_lost_signal(), Some(&indices))
7800 .await?;
7801 outcome.disarm();
7802 Ok(())
7803 }
7804
7805 #[cfg(any(test, feature = "test-util"))]
7807 pub(crate) async fn save_for_startup<S>(&self, pools: Vec<Arc<S>>) -> Result<()>
7808 where
7809 S: EcstoreObjectIO,
7810 {
7811 let mut write_state = PoolMetaWriteState::for_test_bootstrap();
7812 self.save_for_startup_observing(pools, &mut write_state).await.map(|_| ())
7813 }
7814
7815 pub(crate) async fn save_for_startup_observing<S>(
7816 &self,
7817 pools: Vec<Arc<S>>,
7818 write_state: &mut PoolMetaWriteState,
7819 ) -> Result<PoolMeta>
7820 where
7821 S: EcstoreObjectIO,
7822 {
7823 let outcome = self.save_no_lock_armed_scoped(pools, write_state, None, None).await?;
7824 Ok(outcome.into_committed())
7825 }
7826
7827 async fn save_no_lock_with_fence<S>(
7828 &self,
7829 pools: Vec<Arc<S>>,
7830 lock_lost: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
7831 indices: &[usize],
7832 ) -> Result<()>
7833 where
7834 S: EcstoreObjectIO,
7835 {
7836 let mut write_state = PoolMetaWriteState::default();
7837 let outcome = self
7838 .save_no_lock_armed_scoped(pools, &mut write_state, lock_lost, Some(indices))
7839 .await?;
7840 outcome.disarm();
7841 Ok(())
7842 }
7843
7844 async fn save_no_lock_with_activation_fence<S>(
7845 &self,
7846 pools: Vec<Arc<S>>,
7847 write_state: &mut PoolMetaWriteState,
7848 activation_fence: &PoolRebalanceActivationFence,
7849 indices: &[usize],
7850 ) -> Result<PoolMetaSaveOutcome>
7851 where
7852 S: EcstoreObjectIO,
7853 {
7854 write_state.ensure_write_safe("pool metadata activation save failed")?;
7855 let mut transaction_arm = write_state.arm_transaction();
7856 let fence = PoolMetaPersistenceFence::Activation(activation_fence);
7857 let committed = self
7858 .save_no_lock_transaction(pools, write_state, &fence, Some(indices), &mut transaction_arm)
7859 .await
7860 .map_err(|err| transaction_arm.failure(err))?;
7861 if transaction_arm.phase.is_some() {
7862 transaction_arm.phase = Some("publication");
7863 }
7864 Ok(PoolMetaSaveOutcome {
7865 transaction_arm,
7866 committed,
7867 })
7868 }
7869
7870 async fn save_no_lock_armed<S>(
7871 &self,
7872 pools: Vec<Arc<S>>,
7873 write_state: &mut PoolMetaWriteState,
7874 lock_lost: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
7875 indices: &[usize],
7876 ) -> Result<PoolMetaSaveOutcome>
7877 where
7878 S: EcstoreObjectIO,
7879 {
7880 self.save_no_lock_armed_scoped(pools, write_state, lock_lost, Some(indices))
7881 .await
7882 }
7883
7884 async fn save_no_lock_armed_scoped<S>(
7885 &self,
7886 pools: Vec<Arc<S>>,
7887 write_state: &mut PoolMetaWriteState,
7888 lock_lost: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
7889 indices: Option<&[usize]>,
7890 ) -> Result<PoolMetaSaveOutcome>
7891 where
7892 S: EcstoreObjectIO,
7893 {
7894 write_state.ensure_write_safe("pool metadata save failed")?;
7895 let mut transaction_arm = write_state.arm_transaction();
7899 let fence = PoolMetaPersistenceFence::Distributed(lock_lost);
7900 let committed = self
7901 .save_no_lock_transaction(pools, write_state, &fence, indices, &mut transaction_arm)
7902 .await
7903 .map_err(|err| transaction_arm.failure(err))?;
7904 if transaction_arm.phase.is_some() {
7905 transaction_arm.phase = Some("publication");
7906 }
7907 Ok(PoolMetaSaveOutcome {
7908 transaction_arm,
7909 committed,
7910 })
7911 }
7912
7913 async fn save_no_lock_transaction<S>(
7914 &self,
7915 pools: Vec<Arc<S>>,
7916 write_state: &mut PoolMetaWriteState,
7917 fence: &PoolMetaPersistenceFence<'_>,
7918 indices: Option<&[usize]>,
7919 transaction_arm: &mut PoolMetaTransactionArm,
7920 ) -> Result<PoolMeta>
7921 where
7922 S: EcstoreObjectIO,
7923 {
7924 if pools.is_empty() {
7925 return Err(Error::other("pool metadata save failed: no storage pools available"));
7926 }
7927 if self.dont_save {
7928 return Ok(self.clone());
7929 }
7930 for attempt in 0..POOL_META_CAS_MAX_ATTEMPTS {
7931 match self
7932 .save_no_lock_transaction_once(pools.clone(), write_state, fence, indices, transaction_arm)
7933 .await
7934 {
7935 Ok(committed) => return Ok(committed),
7936 Err(Error::PreconditionFailed) if attempt + 1 < POOL_META_CAS_MAX_ATTEMPTS => continue,
7937 Err(err) => return Err(err),
7938 }
7939 }
7940 Err(Error::PreconditionFailed)
7941 }
7942
7943 async fn save_no_lock_transaction_once<S>(
7944 &self,
7945 pools: Vec<Arc<S>>,
7946 write_state: &mut PoolMetaWriteState,
7947 fence: &PoolMetaPersistenceFence<'_>,
7948 indices: Option<&[usize]>,
7949 transaction_arm: &mut PoolMetaTransactionArm,
7950 ) -> Result<PoolMeta>
7951 where
7952 S: EcstoreObjectIO,
7953 {
7954 fence.ensure_held()?;
7955 let selection = load_pool_meta_replicas_observing(pools.clone(), true, write_state).await?;
7956 write_state.observe_replicas(selection.replica_state);
7957 write_state.ensure_write_safe("pool metadata save failed")?;
7958 let mut bootstrap_generation_required = false;
7959 if let Some(cluster_id) = write_state.expected_cluster_id {
7960 let identity = load_pool_meta_identity_selection_observing(pools.clone(), write_state, cluster_id).await?;
7961 bootstrap_generation_required = selection.absent || write_state.identity_initialized == Some(false);
7962 if !identity.repair_write_safe {
7963 write_state.block_writes();
7964 return Err(Error::other(
7965 "pool metadata recovery required: cluster identity has an unreadable replica",
7966 ));
7967 }
7968 if let Some(identity) = identity.identity
7969 && selection.revision.is_generation_protocol()
7970 && identity.epoch != selection.revision.epoch
7971 {
7972 write_state.block_writes();
7973 return Err(Error::other(format!(
7974 "pool metadata recovery required: committed epoch {} does not match cluster identity epoch {}",
7975 selection.revision.epoch, identity.epoch
7976 )));
7977 }
7978 if !selection.absent && write_state.identity_requires_repair() {
7979 let initialized = write_state.identity_initialized != Some(false) || selection.revision.is_generation_protocol();
7980 persist_pool_meta_identity(
7981 pools.clone(),
7982 write_state,
7983 initialized,
7984 PoolMetaIdentityWriteScope::All,
7985 fence,
7986 transaction_arm,
7987 )
7988 .await?;
7989 }
7990 }
7991 write_state.ensure_missing_metadata_can_initialize()?;
7994
7995 let mut committed = if let Some(indices) = indices {
7996 if selection.meta.pools.is_empty() {
7997 self.clone()
7998 } else {
7999 let mut requested = self.clone();
8000 requested.version = selection.meta.version;
8001 let mut latest = selection.meta.clone();
8002 merge_pool_meta_updates_for_save(&mut latest, &requested, indices, "pool metadata save failed")?;
8003 latest
8004 }
8005 } else {
8006 self.clone()
8007 };
8008 let target_version = if selection.generation_protocol_observed
8009 || selection.revision.is_generation_protocol()
8010 || pool_meta_v3_writer_enabled()
8011 || bootstrap_generation_required
8012 {
8013 POOL_META_GENERATION_VERSION
8014 } else if selection.meta.version == POOL_META_VERSION
8015 || self.version == POOL_META_VERSION
8016 || pool_meta_v2_writer_enabled()
8017 {
8018 POOL_META_VERSION
8019 } else {
8020 POOL_META_V1_VERSION
8021 };
8022 committed.version = target_version;
8023
8024 if target_version != POOL_META_GENERATION_VERSION {
8025 let data = committed.encode_config_data_for_v2_gate(target_version == POOL_META_VERSION)?;
8026 let mut canonical_saved = false;
8027 for (pool_index, (pool, token)) in pools.iter().cloned().zip(&selection.cas_tokens).enumerate() {
8028 let result = save_pool_meta_object_cas(
8029 pool.clone(),
8030 POOL_META_NAME,
8031 data.clone(),
8032 token,
8033 fence,
8034 "legacy_cas",
8035 transaction_arm,
8036 )
8037 .await;
8038 if result.is_ok() && pool_index == 0 {
8039 canonical_saved = true;
8040 #[cfg(test)]
8041 if let PoolMetaPersistenceFence::Activation(activation_fence) = fence {
8042 pause_pool_activation_after_durable_save(&pool, activation_fence).await;
8043 }
8044 }
8045 if let Err(err) = result {
8046 if canonical_saved && fence.is_activation() {
8047 warn!(
8048 event = EVENT_DECOMMISSION_STATE,
8049 component = LOG_COMPONENT_ECSTORE,
8050 subsystem = LOG_SUBSYSTEM_POOLS,
8051 pool_index,
8052 state = "activation_replica_repair_pending",
8053 error = %err,
8054 "Decommission activation replica repair pending"
8055 );
8056 continue;
8057 }
8058 return Err(err);
8059 }
8060 }
8061 let confirmed = if fence.is_activation() {
8062 load_pool_meta_replicas(pools, true).await?
8063 } else {
8064 let confirmed = load_pool_meta_replicas_observing(pools, true, write_state).await?;
8065 write_state.observe_replicas(confirmed.replica_state);
8066 confirmed
8067 };
8068 let expected = committed.encode_config_data_for_v2_gate(true)?;
8069 if confirmed.canonical.as_ref() != Some(&expected) {
8070 record_pool_meta_stale_write_rejection("legacy_verify");
8071 return Err(Error::PreconditionFailed);
8072 }
8073 return Ok(confirmed.meta);
8074 }
8075
8076 let cluster_id = selection
8077 .revision
8078 .cluster_id
8079 .or(selection.generation_identity.map(|(cluster_id, _)| cluster_id))
8080 .or(write_state.expected_cluster_id)
8081 .ok_or_else(|| Error::other("pool metadata V3 save failed: cluster identity is not initialized"))?;
8082 let epoch = if selection.revision.is_generation_protocol() {
8083 selection.revision.epoch
8084 } else {
8085 selection
8086 .generation_identity
8087 .map(|(_, epoch)| epoch)
8088 .or(write_state.cluster_epoch)
8089 .unwrap_or(POOL_META_INITIAL_EPOCH)
8090 };
8091 let generation = if selection.revision.is_generation_protocol() {
8092 selection
8093 .revision
8094 .generation
8095 .checked_add(1)
8096 .ok_or_else(|| Error::other("pool metadata V3 generation exhausted"))?
8097 } else {
8098 1
8099 };
8100 let revision = PoolMetaRevision {
8101 version: POOL_META_GENERATION_VERSION,
8102 cluster_id: Some(cluster_id),
8103 epoch,
8104 generation,
8105 transaction_id: Some(uuid::Uuid::new_v4()),
8106 };
8107 let previous = if let Some(canonical) = selection.canonical.clone() {
8108 PoolMetaCommittedCandidate {
8109 canonical,
8110 meta: selection.meta.clone(),
8111 revision: selection.revision,
8112 }
8113 } else {
8114 let empty = PoolMeta {
8115 version: POOL_META_V1_VERSION,
8116 ..Default::default()
8117 };
8118 PoolMetaCommittedCandidate {
8119 canonical: empty.encode_config_data_for_v2_gate(true)?,
8120 meta: empty,
8121 revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
8122 }
8123 };
8124 let pending = encode_pool_meta_v3_envelope(&committed, revision, false, Some(&previous))?;
8125 let durable = encode_pool_meta_v3_envelope(&committed, revision, true, None)?;
8126 let concurrent_phases = pools.len() > 1
8127 && selection.revision.is_generation_protocol()
8128 && !selection.replica_state.needs_repair
8129 && !bootstrap_generation_required
8130 && matches!(fence, PoolMetaPersistenceFence::Distributed(Some(_)));
8131 #[cfg(feature = "e2e-test-hooks")]
8132 let phase_barrier = if concurrent_phases && pools.len() == 4 {
8133 PoolMetaPhaseBarrier::bind(&previous, &committed, revision, &durable).await?
8134 } else {
8135 None
8136 };
8137 let mut pending_tokens = Vec::with_capacity(pools.len());
8138 if concurrent_phases {
8139 for result in save_pool_meta_phase(
8141 pools.as_slice(),
8142 &pending,
8143 &selection.cas_tokens,
8144 fence,
8145 "prepare_cas",
8146 transaction_arm,
8147 #[cfg(feature = "e2e-test-hooks")]
8148 phase_barrier.as_ref(),
8149 )
8150 .await
8151 {
8152 let etag = result?
8153 .etag
8154 .filter(|etag| !etag.trim().is_empty())
8155 .ok_or_else(|| Error::other("pool metadata V3 prepare succeeded without a conditional-write revision"))?;
8156 pending_tokens.push(PoolMetaCasToken::Existing(etag));
8157 }
8158 } else {
8159 for (pool, token) in pools.iter().cloned().zip(&selection.cas_tokens) {
8160 let object_info = save_pool_meta_object_cas(
8161 pool,
8162 POOL_META_NAME,
8163 pending.clone(),
8164 token,
8165 fence,
8166 "prepare_cas",
8167 transaction_arm,
8168 )
8169 .await?;
8170 let etag = object_info
8171 .etag
8172 .filter(|etag| !etag.trim().is_empty())
8173 .ok_or_else(|| Error::other("pool metadata V3 prepare succeeded without a conditional-write revision"))?;
8174 pending_tokens.push(PoolMetaCasToken::Existing(etag));
8175 }
8176 }
8177
8178 #[cfg(feature = "e2e-test-hooks")]
8179 if let Some(barrier) = &phase_barrier
8180 && barrier.arm.case == PoolMetaPhaseBarrierCase::PreparedAll
8181 {
8182 barrier.ready().await?;
8183 }
8184 let mut commit_error = None;
8185 let mut commit_succeeded = false;
8186 if concurrent_phases {
8187 for result in save_pool_meta_phase(
8188 pools.as_slice(),
8189 &durable,
8190 &pending_tokens,
8191 fence,
8192 "commit_cas",
8193 transaction_arm,
8194 #[cfg(feature = "e2e-test-hooks")]
8195 phase_barrier.as_ref(),
8196 )
8197 .await
8198 {
8199 match result {
8200 Ok(_) => commit_succeeded = true,
8201 Err(err) => {
8202 commit_error.get_or_insert(err);
8203 }
8204 }
8205 }
8206 } else {
8207 #[cfg(test)]
8208 let mut first_pool = true;
8209 for (pool, token) in pools.iter().cloned().zip(&pending_tokens) {
8210 match save_pool_meta_object_cas(
8211 pool.clone(),
8212 POOL_META_NAME,
8213 durable.clone(),
8214 token,
8215 fence,
8216 "commit_cas",
8217 transaction_arm,
8218 )
8219 .await
8220 {
8221 Ok(_) => {
8222 commit_succeeded = true;
8223 #[cfg(test)]
8224 if first_pool && let PoolMetaPersistenceFence::Activation(activation_fence) = fence {
8225 pause_pool_activation_after_durable_save(&pool, activation_fence).await;
8226 }
8227 }
8228 Err(err) => {
8229 commit_error.get_or_insert(err);
8230 }
8231 }
8232 #[cfg(test)]
8233 {
8234 first_pool = false;
8235 }
8236 }
8237 }
8238 let confirmed = if fence.is_activation() {
8239 load_pool_meta_replicas(pools.clone(), true).await?
8240 } else {
8241 let confirmed = load_pool_meta_replicas_observing(pools.clone(), true, write_state).await?;
8242 write_state.observe_replicas(confirmed.replica_state);
8243 confirmed
8244 };
8245 if confirmed.revision == revision && confirmed.canonical.as_ref() == Some(&durable) {
8246 persist_pool_meta_identity(pools, write_state, true, PoolMetaIdentityWriteScope::All, fence, transaction_arm).await?;
8247 #[cfg(feature = "e2e-test-hooks")]
8248 startup_cas_test_observe(serde_json::json!({
8249 "kind": "confirmed", "object": POOL_META_NAME,
8250 "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&durable)),
8251 "generation": confirmed.revision.generation,
8252 "transaction_id": confirmed.revision.transaction_id,
8253 }));
8254 #[cfg(feature = "e2e-test-hooks")]
8255 if let Some(barrier) = &phase_barrier
8256 && barrier.arm.case == PoolMetaPhaseBarrierCase::BeforePublish
8257 {
8258 barrier.ready().await?;
8259 }
8260 return Ok(confirmed.meta);
8261 }
8262 if !commit_succeeded {
8263 return Err(commit_error.unwrap_or(Error::PreconditionFailed));
8264 }
8265 Err(commit_error.unwrap_or_else(|| {
8266 Error::other("pool metadata recovery required: committed V3 transaction was not selected after write")
8267 }))
8268 }
8269
8270 #[cfg(test)]
8271 async fn save_no_lock_observing<S>(&self, pools: Vec<Arc<S>>, write_state: &mut PoolMetaWriteState) -> Result<()>
8272 where
8273 S: EcstoreObjectIO,
8274 {
8275 let indices = (0..self.pools.len()).collect::<Vec<_>>();
8276 let outcome = self.save_no_lock_armed(pools, write_state, None, &indices).await?;
8277 outcome.disarm();
8278 Ok(())
8279 }
8280
8281 pub fn decommission_cancel(&mut self, idx: usize) -> bool {
8282 self.decommission_cancel_at(idx, OffsetDateTime::now_utc(), None)
8283 }
8284
8285 #[cfg(test)]
8286 pub(crate) fn decommission_cancel_at_for_test(
8287 &mut self,
8288 idx: usize,
8289 now: OffsetDateTime,
8290 rebalance_meta: Option<&RebalanceMeta>,
8291 ) -> bool {
8292 self.decommission_cancel_at(idx, now, rebalance_meta)
8293 }
8294
8295 fn decommission_cancel_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool {
8296 let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else {
8297 return false;
8298 };
8299 if d.canceled {
8300 return false;
8301 }
8302
8303 let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8304 let mut pd = d.clone();
8305 pd.canceled = true;
8306 pd.failed = false;
8307 pd.complete = false;
8308 pd.start_time = None;
8309 pd.terminal_reload_attempt_at = None;
8310 pd.terminal_reload_failures.clear();
8311 pd.capacity_blocked_reason = None;
8312 release_decommission_capacity_reservation(&mut pd, DECOMMISSION_CAPACITY_RELEASE_CANCELED, last_update);
8313
8314 let Some(stats) = self.pools.get_mut(idx) else {
8315 return false;
8316 };
8317 stats.last_update = last_update;
8318 stats.decommission = Some(pd);
8319 true
8320 }
8321 pub fn decommission_failed(&mut self, idx: usize) -> bool {
8322 self.decommission_failed_at(idx, OffsetDateTime::now_utc(), None)
8323 }
8324
8325 #[cfg(test)]
8326 pub(crate) fn decommission_failed_at_for_test(
8327 &mut self,
8328 idx: usize,
8329 now: OffsetDateTime,
8330 rebalance_meta: Option<&RebalanceMeta>,
8331 ) -> bool {
8332 self.decommission_failed_at(idx, now, rebalance_meta)
8333 }
8334
8335 fn decommission_failed_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool {
8336 let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else {
8337 return false;
8338 };
8339 if !is_decommission_active(d.complete, d.failed, d.canceled) {
8340 return false;
8341 }
8342
8343 let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8344 let mut pd = d.clone();
8345 pd.canceled = false;
8346 pd.failed = true;
8347 pd.complete = false;
8348 pd.start_time = None;
8349 pd.terminal_reload_attempt_at = None;
8350 pd.terminal_reload_failures.clear();
8351 pd.capacity_blocked_reason = None;
8352 release_decommission_capacity_reservation(&mut pd, DECOMMISSION_CAPACITY_RELEASE_FAILED, last_update);
8353
8354 let Some(stats) = self.pools.get_mut(idx) else {
8355 return false;
8356 };
8357 stats.last_update = last_update;
8358 stats.decommission = Some(pd);
8359 true
8360 }
8361
8362 pub fn clear_decommission(&mut self, idx: usize) -> Result<bool> {
8363 self.clear_decommission_at(idx, OffsetDateTime::now_utc(), None)
8364 }
8365
8366 #[cfg(test)]
8367 pub(crate) fn clear_decommission_at_for_test(
8368 &mut self,
8369 idx: usize,
8370 now: OffsetDateTime,
8371 rebalance_meta: Option<&RebalanceMeta>,
8372 ) -> Result<bool> {
8373 self.clear_decommission_at(idx, now, rebalance_meta)
8374 }
8375
8376 fn clear_decommission_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> Result<bool> {
8377 let pool_count = self.pools.len();
8378 ensure_valid_decommission_pool_index(pool_count, idx)?;
8379
8380 let Some(pool) = self.pools.get(idx) else {
8381 return Err(invalid_decommission_pool_index_error(pool_count, idx));
8382 };
8383
8384 let (decommission_present, complete, failed, canceled, unresolved_entries) = pool
8385 .decommission
8386 .as_ref()
8387 .map(|info| {
8388 (
8389 info.has_decommission_state(),
8390 info.complete,
8391 info.failed,
8392 info.canceled,
8393 info.unresolved_entries.len(),
8394 )
8395 })
8396 .unwrap_or((false, false, false, false, 0));
8397
8398 ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled, unresolved_entries)?;
8399
8400 let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8401 let Some(pool) = self.pools.get_mut(idx) else {
8402 return Err(invalid_decommission_pool_index_error(pool_count, idx));
8403 };
8404 pool.last_update = last_update;
8405 pool.decommission = Some(PoolDecommissionInfo::default());
8408 Ok(true)
8409 }
8410
8411 pub fn decommission_complete(&mut self, idx: usize) -> bool {
8412 self.decommission_complete_at(idx, OffsetDateTime::now_utc(), None)
8413 }
8414
8415 #[cfg(test)]
8416 pub(crate) fn decommission_complete_at_for_test(
8417 &mut self,
8418 idx: usize,
8419 now: OffsetDateTime,
8420 rebalance_meta: Option<&RebalanceMeta>,
8421 ) -> bool {
8422 self.decommission_complete_at(idx, now, rebalance_meta)
8423 }
8424
8425 fn decommission_complete_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool {
8426 let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else {
8427 return false;
8428 };
8429 if !is_decommission_active(d.complete, d.failed, d.canceled) {
8430 return false;
8431 }
8432
8433 let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8434 let mut pd = d.clone();
8435 pd.canceled = false;
8436 pd.failed = false;
8437 pd.complete = true;
8438 pd.terminal_reload_attempt_at = None;
8439 pd.terminal_reload_failures.clear();
8440 pd.capacity_blocked_reason = None;
8441 release_decommission_capacity_reservation(&mut pd, DECOMMISSION_CAPACITY_RELEASE_COMPLETED, last_update);
8442
8443 let Some(stats) = self.pools.get_mut(idx) else {
8444 return false;
8445 };
8446 stats.last_update = last_update;
8447 stats.decommission = Some(pd);
8448 true
8449 }
8450 fn set_decommission_state_at(
8451 &mut self,
8452 idx: usize,
8453 pi: PoolSpaceInfo,
8454 queued: bool,
8455 now: OffsetDateTime,
8456 rebalance_meta: Option<&RebalanceMeta>,
8457 ) -> Result<()> {
8458 let pool_count = self.pools.len();
8459 ensure_valid_decommission_pool_index(pool_count, idx)?;
8460
8461 let Some(pool) = self.pools.get(idx) else {
8462 return Err(invalid_decommission_pool_index_error(pool_count, idx));
8463 };
8464
8465 ensure_decommission_start_allowed(decommission_start_pool_state(Some(pool)))?;
8466
8467 let generation = self.next_scanner_data_movement_update(now, rebalance_meta);
8468 let Some(pool) = self.pools.get_mut(idx) else {
8469 return Err(invalid_decommission_pool_index_error(pool_count, idx));
8470 };
8471 let previous = pool.decommission.as_ref();
8472 pool.last_update = generation;
8473 pool.decommission = Some(build_decommission_start_state(pi, queued, generation, previous));
8474
8475 Ok(())
8476 }
8477
8478 fn set_decommission_state(&mut self, idx: usize, pi: PoolSpaceInfo, queued: bool) -> Result<()> {
8479 self.set_decommission_state_at(idx, pi, queued, OffsetDateTime::now_utc(), None)
8480 }
8481
8482 #[cfg(test)]
8483 pub(crate) fn decommission_at_for_test(
8484 &mut self,
8485 idx: usize,
8486 pi: PoolSpaceInfo,
8487 now: OffsetDateTime,
8488 rebalance_meta: Option<&RebalanceMeta>,
8489 ) -> Result<()> {
8490 self.set_decommission_state_at(idx, pi, false, now, rebalance_meta)
8491 }
8492
8493 pub fn decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> {
8494 self.set_decommission_state(idx, pi, false)
8495 }
8496
8497 pub fn queue_decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> {
8498 self.set_decommission_state(idx, pi, true)
8499 }
8500
8501 pub fn record_decommission_terminal_reload_failure(&mut self, idx: usize, stage: &str, message: String) -> Result<bool> {
8502 self.record_decommission_terminal_reload_failure_at(idx, stage, message, OffsetDateTime::now_utc(), None)
8503 }
8504
8505 #[cfg(test)]
8506 pub(crate) fn record_decommission_terminal_reload_failure_at_for_test(
8507 &mut self,
8508 idx: usize,
8509 stage: &str,
8510 message: String,
8511 now: OffsetDateTime,
8512 rebalance_meta: Option<&RebalanceMeta>,
8513 ) -> Result<bool> {
8514 self.record_decommission_terminal_reload_failure_at(idx, stage, message, now, rebalance_meta)
8515 }
8516
8517 fn record_decommission_terminal_reload_failure_at(
8518 &mut self,
8519 idx: usize,
8520 stage: &str,
8521 message: String,
8522 now: OffsetDateTime,
8523 rebalance_meta: Option<&RebalanceMeta>,
8524 ) -> Result<bool> {
8525 let pool_count = self.pools.len();
8526 ensure_valid_decommission_pool_index(pool_count, idx)?;
8527
8528 let Some(pool) = self.pools.get(idx) else {
8529 return Err(invalid_decommission_pool_index_error(pool_count, idx));
8530 };
8531 let Some(info) = pool.decommission.as_ref() else {
8532 return Err(decommission_metadata_not_initialized_error("record decommission terminal reload failure"));
8533 };
8534
8535 let failure = format!("{stage}: {message}");
8536 if info.terminal_reload_failures.last().is_some_and(|last| last == &failure) {
8537 return Ok(false);
8538 }
8539
8540 let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8541 let Some(pool) = self.pools.get_mut(idx) else {
8542 return Err(invalid_decommission_pool_index_error(pool_count, idx));
8543 };
8544 let Some(info) = pool.decommission.as_mut() else {
8545 return Err(decommission_metadata_not_initialized_error("record decommission terminal reload failure"));
8546 };
8547 pool.last_update = last_update;
8548 info.terminal_reload_attempt_at = Some(last_update);
8549 info.terminal_reload_failures.push(failure);
8550 Ok(true)
8551 }
8552
8553 fn mark_decommission_capacity_blocked(&mut self, idx: usize, reason: String, now: OffsetDateTime) -> Result<bool> {
8554 let pool_count = self.pools.len();
8555 let pool = self
8556 .pools
8557 .get_mut(idx)
8558 .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, idx))?;
8559 let info = pool
8560 .decommission
8561 .as_mut()
8562 .ok_or_else(|| decommission_metadata_not_initialized_error("pause decommission for target capacity"))?;
8563 if !is_decommission_active(info.complete, info.failed, info.canceled) {
8564 return Ok(false);
8565 }
8566 if let Some(reservation) = info.capacity_reservation.as_mut().filter(|reservation| reservation.active()) {
8567 renew_decommission_capacity_reservation(reservation, now, true);
8568 }
8569 let changed = info.capacity_blocked_reason.as_deref() != Some(reason.as_str());
8570 info.capacity_blocked_reason = Some(reason);
8571 pool.last_update = now;
8572 Ok(changed)
8573 }
8574
8575 pub fn promote_queued_decommission(&mut self, idx: usize) -> bool {
8576 self.promote_queued_decommission_at(idx, OffsetDateTime::now_utc(), None)
8577 }
8578
8579 #[cfg(test)]
8580 pub(crate) fn promote_queued_decommission_at_for_test(
8581 &mut self,
8582 idx: usize,
8583 now: OffsetDateTime,
8584 rebalance_meta: Option<&RebalanceMeta>,
8585 ) -> bool {
8586 self.promote_queued_decommission_at(idx, now, rebalance_meta)
8587 }
8588
8589 fn promote_queued_decommission_at(
8590 &mut self,
8591 idx: usize,
8592 now: OffsetDateTime,
8593 rebalance_meta: Option<&RebalanceMeta>,
8594 ) -> bool {
8595 let Some(info) = self.pools.get(idx).and_then(|pool| pool.decommission.as_ref()) else {
8596 return false;
8597 };
8598 if !info.queued || !is_decommission_active(info.complete, info.failed, info.canceled) {
8599 return false;
8600 }
8601
8602 let generation = self.next_scanner_data_movement_update(now, rebalance_meta);
8603 let Some(pool) = self.pools.get_mut(idx) else {
8604 return false;
8605 };
8606 let Some(info) = pool.decommission.as_mut() else {
8607 return false;
8608 };
8609 pool.last_update = generation;
8610 info.queued = false;
8611 info.start_time = Some(generation);
8612 for entry in &mut info.unresolved_entries {
8613 entry.source_generation = generation;
8614 }
8615 true
8616 }
8617 pub fn queue_buckets(&mut self, idx: usize, bks: Vec<DecomBucketInfo>) {
8618 if let Some(pool) = self.pools.get_mut(idx)
8619 && let Some(dec) = pool.decommission.as_mut()
8620 {
8621 for bk in bks.iter() {
8622 dec.bucket_push(bk);
8623 }
8624 }
8625 }
8626 pub fn pending_buckets(&self, idx: usize) -> Vec<DecomBucketInfo> {
8627 let mut list = Vec::new();
8628
8629 if let Some(pool) = self.pools.get(idx)
8630 && let Some(ref info) = pool.decommission
8631 {
8632 for bk in info.queued_buckets.iter() {
8633 let (name, prefix) = path2_bucket_object(bk);
8634 list.push(DecomBucketInfo { name, prefix });
8635 }
8636 }
8637
8638 list
8639 }
8640
8641 pub fn is_bucket_decommissioned(&self, idx: usize, bucket: String) -> bool {
8642 self.pools
8643 .get(idx)
8644 .and_then(|pool| pool.decommission.as_ref())
8645 .is_some_and(|info| info.is_bucket_decommissioned(&bucket))
8646 }
8647
8648 pub fn bucket_done(&mut self, idx: usize, bucket: String) -> bool {
8649 if let Some(pool) = self.pools.get_mut(idx) {
8650 if let Some(info) = pool.decommission.as_mut() {
8651 info.bucket_pop(&bucket)
8652 } else {
8653 false
8654 }
8655 } else {
8656 false
8657 }
8658 }
8659
8660 pub fn count_item(&mut self, idx: usize, size: usize, failed: bool) {
8661 if let Some(pool) = self.pools.get_mut(idx)
8662 && let Some(info) = pool.decommission.as_mut()
8663 {
8664 if failed {
8665 info.items_decommission_failed += 1;
8666 info.bytes_failed += size;
8667 } else {
8668 info.items_decommissioned += 1;
8669 info.bytes_done += size;
8670 }
8671 }
8672 }
8673
8674 pub fn track_current_bucket_object(&mut self, idx: usize, bucket: String, object: String) {
8675 self.track_current_bucket_object_stage(idx, bucket, object, String::new());
8676 }
8677
8678 pub fn track_current_bucket_object_stage(&mut self, idx: usize, bucket: String, object: String, stage: String) {
8679 if self.pools.get(idx).is_none_or(|v| v.decommission.is_none()) {
8680 return;
8681 }
8682
8683 if let Some(pool) = self.pools.get_mut(idx)
8684 && let Some(info) = pool.decommission.as_mut()
8685 {
8686 info.object = object;
8687 info.bucket = bucket;
8688 info.stage = stage;
8689 }
8690 }
8691
8692 pub fn update_after(&mut self, idx: usize, duration: Duration) -> Result<bool> {
8693 Ok(self
8694 .decommission_progress_checkpoint(idx, duration, OffsetDateTime::now_utc(), None)?
8695 .is_some())
8696 }
8697
8698 pub fn validate(&self, pools: Vec<Arc<Sets>>) -> Result<bool> {
8699 struct PoolInfo {
8700 position: usize,
8701 completed: bool,
8702 #[allow(dead_code, reason = "written but never read back (backlog#1823)")]
8703 decom_started: bool,
8704 }
8705
8706 let mut remembered_pools = HashMap::new();
8707 for (idx, pool) in self.pools.iter().enumerate() {
8708 let mut complete = false;
8709 let mut decom_started = false;
8710 if let Some(decommission) = &pool.decommission {
8711 if decommission.complete {
8712 complete = true;
8713 }
8714 decom_started = true;
8715 }
8716 remembered_pools.insert(
8717 pool.cmd_line.clone(),
8718 PoolInfo {
8719 position: idx,
8720 completed: complete,
8721 decom_started,
8722 },
8723 );
8724 }
8725
8726 let mut specified_pools = HashMap::new();
8727 for (idx, pool) in pools.iter().enumerate() {
8728 specified_pools.insert(pool.endpoints.cmd_line.clone(), idx);
8729 }
8730
8731 let mut update = false;
8732
8733 for k in specified_pools.keys() {
8735 if let Some(pi) = remembered_pools.get(k) {
8736 ensure_pool_not_left_in_cmdline_after_decommission(pi.position, k, pi.completed)?;
8737 } else {
8738 update = true;
8740 }
8741 }
8742
8743 if specified_pools.len() == remembered_pools.len() {
8744 for (k, pi) in remembered_pools.iter() {
8745 if let Some(pos) = specified_pools.get(k)
8746 && *pos != pi.position
8747 {
8748 update = true; }
8750 }
8751 }
8752
8753 if !update {
8754 update = specified_pools.len() != remembered_pools.len();
8755 }
8756
8757 Ok(update)
8758 }
8759
8760 pub fn return_resumable_pools(&self) -> Vec<PoolStatus> {
8761 resumable_decommission_queue_indices(self)
8762 .into_iter()
8763 .map(|idx| self.pools[idx].clone())
8764 .collect()
8765 }
8766}
8767
8768pub fn path2_bucket_object(name: &str) -> (String, String) {
8769 path_to_bucket_object(name)
8770}
8771
8772pub fn path2_bucket_object_with_base_path(base_path: &str, path: &str) -> (String, String) {
8773 path_to_bucket_object_with_base_path(base_path, path)
8774}
8775
8776#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
8777#[serde(deny_unknown_fields)]
8778pub struct DecommissionUnresolvedEntry {
8779 pub bucket: String,
8780 pub object: String,
8781 #[serde(rename = "poolIndex")]
8782 pub pool_index: usize,
8783 #[serde(rename = "setIndex")]
8784 pub set_index: usize,
8785 #[serde(rename = "sourceGeneration", with = "time::serde::rfc3339")]
8786 pub source_generation: OffsetDateTime,
8787 #[serde(rename = "candidateCount")]
8788 pub candidate_count: usize,
8789 #[serde(rename = "diskErrorCount")]
8790 pub disk_error_count: usize,
8791 #[serde(rename = "observedAt", with = "time::serde::rfc3339")]
8792 pub observed_at: OffsetDateTime,
8793 pub reason: String,
8794}
8795
8796#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
8797#[serde(rename_all = "camelCase", deny_unknown_fields)]
8798pub struct DecommissionErasureLayout {
8799 pub data: usize,
8800 pub parity: usize,
8801}
8802
8803impl DecommissionErasureLayout {
8804 fn width(self) -> usize {
8805 self.data.saturating_add(self.parity)
8806 }
8807
8808 fn is_valid(self) -> bool {
8809 self.data > 0 && self.data.checked_add(self.parity).is_some()
8810 }
8811}
8812
8813#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
8814#[serde(rename_all = "camelCase", deny_unknown_fields)]
8815pub struct DecommissionCapacityTarget {
8816 pub pool_index: usize,
8817 pub layout: DecommissionErasureLayout,
8818 pub physical_total_at_reservation: usize,
8819 pub physical_free_at_reservation: usize,
8820 pub reserved_physical_bytes: usize,
8821 #[serde(default)]
8822 pub consumed_physical_bytes: usize,
8823 #[serde(default)]
8824 pub observed_physical_bytes: usize,
8825 #[serde(default)]
8826 pub inflight_physical_bytes: usize,
8827 #[serde(default)]
8828 pub pending_physical_bytes: usize,
8829 #[serde(default)]
8830 pub pending_mutation_id: Option<uuid::Uuid>,
8831 #[serde(default, skip_serializing_if = "Vec::is_empty")]
8832 pub temporary_mutations: Vec<DecommissionCapacityTemporaryMutation>,
8833}
8834
8835#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
8836#[serde(rename_all = "camelCase", deny_unknown_fields)]
8837pub struct DecommissionCapacityTemporaryMutation {
8838 pub mutation_id: uuid::Uuid,
8839 pub physical_bytes: usize,
8840}
8841
8842#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
8843#[serde(rename_all = "camelCase", deny_unknown_fields)]
8844pub struct DecommissionCapacityReservation {
8845 pub model_version: u16,
8846 pub operation_id: uuid::Uuid,
8847 pub generation: u64,
8848 pub owner_nonce: uuid::Uuid,
8849 pub source_pool_index: usize,
8850 pub source_layout: DecommissionErasureLayout,
8851 pub source_physical_total_bytes: usize,
8852 pub source_physical_bytes: usize,
8853 pub source_data_equivalent_bytes: usize,
8854 pub predicted_physical_bytes: usize,
8855 pub temporary_copies: usize,
8856 pub temporary_physical_bytes: usize,
8857 pub peak_physical_bytes: usize,
8858 pub committed_data_bytes: usize,
8859 #[serde(default)]
8860 pub consumed_target_physical_bytes: usize,
8861 #[serde(default)]
8862 pub observed_target_physical_bytes: usize,
8863 #[serde(default)]
8864 pub inflight_target_physical_bytes: usize,
8865 #[serde(default)]
8866 pub pending_target_physical_bytes: usize,
8867 pub prediction_error_bytes: i64,
8868 pub targets: Vec<DecommissionCapacityTarget>,
8869 #[serde(with = "time::serde::rfc3339")]
8870 pub created_at: OffsetDateTime,
8871 #[serde(with = "time::serde::rfc3339")]
8872 pub renewed_at: OffsetDateTime,
8873 #[serde(with = "time::serde::rfc3339")]
8874 pub expires_at: OffsetDateTime,
8875 #[serde(with = "time::serde::rfc3339::option", default)]
8876 pub recovered_at: Option<OffsetDateTime>,
8877 #[serde(with = "time::serde::rfc3339::option", default)]
8878 pub released_at: Option<OffsetDateTime>,
8879 #[serde(default, skip_serializing_if = "Option::is_none")]
8880 pub release_reason: Option<String>,
8881}
8882
8883#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8884enum DecommissionCapacityMutationMode {
8885 Durable,
8886 Temporary,
8887 NonGrowingReplacement,
8888 TemporaryRelease,
8889}
8890
8891#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8892struct DecommissionCapacityReleaseProof {
8893 confirmed_absent: bool,
8894 clear_pending: bool,
8895}
8896
8897impl DecommissionCapacityReleaseProof {
8898 const fn confirmed_absence(clear_pending: bool) -> Self {
8899 Self {
8900 confirmed_absent: true,
8901 clear_pending,
8902 }
8903 }
8904}
8905
8906#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8907pub(crate) struct DecommissionCapacityOwner {
8908 pub(crate) source_pool_index: usize,
8909 pub(crate) operation_id: uuid::Uuid,
8910 pub(crate) generation: u64,
8911 pub(crate) owner_nonce: uuid::Uuid,
8912 pub(crate) mutation_id: Option<uuid::Uuid>,
8913}
8914
8915#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
8916struct DecommissionCapacityTargetPermitKey {
8917 store_id: uuid::Uuid,
8918 target_pool_index: usize,
8919 source_pool_index: usize,
8920 operation_id: uuid::Uuid,
8921 generation: u64,
8922 owner_nonce: uuid::Uuid,
8923 mutation_id: uuid::Uuid,
8924}
8925
8926impl DecommissionCapacityTargetPermitKey {
8927 fn new(store_id: uuid::Uuid, target_pool_index: usize, owner: DecommissionCapacityOwner) -> Option<Self> {
8928 Some(Self {
8929 store_id,
8930 target_pool_index,
8931 source_pool_index: owner.source_pool_index,
8932 operation_id: owner.operation_id,
8933 generation: owner.generation,
8934 owner_nonce: owner.owner_nonce,
8935 mutation_id: owner.mutation_id?,
8936 })
8937 }
8938
8939 fn owns_same_mutation(&self, store_id: uuid::Uuid, owner: DecommissionCapacityOwner) -> bool {
8940 owner.mutation_id.is_some_and(|mutation_id| {
8941 self.store_id == store_id
8942 && self.source_pool_index == owner.source_pool_index
8943 && self.operation_id == owner.operation_id
8944 && self.generation == owner.generation
8945 && self.owner_nonce == owner.owner_nonce
8946 && self.mutation_id == mutation_id
8947 })
8948 }
8949}
8950
8951struct DecommissionCapacityTargetPermit {
8952 key: DecommissionCapacityTargetPermitKey,
8953}
8954
8955static DECOMMISSION_CAPACITY_TARGET_PERMITS: std::sync::OnceLock<
8959 std::sync::Mutex<HashMap<DecommissionCapacityTargetPermitKey, rustfs_lock::NamespaceLockGuard>>,
8960> = std::sync::OnceLock::new();
8961
8962fn decommission_capacity_target_permits()
8963-> &'static std::sync::Mutex<HashMap<DecommissionCapacityTargetPermitKey, rustfs_lock::NamespaceLockGuard>> {
8964 DECOMMISSION_CAPACITY_TARGET_PERMITS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
8965}
8966
8967fn install_decommission_capacity_target_permit(
8968 store_id: uuid::Uuid,
8969 target_pool_index: usize,
8970 owner: DecommissionCapacityOwner,
8971 guard: rustfs_lock::NamespaceLockGuard,
8972) -> Result<DecommissionCapacityTargetPermit> {
8973 let key = DecommissionCapacityTargetPermitKey::new(store_id, target_pool_index, owner)
8974 .ok_or_else(|| Error::other("decommission target permit is missing its mutation identity"))?;
8975 let mut permits = decommission_capacity_target_permits()
8976 .lock()
8977 .unwrap_or_else(std::sync::PoisonError::into_inner);
8978 if permits.keys().any(|candidate| candidate.owns_same_mutation(store_id, owner)) {
8979 return Err(Error::other("decommission target permit already exists for this mutation"));
8980 }
8981 permits.insert(key, guard);
8982 Ok(DecommissionCapacityTargetPermit { key })
8983}
8984
8985fn take_decommission_capacity_target_permit(
8986 store_id: uuid::Uuid,
8987 target_pool_index: usize,
8988 owner: DecommissionCapacityOwner,
8989) -> Option<rustfs_lock::NamespaceLockGuard> {
8990 let key = DecommissionCapacityTargetPermitKey::new(store_id, target_pool_index, owner)?;
8991 decommission_capacity_target_permits()
8992 .lock()
8993 .unwrap_or_else(std::sync::PoisonError::into_inner)
8994 .remove(&key)
8995}
8996
8997fn decommission_capacity_target_permit_index(store_id: uuid::Uuid, owner: DecommissionCapacityOwner) -> Option<usize> {
8998 decommission_capacity_target_permits()
8999 .lock()
9000 .unwrap_or_else(std::sync::PoisonError::into_inner)
9001 .keys()
9002 .find(|key| key.owns_same_mutation(store_id, owner))
9003 .map(|key| key.target_pool_index)
9004}
9005
9006fn discard_decommission_capacity_target_permit_except(
9007 store_id: uuid::Uuid,
9008 owner: DecommissionCapacityOwner,
9009 retained_target_pool_index: Option<usize>,
9010) {
9011 let guard = {
9012 let mut permits = decommission_capacity_target_permits()
9013 .lock()
9014 .unwrap_or_else(std::sync::PoisonError::into_inner);
9015 let key = permits
9016 .keys()
9017 .find(|key| key.owns_same_mutation(store_id, owner) && retained_target_pool_index != Some(key.target_pool_index))
9018 .copied();
9019 key.and_then(|key| permits.remove(&key))
9020 };
9021 drop(guard);
9022}
9023
9024impl Drop for DecommissionCapacityTargetPermit {
9025 fn drop(&mut self) {
9026 let guard = decommission_capacity_target_permits()
9027 .lock()
9028 .unwrap_or_else(std::sync::PoisonError::into_inner)
9029 .remove(&self.key);
9030 drop(guard);
9031 }
9032}
9033
9034#[derive(Debug, Clone, PartialEq, Eq)]
9035struct DecommissionCapacityTerminalFencePlan {
9036 model_version: u16,
9037 operation_id: uuid::Uuid,
9038 generation: u64,
9039 owner_nonce: uuid::Uuid,
9040 target_pool_indices: Vec<usize>,
9041}
9042
9043fn decommission_capacity_terminal_fence_plan(
9044 meta: &PoolMeta,
9045 source_pool_index: usize,
9046) -> Result<Option<DecommissionCapacityTerminalFencePlan>> {
9047 let Some(reservation) = meta
9048 .pools
9049 .get(source_pool_index)
9050 .and_then(|pool| pool.decommission.as_ref())
9051 .and_then(|info| info.capacity_reservation.as_ref())
9052 .filter(|reservation| reservation.active())
9053 else {
9054 return Ok(None);
9055 };
9056 if reservation.source_pool_index != source_pool_index {
9057 return Err(Error::DecommissionCapacity(
9058 "decommission terminal transition found a mismatched capacity source".to_string(),
9059 ));
9060 }
9061 if !matches!(
9062 reservation.model_version,
9063 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
9064 ) {
9065 return Err(Error::DecommissionCapacity(format!(
9066 "decommission terminal transition found unsupported capacity lock model {}",
9067 reservation.model_version
9068 )));
9069 }
9070 let mut target_pool_indices = reservation.targets.iter().map(|target| target.pool_index).collect::<Vec<_>>();
9071 target_pool_indices.sort_unstable();
9072 Ok(Some(DecommissionCapacityTerminalFencePlan {
9073 model_version: reservation.model_version,
9074 operation_id: reservation.operation_id,
9075 generation: reservation.generation,
9076 owner_nonce: reservation.owner_nonce,
9077 target_pool_indices,
9078 }))
9079}
9080
9081#[derive(Debug, Clone, PartialEq, Eq)]
9082pub(crate) struct DecommissionDurableIlmCheckpointTarget {
9083 pub(crate) source_pool_index: usize,
9084 pub(crate) target_pool_index: usize,
9085 pub(crate) capacity_owner: DecommissionCapacityOwner,
9086 pub(crate) already_committed: bool,
9087 pub(crate) target_etag: Option<String>,
9088}
9089
9090impl DecommissionCapacityOwner {
9091 pub(crate) fn apply_to(self, opts: &mut ObjectOptions) {
9092 opts.src_pool_idx = self.source_pool_index;
9093 let admission = opts
9094 .decommission_capacity
9095 .get_or_insert_with(|| Box::new(DecommissionCapacityOptions::default()));
9096 admission.operation_id = Some(self.operation_id);
9097 admission.generation = Some(self.generation);
9098 admission.owner_nonce = Some(self.owner_nonce);
9099 admission.mutation_id = self.mutation_id;
9100 }
9101
9102 pub(crate) fn from_options(opts: &ObjectOptions) -> Option<Self> {
9103 let capacity = opts.decommission_capacity.as_deref()?;
9104 Some(Self {
9105 source_pool_index: opts.src_pool_idx,
9106 operation_id: capacity.operation_id?,
9107 generation: capacity.generation?,
9108 owner_nonce: capacity.owner_nonce?,
9109 mutation_id: capacity.mutation_id,
9110 })
9111 }
9112
9113 pub(crate) fn with_mutation_id(self, mutation_id: uuid::Uuid) -> Self {
9114 Self {
9115 mutation_id: Some(mutation_id),
9116 ..self
9117 }
9118 }
9119}
9120
9121pub(crate) fn decommission_capacity_mutation_id(
9122 owner: DecommissionCapacityOwner,
9123 bucket: &str,
9124 object: &str,
9125 version_id: Option<&str>,
9126 delete_marker: bool,
9127 mod_time: Option<OffsetDateTime>,
9128) -> uuid::Uuid {
9129 let mut hasher = Sha256::new();
9130 hasher.update(owner.operation_id.as_bytes());
9131 hasher.update(owner.generation.to_le_bytes());
9132 hasher.update(owner.source_pool_index.to_le_bytes());
9133 for value in [bucket, object] {
9134 hasher.update((value.len() as u64).to_le_bytes());
9135 hasher.update(value.as_bytes());
9136 }
9137 hasher.update([u8::from(delete_marker)]);
9138 if let Some(version_id) = version_id {
9139 hasher.update([1]);
9140 hasher.update((version_id.len() as u64).to_le_bytes());
9141 hasher.update(version_id.as_bytes());
9142 } else {
9143 hasher.update([0]);
9144 }
9145 if let Some(mod_time) = mod_time {
9146 hasher.update([1]);
9147 hasher.update(mod_time.unix_timestamp_nanos().to_le_bytes());
9148 } else {
9149 hasher.update([0]);
9150 }
9151 let digest = hasher.finalize();
9152 let mut bytes = [0; 16];
9153 bytes.copy_from_slice(&digest[..16]);
9154 uuid::Uuid::from_bytes(bytes)
9155}
9156
9157#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9158struct ExactDeleteCapacityReconciliation {
9159 source_pool_index: usize,
9160 target_pool_index: usize,
9161 mutation_id: uuid::Uuid,
9162 expected_data_bytes: usize,
9163 expected_target_physical_bytes: usize,
9164}
9165
9166fn plan_exact_delete_capacity_reconciliations(
9167 meta: &PoolMeta,
9168 object: &str,
9169 exact: &ObjectInfo,
9170) -> Result<Vec<ExactDeleteCapacityReconciliation>> {
9171 let version_id = exact.version_id.map(|version_id| version_id.to_string());
9172 let mut matches = Vec::new();
9173
9174 for (source_pool_index, pool) in meta.pools.iter().enumerate() {
9175 let Some(reservation) = pool
9176 .decommission
9177 .as_ref()
9178 .and_then(|info| info.capacity_reservation.as_ref())
9179 .filter(|reservation| reservation.active())
9180 else {
9181 continue;
9182 };
9183 let owner = DecommissionCapacityOwner {
9184 source_pool_index,
9185 operation_id: reservation.operation_id,
9186 generation: reservation.generation,
9187 owner_nonce: reservation.owner_nonce,
9188 mutation_id: None,
9189 };
9190 let logical_mutation_id = decommission_capacity_mutation_id(
9191 owner,
9192 &exact.bucket,
9193 &exact.name,
9194 version_id.as_deref(),
9195 exact.delete_marker,
9196 exact.mod_time,
9197 );
9198 let internal_mutation_id = if object == exact.name {
9202 logical_mutation_id
9203 } else {
9204 decommission_capacity_mutation_id(
9205 owner,
9206 &exact.bucket,
9207 object,
9208 version_id.as_deref(),
9209 exact.delete_marker,
9210 exact.mod_time,
9211 )
9212 };
9213 let mut source_match = None;
9214
9215 for target in &reservation.targets {
9216 if target.pending_physical_bytes == 0 {
9217 continue;
9218 }
9219 let Some(pending_mutation_id) = target.pending_mutation_id else {
9220 return Err(decommission_capacity_blocked_error(format!(
9221 "source pool {source_pool_index} target pool {} has pending capacity without an object identity",
9222 target.pool_index
9223 )));
9224 };
9225 if pending_mutation_id != logical_mutation_id && pending_mutation_id != internal_mutation_id {
9226 continue;
9227 }
9228 if source_match.is_some() {
9229 return Err(decommission_capacity_blocked_error(format!(
9230 "source pool {source_pool_index} has the same exact-delete capacity intent on multiple targets"
9231 )));
9232 }
9233 source_match = Some((target.pool_index, target.layout, target.pending_physical_bytes, pending_mutation_id));
9234 }
9235
9236 let Some((target_pool_index, target_layout, pending_physical_bytes, mutation_id)) = source_match else {
9237 continue;
9238 };
9239 if exact.version_id.is_none() && exact.mod_time.is_none() {
9240 return Err(decommission_capacity_blocked_error(
9241 "unversioned exact delete cannot identify pending capacity without a modification time",
9242 ));
9243 }
9244 let expected_data_bytes = if exact.delete_marker {
9245 0
9246 } else {
9247 usize::try_from(exact.size).map_err(|_| {
9248 decommission_capacity_blocked_error("exact delete cannot reconcile a negative or overflowing object size")
9249 })?
9250 };
9251 let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
9252 if pending_physical_bytes != expected_target_physical_bytes {
9253 return Err(decommission_capacity_blocked_error(format!(
9254 "source pool {source_pool_index} target pool {target_pool_index} pending capacity does not match the exact object size"
9255 )));
9256 }
9257 let remaining_target_physical_bytes = reservation
9258 .targets
9259 .iter()
9260 .find(|target| target.pool_index == target_pool_index)
9261 .map(|target| {
9262 target.remaining_reserved_physical_bytes(reservation.temporary_copies)
9263 / 1usize.saturating_add(reservation.temporary_copies)
9264 })
9265 .unwrap_or_default();
9266 let remaining_total_physical_bytes = reservation
9267 .predicted_physical_bytes
9268 .saturating_sub(reservation.consumed_target_physical_bytes);
9269 let remaining_data_bytes = reservation
9270 .source_data_equivalent_bytes
9271 .saturating_sub(reservation.committed_data_bytes);
9272 if expected_target_physical_bytes > remaining_target_physical_bytes
9273 || expected_target_physical_bytes > remaining_total_physical_bytes
9274 || expected_data_bytes > remaining_data_bytes
9275 {
9276 return Err(decommission_capacity_blocked_error(format!(
9277 "source pool {source_pool_index} target pool {target_pool_index} lacks reservation capacity for the exact object"
9278 )));
9279 }
9280 matches.push(ExactDeleteCapacityReconciliation {
9281 source_pool_index,
9282 target_pool_index,
9283 mutation_id,
9284 expected_data_bytes,
9285 expected_target_physical_bytes,
9286 });
9287 }
9288
9289 Ok(matches)
9290}
9291
9292fn ensure_exact_delete_capacity_namespace_fences(opts: &ObjectOptions, bucket: &str, object: &str) -> Result<()> {
9293 let object_fence = opts.namespace_lock_fence.as_ref().ok_or_else(|| {
9294 decommission_capacity_blocked_error("exact delete capacity reconciliation requires an object namespace fence")
9295 })?;
9296 if object_fence.is_lock_lost() {
9297 return Err(StorageError::NamespaceLockQuorumUnavailable {
9298 mode: "exact_delete_capacity_reconciliation",
9299 bucket: bucket.to_string(),
9300 object: decode_dir_object(object),
9301 required: 1,
9302 achieved: 0,
9303 });
9304 }
9305 if opts
9306 .bucket_lifecycle_lock_fence
9307 .as_ref()
9308 .is_some_and(crate::object_api::NamespaceLockFence::is_lock_lost)
9309 {
9310 return Err(StorageError::NamespaceLockQuorumUnavailable {
9311 mode: "exact_delete_capacity_bucket_generation",
9312 bucket: bucket.to_string(),
9313 object: decode_dir_object(object),
9314 required: 1,
9315 achieved: 0,
9316 });
9317 }
9318 Ok(())
9319}
9320
9321pub(crate) fn ensure_decommission_capacity_mutation_id(bucket: &str, object: &str, opts: &mut ObjectOptions) {
9322 if opts
9323 .decommission_capacity
9324 .as_deref()
9325 .is_none_or(|capacity| capacity.mutation_id.is_some())
9326 {
9327 return;
9328 }
9329 let Some(owner) = DecommissionCapacityOwner::from_options(opts) else {
9330 return;
9331 };
9332 let mutation_id =
9333 decommission_capacity_mutation_id(owner, bucket, object, opts.version_id.as_deref(), opts.delete_marker, opts.mod_time);
9334 if let Some(capacity) = opts.decommission_capacity.as_mut() {
9335 capacity.mutation_id = Some(mutation_id);
9336 }
9337}
9338
9339impl DecommissionCapacityReservation {
9340 fn active(&self) -> bool {
9341 self.released_at.is_none()
9342 }
9343
9344 fn lease_active_at(&self, now: OffsetDateTime) -> bool {
9345 self.active() && self.expires_at > now
9346 }
9347
9348 fn admits_owner(&self, owner: DecommissionCapacityOwner, now: OffsetDateTime) -> bool {
9349 self.lease_active_at(now)
9350 && self.source_pool_index == owner.source_pool_index
9351 && self.operation_id == owner.operation_id
9352 && self.generation == owner.generation
9353 && self.owner_nonce == owner.owner_nonce
9354 }
9355
9356 fn admits_cleanup_owner(&self, owner: DecommissionCapacityOwner) -> bool {
9357 self.active()
9358 && self.source_pool_index == owner.source_pool_index
9359 && self.operation_id == owner.operation_id
9360 && self.generation == owner.generation
9361 }
9362
9363 fn remaining_peak_physical_bytes(&self) -> usize {
9364 self.predicted_physical_bytes
9365 .saturating_sub(self.consumed_target_physical_bytes)
9366 .saturating_mul(1usize.saturating_add(self.temporary_copies))
9367 }
9368}
9369
9370impl DecommissionCapacityTarget {
9371 fn remaining_reserved_physical_bytes(&self, temporary_copies: usize) -> usize {
9372 self.reserved_physical_bytes.saturating_sub(
9373 self.consumed_physical_bytes
9374 .saturating_mul(1usize.saturating_add(temporary_copies)),
9375 )
9376 }
9377}
9378
9379#[derive(Debug, Clone, Serialize, Deserialize, Default)]
9380pub struct PoolDecommissionInfo {
9381 #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
9382 pub start_time: Option<OffsetDateTime>,
9383 #[serde(rename = "startSize")]
9384 pub start_size: usize,
9385 #[serde(rename = "totalSize")]
9386 pub total_size: usize,
9387 #[serde(rename = "currentSize")]
9388 pub current_size: usize,
9389 #[serde(rename = "complete")]
9390 pub complete: bool,
9391 #[serde(rename = "failed")]
9392 pub failed: bool,
9393 #[serde(rename = "canceled")]
9394 pub canceled: bool,
9395 #[serde(skip)]
9396 pub queued: bool,
9397
9398 #[serde(skip)]
9399 pub queued_buckets: Vec<String>,
9400 #[serde(skip)]
9401 pub decommissioned_buckets: Vec<String>,
9402 #[serde(skip)]
9403 pub bucket: String,
9404 #[serde(skip)]
9405 pub prefix: String,
9406 #[serde(skip)]
9407 pub object: String,
9408 #[serde(skip)]
9409 pub stage: String,
9410
9411 #[serde(rename = "objectsDecommissioned")]
9412 pub items_decommissioned: usize,
9413 #[serde(rename = "objectsDecommissionedFailed")]
9414 pub items_decommission_failed: usize,
9415 #[serde(rename = "bytesDecommissioned")]
9416 pub bytes_done: usize,
9417 #[serde(rename = "bytesDecommissionedFailed")]
9418 pub bytes_failed: usize,
9419 #[serde(rename = "terminalReloadAttemptAt", with = "time::serde::rfc3339::option", default)]
9420 pub terminal_reload_attempt_at: Option<OffsetDateTime>,
9421 #[serde(rename = "terminalReloadFailures", default)]
9422 pub terminal_reload_failures: Vec<String>,
9423 #[serde(rename = "capacityReservation", default, skip_serializing_if = "Option::is_none")]
9424 pub capacity_reservation: Option<DecommissionCapacityReservation>,
9425 #[serde(rename = "capacityBlockedReason", default, skip_serializing_if = "Option::is_none")]
9426 pub capacity_blocked_reason: Option<String>,
9427 #[serde(skip)]
9428 pub unresolved_entries: Vec<DecommissionUnresolvedEntry>,
9429 #[serde(skip)]
9430 pub progress_save_item_baseline: usize,
9431 #[serde(skip)]
9432 pub progress_save_retry_after: Option<OffsetDateTime>,
9433 #[serde(skip)]
9434 pub progress_save_last_at: Option<OffsetDateTime>,
9435}
9436
9437#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9438struct DecommissionProgressCheckpoint {
9439 start_time: Option<OffsetDateTime>,
9440 queued: bool,
9441 counted_items: usize,
9442 checkpoint_at: OffsetDateTime,
9443 capacity_operation_id: Option<uuid::Uuid>,
9444 capacity_owner_nonce: Option<uuid::Uuid>,
9445 capacity_lease_expires_at: Option<OffsetDateTime>,
9446}
9447
9448impl PoolDecommissionInfo {
9449 pub fn has_decommission_state(&self) -> bool {
9450 self.complete
9451 || self.failed
9452 || self.canceled
9453 || self.queued
9454 || self.start_time.is_some()
9455 || self.start_size > 0
9456 || !self.queued_buckets.is_empty()
9457 || !self.decommissioned_buckets.is_empty()
9458 || !self.bucket.is_empty()
9459 || !self.prefix.is_empty()
9460 || !self.object.is_empty()
9461 || !self.stage.is_empty()
9462 || self.items_decommissioned > 0
9463 || self.items_decommission_failed > 0
9464 || self.bytes_done > 0
9465 || self.bytes_failed > 0
9466 || self.terminal_reload_attempt_at.is_some()
9467 || !self.terminal_reload_failures.is_empty()
9468 || self.capacity_reservation.is_some()
9469 || self.capacity_blocked_reason.is_some()
9470 || !self.unresolved_entries.is_empty()
9471 }
9472
9473 fn counted_items(&self) -> usize {
9474 self.items_decommissioned.saturating_add(self.items_decommission_failed)
9475 }
9476
9477 fn items_since_last_progress_save(&self) -> usize {
9478 self.counted_items().saturating_sub(self.progress_save_item_baseline)
9479 }
9480
9481 fn mark_progress_saved(&mut self) {
9482 self.progress_save_item_baseline = self.counted_items();
9483 self.progress_save_retry_after = None;
9484 }
9485
9486 pub fn bucket_push(&mut self, bucket: &DecomBucketInfo) {
9487 let bucket_key = bucket.to_string();
9488 if self.is_bucket_decommissioned(&bucket_key) {
9489 return;
9490 }
9491
9492 for b in self.queued_buckets.iter() {
9493 if b == &bucket_key {
9494 return;
9495 }
9496 }
9497
9498 self.queued_buckets.push(bucket_key);
9499
9500 self.bucket = bucket.name.clone();
9501 self.prefix = bucket.prefix.clone();
9502 }
9503 pub fn is_bucket_decommissioned(&self, bucket: &String) -> bool {
9504 for b in self.decommissioned_buckets.iter() {
9505 if b == bucket {
9506 return true;
9507 }
9508 }
9509 false
9510 }
9511 pub fn bucket_pop(&mut self, bucket: &String) -> bool {
9512 let mut found = None;
9513 for (i, b) in self.queued_buckets.iter().enumerate() {
9514 if b == bucket {
9515 found = Some(i);
9516 break;
9517 }
9518 }
9519
9520 if let Some(i) = found {
9521 if !self.is_bucket_decommissioned(bucket) {
9522 self.decommissioned_buckets.push(bucket.clone());
9523 }
9524 self.queued_buckets.remove(i);
9525 if &self.bucket == bucket {
9526 self.bucket = "".to_owned();
9527 self.prefix = "".to_owned();
9528 self.object = "".to_owned();
9529 }
9530
9531 return true;
9532 }
9533 false
9534 }
9535}
9536
9537#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9538pub struct PoolSpaceInfo {
9539 pub free: usize,
9540 pub total: usize,
9541 pub used: usize,
9542}
9543
9544#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9545pub(crate) struct DecommissionPoolCapacityInfo {
9546 pool_index: usize,
9547 space: PoolSpaceInfo,
9548 layout: DecommissionErasureLayout,
9549 physical_free: usize,
9550 physical_total: usize,
9551 physical_used: usize,
9552}
9553
9554impl DecommissionPoolCapacityInfo {
9555 #[cfg(any(test, feature = "test-util"))]
9556 pub(crate) fn for_test(
9557 pool_index: usize,
9558 layout: DecommissionErasureLayout,
9559 physical_free: usize,
9560 physical_total: usize,
9561 physical_used: usize,
9562 ) -> Self {
9563 Self {
9564 pool_index,
9565 space: PoolSpaceInfo {
9566 free: physical_free,
9567 total: physical_total,
9568 used: physical_used,
9569 },
9570 layout,
9571 physical_free,
9572 physical_total,
9573 physical_used,
9574 }
9575 }
9576}
9577
9578#[cfg(any(test, feature = "test-util"))]
9579type DecommissionCapacityInfoOverrides =
9580 std::sync::Mutex<HashMap<uuid::Uuid, std::collections::VecDeque<Vec<DecommissionPoolCapacityInfo>>>>;
9581
9582#[cfg(any(test, feature = "test-util"))]
9583static DECOMMISSION_CAPACITY_INFO_OVERRIDES: std::sync::OnceLock<DecommissionCapacityInfoOverrides> = std::sync::OnceLock::new();
9584
9585#[cfg(any(test, feature = "test-util"))]
9589pub(crate) fn set_decommission_capacity_info_overrides_for_test(
9590 store_id: uuid::Uuid,
9591 snapshots: Vec<Vec<DecommissionPoolCapacityInfo>>,
9592) {
9593 DECOMMISSION_CAPACITY_INFO_OVERRIDES
9594 .get_or_init(|| std::sync::Mutex::new(HashMap::new()))
9595 .lock()
9596 .expect("decommission capacity info override should not be poisoned")
9597 .insert(store_id, snapshots.into());
9598}
9599
9600#[cfg(any(test, feature = "test-util"))]
9601fn take_decommission_capacity_info_override_for_test(store_id: uuid::Uuid) -> Option<Vec<DecommissionPoolCapacityInfo>> {
9602 let mut overrides = DECOMMISSION_CAPACITY_INFO_OVERRIDES
9603 .get_or_init(|| std::sync::Mutex::new(HashMap::new()))
9604 .lock()
9605 .expect("decommission capacity info override should not be poisoned");
9606 let queue = overrides.get_mut(&store_id)?;
9607 if queue.len() > 1 {
9608 queue.pop_front()
9609 } else {
9610 queue.front().cloned()
9614 }
9615}
9616
9617#[cfg(test)]
9618struct DecommissionCapacityLockOrderBarrierState {
9619 owner_store_id: uuid::Uuid,
9620 external_store_id: uuid::Uuid,
9621 owner_arrived: tokio::sync::Notify,
9622 owner_release: tokio::sync::Notify,
9623 owner_pause_enabled: AtomicBool,
9624 external_capacity_released: tokio::sync::Notify,
9625 external_object_capacity_probe_acquired: tokio::sync::Notify,
9626 external_object_capacity_probe_release: tokio::sync::Notify,
9627 external_object_capacity_probe_paused: AtomicBool,
9628 external_object_commit_phase_started: tokio::sync::Notify,
9629 external_object_commit_phase_release: tokio::sync::Notify,
9630 external_object_commit_phase_paused: AtomicBool,
9631 external_heal_operation_started: tokio::sync::Notify,
9632 external_heal_target_lock_attempted: tokio::sync::Notify,
9633 target_gate_retry_entered: tokio::sync::Notify,
9634 target_gate_retry_entries: AtomicUsize,
9635 target_gate_exact_reloads: AtomicUsize,
9636 target_gate_acquire_pause_target: AtomicUsize,
9637 target_gate_acquire_entered: tokio::sync::Notify,
9638 target_gate_acquire_release: tokio::sync::Notify,
9639 cancel_before_start_entered: tokio::sync::Notify,
9640 cancel_before_start_release: tokio::sync::Notify,
9641 cancel_before_start_paused: AtomicBool,
9642 cancel_target_timeout_entered: tokio::sync::Notify,
9643 cancel_target_timeout_release: tokio::sync::Notify,
9644 cancel_target_timeout_paused: AtomicBool,
9645 cancel_target_timeouts: AtomicUsize,
9646}
9647
9648#[cfg(test)]
9649pub(crate) struct DecommissionCapacityLockOrderBarrier {
9650 state: Arc<DecommissionCapacityLockOrderBarrierState>,
9651}
9652
9653#[cfg(test)]
9654static DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER: std::sync::OnceLock<
9655 std::sync::Mutex<Option<Arc<DecommissionCapacityLockOrderBarrierState>>>,
9656> = std::sync::OnceLock::new();
9657
9658#[cfg(test)]
9659impl DecommissionCapacityLockOrderBarrier {
9660 pub(crate) fn install(owner_store_id: uuid::Uuid, external_store_id: uuid::Uuid) -> Self {
9661 let state = Arc::new(DecommissionCapacityLockOrderBarrierState {
9662 owner_store_id,
9663 external_store_id,
9664 owner_arrived: tokio::sync::Notify::new(),
9665 owner_release: tokio::sync::Notify::new(),
9666 owner_pause_enabled: AtomicBool::new(true),
9667 external_capacity_released: tokio::sync::Notify::new(),
9668 external_object_capacity_probe_acquired: tokio::sync::Notify::new(),
9669 external_object_capacity_probe_release: tokio::sync::Notify::new(),
9670 external_object_capacity_probe_paused: AtomicBool::new(false),
9671 external_object_commit_phase_started: tokio::sync::Notify::new(),
9672 external_object_commit_phase_release: tokio::sync::Notify::new(),
9673 external_object_commit_phase_paused: AtomicBool::new(false),
9674 external_heal_operation_started: tokio::sync::Notify::new(),
9675 external_heal_target_lock_attempted: tokio::sync::Notify::new(),
9676 target_gate_retry_entered: tokio::sync::Notify::new(),
9677 target_gate_retry_entries: AtomicUsize::new(0),
9678 target_gate_exact_reloads: AtomicUsize::new(0),
9679 target_gate_acquire_pause_target: AtomicUsize::new(usize::MAX),
9680 target_gate_acquire_entered: tokio::sync::Notify::new(),
9681 target_gate_acquire_release: tokio::sync::Notify::new(),
9682 cancel_before_start_entered: tokio::sync::Notify::new(),
9683 cancel_before_start_release: tokio::sync::Notify::new(),
9684 cancel_before_start_paused: AtomicBool::new(false),
9685 cancel_target_timeout_entered: tokio::sync::Notify::new(),
9686 cancel_target_timeout_release: tokio::sync::Notify::new(),
9687 cancel_target_timeout_paused: AtomicBool::new(false),
9688 cancel_target_timeouts: AtomicUsize::new(0),
9689 });
9690 let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9691 .get_or_init(|| std::sync::Mutex::new(None))
9692 .lock()
9693 .expect("decommission capacity lock-order barrier should not be poisoned");
9694 assert!(slot.is_none(), "decommission capacity lock-order barrier must be unique");
9695 *slot = Some(Arc::clone(&state));
9696 Self { state }
9697 }
9698
9699 #[cfg(feature = "test-util")]
9700 pub(crate) async fn wait_until_owner_paused(&self) {
9701 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.owner_arrived.notified())
9702 .await
9703 .expect("owned capacity mutation should reach admission before its metadata write");
9704 }
9705
9706 pub(crate) async fn wait_until_external_capacity_released(&self) {
9707 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.external_capacity_released.notified())
9708 .await
9709 .expect("external mutation should release capacity before waiting for the object namespace");
9710 }
9711
9712 #[cfg(feature = "test-util")]
9713 pub(crate) async fn wait_until_external_object_capacity_probe_acquired(&self) {
9714 tokio::time::timeout(
9715 std::time::Duration::from_secs(30),
9716 self.state.external_object_capacity_probe_acquired.notified(),
9717 )
9718 .await
9719 .expect("external object mutation should acquire its no-active capacity probe");
9720 }
9721
9722 #[cfg(feature = "test-util")]
9723 pub(crate) async fn wait_until_external_object_commit_phase_started(&self) {
9724 tokio::time::timeout(
9725 std::time::Duration::from_secs(30),
9726 self.state.external_object_commit_phase_started.notified(),
9727 )
9728 .await
9729 .expect("external object mutation should reach its staged commit phase before waiting for the namespace");
9730 }
9731
9732 pub(crate) async fn wait_until_external_heal_operation_started(&self) {
9733 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.external_heal_operation_started.notified())
9734 .await
9735 .expect("external heal should reach the target operation after capacity admission");
9736 }
9737
9738 pub(crate) async fn wait_until_external_heal_target_lock_attempted(&self) {
9739 tokio::time::timeout(
9740 std::time::Duration::from_secs(30),
9741 self.state.external_heal_target_lock_attempted.notified(),
9742 )
9743 .await
9744 .expect("external heal should attempt the target namespace lock after capacity admission");
9745 }
9746
9747 #[cfg(feature = "test-util")]
9748 pub(crate) async fn wait_until_target_gate_retry(&self) {
9749 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.target_gate_retry_entered.notified())
9750 .await
9751 .expect("decommission entry should observe target gate contention");
9752 }
9753
9754 #[cfg(feature = "test-util")]
9755 pub(crate) async fn wait_until_target_gate_retries(&self, expected: usize) {
9756 tokio::time::timeout(std::time::Duration::from_secs(30), async {
9757 loop {
9758 let notified = self.state.target_gate_retry_entered.notified();
9759 if self.state.target_gate_retry_entries.load(Ordering::Acquire) >= expected {
9760 return;
9761 }
9762 notified.await;
9763 }
9764 })
9765 .await
9766 .expect("decommission entries should observe target gate contention");
9767 }
9768
9769 #[cfg(feature = "test-util")]
9770 pub(crate) fn target_gate_exact_reloads(&self) -> usize {
9771 self.state.target_gate_exact_reloads.load(Ordering::Acquire)
9772 }
9773
9774 #[cfg(feature = "test-util")]
9775 pub(crate) fn pause_target_gate_acquire(&self, target_pool_index: usize) {
9776 self.state
9777 .target_gate_acquire_pause_target
9778 .store(target_pool_index, Ordering::Release);
9779 }
9780
9781 #[cfg(feature = "test-util")]
9782 pub(crate) async fn wait_until_target_gate_acquire_paused(&self) {
9783 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.target_gate_acquire_entered.notified())
9784 .await
9785 .expect("decommission capacity mutation should pause before acquiring its target gate");
9786 }
9787
9788 #[cfg(feature = "test-util")]
9789 pub(crate) fn release_target_gate_acquire(&self) {
9790 self.state
9791 .target_gate_acquire_pause_target
9792 .store(usize::MAX, Ordering::Release);
9793 self.state.target_gate_acquire_release.notify_one();
9794 }
9795
9796 pub(crate) fn pause_cancel_before_start(&self) {
9797 self.state.cancel_before_start_paused.store(true, Ordering::Release);
9798 }
9799
9800 pub(crate) async fn wait_until_cancel_before_start(&self) {
9801 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.cancel_before_start_entered.notified())
9802 .await
9803 .expect("decommission cancel should pause before acquiring the start gate");
9804 }
9805
9806 pub(crate) fn release_cancel_before_start(&self) {
9807 self.state.cancel_before_start_release.notify_one();
9808 }
9809
9810 fn pause_cancel_target_timeout(&self) {
9811 self.state.cancel_target_timeout_paused.store(true, Ordering::Release);
9812 }
9813
9814 async fn wait_until_cancel_target_timeout(&self) {
9815 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.cancel_target_timeout_entered.notified())
9816 .await
9817 .expect("cancel should observe contention on its target capacity fence");
9818 }
9819
9820 fn release_cancel_target_timeout(&self) {
9821 self.state.cancel_target_timeout_paused.store(false, Ordering::Release);
9822 self.state.cancel_target_timeout_release.notify_one();
9823 }
9824
9825 #[cfg(feature = "test-util")]
9826 pub(crate) fn release_owner(&self) {
9827 self.state.owner_release.notify_one();
9828 self.state.target_gate_acquire_release.notify_one();
9829 }
9830
9831 #[cfg(feature = "test-util")]
9832 pub(crate) fn disable_owner_pause(&self) {
9833 self.state.owner_pause_enabled.store(false, Ordering::Release);
9834 self.state.owner_release.notify_waiters();
9835 }
9836
9837 #[cfg(feature = "test-util")]
9838 pub(crate) fn pause_external_object_commit_phase(&self) {
9839 self.state.external_object_commit_phase_paused.store(true, Ordering::Release);
9840 }
9841
9842 #[cfg(feature = "test-util")]
9843 pub(crate) fn release_external_object_commit_phase(&self) {
9844 self.state.external_object_commit_phase_release.notify_one();
9845 }
9846
9847 #[cfg(feature = "test-util")]
9848 pub(crate) fn pause_external_object_capacity_probe(&self) {
9849 self.state
9850 .external_object_capacity_probe_paused
9851 .store(true, Ordering::Release);
9852 }
9853
9854 #[cfg(feature = "test-util")]
9855 pub(crate) fn release_external_object_capacity_probe(&self) {
9856 self.state.external_object_capacity_probe_release.notify_one();
9857 }
9858}
9859
9860#[cfg(test)]
9861impl Drop for DecommissionCapacityLockOrderBarrier {
9862 fn drop(&mut self) {
9863 self.state.owner_release.notify_one();
9864 self.state.external_object_capacity_probe_release.notify_one();
9865 self.state.external_object_commit_phase_release.notify_one();
9866 self.state.cancel_before_start_release.notify_one();
9867 self.state.cancel_target_timeout_release.notify_one();
9868 let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9869 .get_or_init(|| std::sync::Mutex::new(None))
9870 .lock()
9871 .expect("decommission capacity lock-order barrier should not be poisoned");
9872 if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
9873 *slot = None;
9874 }
9875 }
9876}
9877
9878#[cfg(test)]
9879async fn pause_decommission_capacity_before_owner_write(store_id: uuid::Uuid) {
9880 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9881 .get_or_init(|| std::sync::Mutex::new(None))
9882 .lock()
9883 .expect("decommission capacity lock-order barrier should not be poisoned")
9884 .as_ref()
9885 .filter(|state| state.owner_store_id == store_id && state.owner_pause_enabled.load(Ordering::Acquire))
9886 .cloned();
9887 if let Some(barrier) = barrier {
9888 barrier.owner_arrived.notify_one();
9889 barrier.owner_release.notified().await;
9890 }
9891}
9892
9893#[cfg(test)]
9894async fn pause_decommission_capacity_before_target_gate_acquire(store_id: uuid::Uuid, target_pool_index: usize) {
9895 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9896 .get_or_init(|| std::sync::Mutex::new(None))
9897 .lock()
9898 .expect("decommission capacity lock-order barrier should not be poisoned")
9899 .as_ref()
9900 .filter(|state| {
9901 state.owner_store_id == store_id
9902 && state.target_gate_acquire_pause_target.load(Ordering::Acquire) == target_pool_index
9903 })
9904 .cloned();
9905 if let Some(barrier) = barrier {
9906 barrier.target_gate_acquire_entered.notify_one();
9907 barrier.target_gate_acquire_release.notified().await;
9908 }
9909}
9910
9911#[cfg(test)]
9912async fn pause_decommission_cancel_before_start_gate(store_id: uuid::Uuid) {
9913 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9914 .get_or_init(|| std::sync::Mutex::new(None))
9915 .lock()
9916 .expect("decommission capacity lock-order barrier should not be poisoned")
9917 .as_ref()
9918 .filter(|state| state.owner_store_id == store_id && state.cancel_before_start_paused.load(Ordering::Acquire))
9919 .cloned();
9920 if let Some(barrier) = barrier {
9921 barrier.cancel_before_start_entered.notify_one();
9922 barrier.cancel_before_start_release.notified().await;
9923 }
9924}
9925
9926#[cfg(test)]
9927async fn pause_decommission_cancel_target_timeout(store_id: uuid::Uuid) {
9928 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9929 .get_or_init(|| std::sync::Mutex::new(None))
9930 .lock()
9931 .expect("decommission capacity lock-order barrier should not be poisoned")
9932 .as_ref()
9933 .filter(|state| state.owner_store_id == store_id)
9934 .cloned();
9935 if let Some(barrier) = barrier {
9936 barrier.cancel_target_timeouts.fetch_add(1, Ordering::AcqRel);
9937 barrier.cancel_target_timeout_entered.notify_one();
9938 if barrier.cancel_target_timeout_paused.load(Ordering::Acquire) {
9939 barrier.cancel_target_timeout_release.notified().await;
9940 }
9941 }
9942}
9943
9944#[cfg(test)]
9945fn notify_decommission_target_gate_retry(store_id: uuid::Uuid) {
9946 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9947 .get_or_init(|| std::sync::Mutex::new(None))
9948 .lock()
9949 .expect("decommission capacity lock-order barrier should not be poisoned")
9950 .as_ref()
9951 .filter(|state| state.owner_store_id == store_id)
9952 .cloned();
9953 if let Some(barrier) = barrier {
9954 barrier.target_gate_retry_entries.fetch_add(1, Ordering::AcqRel);
9955 barrier.target_gate_retry_entered.notify_one();
9956 }
9957}
9958
9959#[cfg(test)]
9960fn notify_decommission_target_gate_exact_reload(store_id: uuid::Uuid) {
9961 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9962 .get_or_init(|| std::sync::Mutex::new(None))
9963 .lock()
9964 .expect("decommission capacity lock-order barrier should not be poisoned")
9965 .as_ref()
9966 .filter(|state| state.owner_store_id == store_id)
9967 .cloned();
9968 if let Some(barrier) = barrier {
9969 barrier.target_gate_exact_reloads.fetch_add(1, Ordering::AcqRel);
9970 }
9971}
9972
9973#[cfg(test)]
9974pub(crate) fn notify_decommission_external_object_capacity_released(store_id: uuid::Uuid) {
9975 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9976 .get_or_init(|| std::sync::Mutex::new(None))
9977 .lock()
9978 .expect("decommission capacity lock-order barrier should not be poisoned")
9979 .as_ref()
9980 .filter(|state| state.external_store_id == store_id)
9981 .cloned();
9982 if let Some(barrier) = barrier {
9983 barrier.external_capacity_released.notify_one();
9984 }
9985}
9986
9987#[cfg(test)]
9988pub(crate) fn notify_decommission_external_object_capacity_probe_acquired(store_id: uuid::Uuid) {
9989 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9990 .get_or_init(|| std::sync::Mutex::new(None))
9991 .lock()
9992 .expect("decommission capacity lock-order barrier should not be poisoned")
9993 .as_ref()
9994 .filter(|state| state.external_store_id == store_id)
9995 .cloned();
9996 if let Some(barrier) = barrier {
9997 barrier.external_object_capacity_probe_acquired.notify_one();
9998 }
9999}
10000
10001#[cfg(test)]
10002pub(crate) async fn wait_for_decommission_external_object_capacity_probe_release(store_id: uuid::Uuid) {
10003 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10004 .get_or_init(|| std::sync::Mutex::new(None))
10005 .lock()
10006 .expect("decommission capacity lock-order barrier should not be poisoned")
10007 .as_ref()
10008 .filter(|state| state.external_store_id == store_id)
10009 .cloned();
10010 if let Some(barrier) = barrier
10011 && barrier.external_object_capacity_probe_paused.load(Ordering::Acquire)
10012 {
10013 barrier.external_object_capacity_probe_release.notified().await;
10014 }
10015}
10016
10017#[cfg(test)]
10018pub(crate) fn notify_decommission_external_object_commit_phase_started(store_id: uuid::Uuid) {
10019 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10020 .get_or_init(|| std::sync::Mutex::new(None))
10021 .lock()
10022 .expect("decommission capacity lock-order barrier should not be poisoned")
10023 .as_ref()
10024 .filter(|state| state.external_store_id == store_id)
10025 .cloned();
10026 if let Some(barrier) = barrier {
10027 barrier.external_object_commit_phase_started.notify_one();
10028 }
10029}
10030
10031#[cfg(test)]
10032pub(crate) async fn wait_for_decommission_external_object_commit_phase_release(store_id: uuid::Uuid) {
10033 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10034 .get_or_init(|| std::sync::Mutex::new(None))
10035 .lock()
10036 .expect("decommission capacity lock-order barrier should not be poisoned")
10037 .as_ref()
10038 .filter(|state| state.external_store_id == store_id)
10039 .cloned();
10040 if let Some(barrier) = barrier
10041 && barrier.external_object_commit_phase_paused.load(Ordering::Acquire)
10042 {
10043 barrier.external_object_commit_phase_release.notified().await;
10044 }
10045}
10046
10047#[cfg(test)]
10048pub(crate) fn notify_decommission_external_heal_operation_started(store_id: uuid::Uuid) {
10049 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10050 .get_or_init(|| std::sync::Mutex::new(None))
10051 .lock()
10052 .expect("decommission capacity lock-order barrier should not be poisoned")
10053 .as_ref()
10054 .filter(|state| state.external_store_id == store_id)
10055 .cloned();
10056 if let Some(barrier) = barrier {
10057 barrier.external_heal_operation_started.notify_one();
10058 }
10059}
10060
10061#[cfg(test)]
10062pub(crate) fn notify_decommission_external_heal_target_lock_attempted() {
10063 let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10064 .get_or_init(|| std::sync::Mutex::new(None))
10065 .lock()
10066 .expect("decommission capacity lock-order barrier should not be poisoned")
10067 .as_ref()
10068 .cloned();
10069 if let Some(barrier) = barrier {
10070 barrier.external_heal_target_lock_attempted.notify_one();
10071 }
10072}
10073
10074#[derive(Debug, Default, Clone)]
10075pub struct DecomBucketInfo {
10076 pub name: String,
10077 pub prefix: String,
10078}
10079
10080impl Display for DecomBucketInfo {
10081 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10082 write!(
10083 f,
10084 "{}",
10085 path_join(&[PathBuf::from(self.name.clone()), PathBuf::from(self.prefix.clone())]).to_string_lossy()
10086 )
10087 }
10088}
10089
10090#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10091enum DecommissionFinalState {
10092 Complete,
10093 Failed,
10094}
10095
10096fn determine_decommission_final_state(items_failed: usize, was_cancelled: bool) -> DecommissionFinalState {
10097 if items_failed > 0 || was_cancelled {
10098 DecommissionFinalState::Failed
10099 } else {
10100 DecommissionFinalState::Complete
10101 }
10102}
10103
10104fn decommission_remaining_version_count(versions: &[rustfs_filemeta::FileInfo], expired: usize) -> usize {
10105 versions
10106 .iter()
10107 .filter(|version| !version.tier_free_version())
10108 .count()
10109 .saturating_sub(expired)
10110}
10111
10112fn should_skip_decommission_delete_marker(
10113 version: &rustfs_filemeta::FileInfo,
10114 remaining_versions: usize,
10115 replication_configured: bool,
10116) -> bool {
10117 version.deleted && remaining_versions == 1 && !replication_configured
10120}
10121
10122fn decommission_delete_marker_opts(
10123 version: &rustfs_filemeta::FileInfo,
10124 version_id: Option<String>,
10125 src_pool_idx: usize,
10126 expected_bucket_incarnation_id: Option<uuid::Uuid>,
10127) -> ObjectOptions {
10128 let version_suspended = version.version_id.is_none() && version_id.is_none();
10129 ObjectOptions {
10130 versioned: !version_suspended,
10131 version_suspended,
10132 version_id: version_id.or_else(|| version_suspended.then(|| uuid::Uuid::nil().to_string())),
10133 mod_time: version.mod_time,
10134 src_pool_idx,
10135 data_movement: true,
10136 delete_marker: true,
10137 skip_decommissioned: true,
10138 expected_bucket_incarnation_id,
10139 delete_replication: version
10140 .replication_state_internal
10141 .as_ref()
10142 .map(replication_state_from_filemeta),
10143 ..Default::default()
10144 }
10145}
10146
10147fn decommission_object_migration_read_opts(version_id: Option<String>) -> ObjectOptions {
10148 ObjectOptions {
10149 version_id,
10150 no_lock: true,
10151 data_movement: true,
10152 raw_data_movement_read: true,
10153 skip_decommissioned: true,
10154 skip_rebalancing: true,
10155 ..Default::default()
10156 }
10157}
10158
10159fn decommission_remote_tiered_opts(
10160 version: &rustfs_filemeta::FileInfo,
10161 version_id: Option<String>,
10162 src_pool_idx: usize,
10163 expected_bucket_incarnation_id: Option<uuid::Uuid>,
10164) -> ObjectOptions {
10165 ObjectOptions {
10166 versioned: version_id.is_some(),
10167 version_id,
10168 mod_time: version.mod_time,
10169 user_defined: version.metadata.clone(),
10170 src_pool_idx,
10171 data_movement: true,
10172 incl_free_versions: version.tier_free_version(),
10173 include_part_checksums: true,
10174 http_preconditions: Some(data_movement::data_movement_target_precondition()),
10175 expected_bucket_incarnation_id,
10176 ..Default::default()
10177 }
10178}
10179
10180fn decommission_capacity_version_mutation_id(
10181 owner: DecommissionCapacityOwner,
10182 bucket: &str,
10183 version: &rustfs_filemeta::FileInfo,
10184) -> uuid::Uuid {
10185 let version_id = if version.deleted && version.version_id.is_none() {
10186 Some(uuid::Uuid::nil().to_string())
10187 } else {
10188 version.version_id.map(|version_id| version_id.to_string())
10189 };
10190 decommission_capacity_mutation_id(owner, bucket, &version.name, version_id.as_deref(), version.deleted, version.mod_time)
10191}
10192
10193fn decommission_capacity_owned_opts(mut opts: ObjectOptions, capacity_owner: Option<DecommissionCapacityOwner>) -> ObjectOptions {
10194 if let Some(capacity_owner) = capacity_owner {
10195 capacity_owner.apply_to(&mut opts);
10196 }
10197 opts
10198}
10199
10200fn lifecycle_action_removes_data_movement_version(action: IlmAction) -> bool {
10201 matches!(
10202 action,
10203 IlmAction::DeleteVersionAction | IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction
10204 )
10205}
10206
10207fn lifecycle_action_skips_heal_version(action: IlmAction) -> bool {
10208 action.delete()
10209}
10210
10211#[cfg(test)]
10212struct LifecycleDataMovementMutationBarrierState {
10213 bucket: String,
10214 object: String,
10215 arrived: tokio::sync::Notify,
10216 release: tokio::sync::Notify,
10217}
10218
10219#[cfg(test)]
10220pub(crate) struct LifecycleDataMovementMutationBarrier {
10221 state: Arc<LifecycleDataMovementMutationBarrierState>,
10222}
10223
10224#[cfg(test)]
10225static LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER: std::sync::OnceLock<
10226 std::sync::Mutex<Option<Arc<LifecycleDataMovementMutationBarrierState>>>,
10227> = std::sync::OnceLock::new();
10228
10229#[cfg(test)]
10230impl LifecycleDataMovementMutationBarrier {
10231 pub(crate) fn install(bucket: &str, object: &str) -> Self {
10232 let state = Arc::new(LifecycleDataMovementMutationBarrierState {
10233 bucket: bucket.to_string(),
10234 object: object.to_string(),
10235 arrived: tokio::sync::Notify::new(),
10236 release: tokio::sync::Notify::new(),
10237 });
10238 let mut slot = LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER
10239 .get_or_init(|| std::sync::Mutex::new(None))
10240 .lock()
10241 .expect("lifecycle data movement mutation barrier should not be poisoned");
10242 assert!(slot.is_none(), "lifecycle data movement mutation barrier must be unique");
10243 *slot = Some(Arc::clone(&state));
10244 Self { state }
10245 }
10246
10247 pub(crate) async fn wait_until_paused(&self) {
10248 tokio::time::timeout(std::time::Duration::from_secs(30), self.state.arrived.notified())
10249 .await
10250 .expect("lifecycle data movement should reach its mutation boundary");
10251 }
10252
10253 pub(crate) fn release(&self) {
10254 self.state.release.notify_one();
10255 }
10256}
10257
10258#[cfg(test)]
10259impl Drop for LifecycleDataMovementMutationBarrier {
10260 fn drop(&mut self) {
10261 self.state.release.notify_one();
10262 let mut slot = LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER
10263 .get_or_init(|| std::sync::Mutex::new(None))
10264 .lock()
10265 .expect("lifecycle data movement mutation barrier should not be poisoned");
10266 if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
10267 *slot = None;
10268 }
10269 }
10270}
10271
10272#[cfg(test)]
10273async fn pause_lifecycle_data_movement_mutation(bucket: &str, object: &str, has_run_fence_signal: bool) {
10274 if !has_run_fence_signal {
10275 return;
10276 }
10277 let barrier = LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER
10278 .get_or_init(|| std::sync::Mutex::new(None))
10279 .lock()
10280 .expect("lifecycle data movement mutation barrier should not be poisoned")
10281 .as_ref()
10282 .filter(|barrier| barrier.bucket == bucket && barrier.object == object)
10283 .cloned();
10284 if let Some(barrier) = barrier {
10285 barrier.arrived.notify_one();
10286 barrier.release.notified().await;
10287 }
10288}
10289
10290fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_actions: bool, applied: bool) -> Result<bool> {
10291 if !apply_actions || applied {
10292 return Ok(true);
10293 }
10294
10295 Err(Error::other(format!(
10296 "failed to apply lifecycle expiry action {action:?} during data movement"
10297 )))
10298}
10299
10300#[allow(clippy::too_many_arguments)]
10301pub(crate) async fn should_skip_lifecycle_for_data_movement(
10302 store: Arc<ECStore>,
10303 bucket: &str,
10304 version: &rustfs_filemeta::FileInfo,
10305 lifecycle_config: Option<&BucketLifecycleConfiguration>,
10306 object_lock_config: Option<&ObjectLockConfiguration>,
10307 apply_actions: bool,
10308 event_source: &LcEventSrc,
10309 lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
10310) -> Result<bool> {
10311 let Some(lifecycle_config) = lifecycle_config else {
10312 return Ok(false);
10313 };
10314
10315 let versioned = match BucketVersioningSys::get_in(&store.ctx, bucket).await {
10316 Ok(config) => config.prefix_enabled(&version.name),
10317 Err(err) => {
10318 warn!(
10319 event = EVENT_DECOMMISSION_ENTRY,
10320 component = LOG_COMPONENT_ECSTORE,
10321 subsystem = LOG_SUBSYSTEM_POOLS,
10322 state = "versioning_config_unavailable",
10323 bucket = %bucket,
10324 error = %err,
10325 "Decommission lifecycle versioning config unavailable; treating object as unversioned"
10326 );
10327 false
10328 }
10329 };
10330 let object_info = crate::object_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned);
10331 let event = eval_action_from_lifecycle(lifecycle_config, object_lock_config, &object_info).await;
10332
10333 match event.action {
10334 IlmAction::DeleteRestoredAction | IlmAction::DeleteRestoredVersionAction => {
10335 if apply_actions && object_info.is_remote() {
10336 let Ok(bucket_incarnation_id) = store.bucket_incarnation_id_from_disk(bucket).await else {
10337 return Ok(false);
10338 };
10339 let _ = match lock_lost_signal {
10340 Some(signal) => {
10341 apply_expiry_rule_for_data_movement(store, &event, event_source, &object_info, Some(signal)).await
10342 }
10343 None => {
10344 apply_expiry_on_transitioned_object(store, &object_info, &event, event_source, bucket_incarnation_id)
10345 .await
10346 }
10347 };
10348 }
10349 Ok(false)
10350 }
10351 action if lifecycle_action_removes_data_movement_version(action) => {
10352 #[cfg(test)]
10353 pause_lifecycle_data_movement_mutation(bucket, &version.name, lock_lost_signal.is_some()).await;
10354 if lifecycle_delete_all_versions_blocked_by_replication(store.clone(), bucket, &object_info.name, action).await? {
10355 return Ok(false);
10356 }
10357 let applied = !apply_actions
10358 || match lock_lost_signal {
10359 Some(signal) => {
10360 apply_expiry_rule_for_data_movement(store, &event, event_source, &object_info, Some(signal)).await
10361 }
10362 None => apply_expiry_rule_in(store, &event, event_source, &object_info).await,
10363 };
10364 resolve_data_movement_lifecycle_expiry_result(action, apply_actions, applied)
10365 }
10366 _ => Ok(false),
10367 }
10368}
10369
10370pub struct HealLifecycleExpiryContext {
10371 configs: LifecycleExpiryConfigs,
10372}
10373
10374impl ECStore {
10375 pub async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
10376 if bucket == RUSTFS_META_BUCKET {
10377 return Ok(None);
10378 }
10379
10380 let configs = get_expiry_configs(self, bucket).await?;
10381 if configs.lifecycle.is_none() {
10382 return Ok(None);
10383 }
10384
10385 Ok(Some(HealLifecycleExpiryContext { configs }))
10386 }
10387
10388 pub async fn enqueue_heal_lifecycle_expiry(
10389 self: &Arc<Self>,
10390 context: &HealLifecycleExpiryContext,
10391 bucket: &str,
10392 object: &str,
10393 version_id: Option<&str>,
10394 object_info: Option<&crate::object_api::ObjectInfo>,
10395 ) -> Result<bool> {
10396 let Some(lifecycle_config) = context.configs.lifecycle.as_ref() else {
10397 return Ok(false);
10398 };
10399
10400 let object_info = if let Some(object_info) = object_info {
10401 if object_info.bucket != bucket || object_info.name != object {
10402 return Ok(false);
10403 }
10404 let snapshot_version_id = object_info
10405 .version_id
10406 .filter(|version_id| !version_id.is_nil())
10407 .map(|version_id| version_id.to_string());
10408 if snapshot_version_id.as_deref() != version_id {
10409 return Ok(false);
10410 }
10411 object_info.clone()
10412 } else {
10413 match self
10414 .get_object_info(
10415 bucket,
10416 object,
10417 &ObjectOptions {
10418 version_id: version_id.map(str::to_string),
10419 versioned: version_id.is_some(),
10420 expected_bucket_incarnation_id: Some(context.configs.bucket_incarnation_id),
10421 ..Default::default()
10422 },
10423 )
10424 .await
10425 {
10426 Ok(object_info) => object_info,
10427 Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => return Ok(false),
10428 Err(err) => return Err(err),
10429 }
10430 };
10431
10432 let event = eval_action_from_lifecycle(lifecycle_config, context.configs.object_lock.as_deref(), &object_info).await;
10433 if !lifecycle_action_skips_heal_version(event.action) {
10434 return Ok(false);
10435 }
10436
10437 if lifecycle_delete_all_versions_blocked_by_replication(self.clone(), bucket, &object_info.name, event.action).await? {
10438 return Ok(false);
10439 }
10440
10441 Ok(apply_expiry_rule_in(self.clone(), &event, &LcEventSrc::Scanner, &object_info).await)
10442 }
10443
10444 async fn acquire_pool_meta_write_guard(
10445 &self,
10446 write_state: &mut PoolMetaWriteState,
10447 operation: &str,
10448 ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> {
10449 self.acquire_pool_meta_write_guard_with_lock_error(write_state, operation, activation_pool_meta_lock_error)
10450 .await
10451 }
10452
10453 async fn acquire_pool_meta_write_guard_with_lock_error<F>(
10454 &self,
10455 write_state: &mut PoolMetaWriteState,
10456 operation: &str,
10457 map_lock_error: F,
10458 ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)>
10459 where
10460 F: FnOnce(rustfs_lock::LockError) -> Error,
10461 {
10462 write_state.ensure_write_safe(operation)?;
10463 load_pool_meta_identity_observing(self.pools.clone(), write_state).await?;
10464 let pool = self.pools.first().cloned().ok_or_else(|| {
10465 Error::InvalidArgument(
10466 operation.to_string(),
10467 "storage-pools".to_string(),
10468 "no storage pools available".to_string(),
10469 )
10470 })?;
10471 let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
10472 let pool_meta_guard = pool_meta_lock
10473 .get_write_lock(get_lock_acquire_timeout())
10474 .await
10475 .map_err(map_lock_error)?;
10476 let selection = load_pool_meta_replicas_observing(self.pools.clone(), true, write_state).await?;
10477 write_state.observe_replicas(selection.replica_state);
10478 write_state.ensure_write_safe(operation)?;
10479 Ok((pool_meta_guard, selection.meta))
10480 }
10481
10482 async fn acquire_pool_meta_read_guard(
10483 &self,
10484 write_state: &PoolMetaWriteState,
10485 operation: &str,
10486 ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> {
10487 write_state.ensure_write_safe(operation)?;
10488 let pool = self.pools.first().cloned().ok_or_else(|| {
10489 Error::InvalidArgument(
10490 operation.to_string(),
10491 "storage-pools".to_string(),
10492 "no storage pools available".to_string(),
10493 )
10494 })?;
10495 let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
10496 let pool_meta_guard = pool_meta_lock.get_read_lock(get_lock_acquire_timeout()).await?;
10497 let selection = load_pool_meta_replicas_for_read_probe(self.pools.clone(), true, write_state, operation).await?;
10498 Ok((pool_meta_guard, selection.meta))
10499 }
10500
10501 async fn acquire_decommission_capacity_target_guard(
10502 &self,
10503 target_pool_index: usize,
10504 ) -> Result<rustfs_lock::NamespaceLockGuard> {
10505 let pool = self.pools.first().cloned().ok_or_else(|| {
10509 Error::InvalidArgument(
10510 "decommission-capacity".to_string(),
10511 "storage-pools".to_string(),
10512 "no storage pools available".to_string(),
10513 )
10514 })?;
10515 let object = format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}");
10516 let target_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
10517 #[cfg(test)]
10518 pause_decommission_capacity_before_target_gate_acquire(self.id, target_pool_index).await;
10519 match target_lock
10520 .get_write_lock_quiet(DECOMMISSION_CAPACITY_TARGET_LOCK_TIMEOUT)
10521 .await
10522 {
10523 Ok(guard) => Ok(guard),
10524 Err(rustfs_lock::LockError::Timeout { .. } | rustfs_lock::LockError::AlreadyLocked { .. }) => {
10525 Err(decommission_capacity_blocked_error(format!(
10526 "{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}{target_pool_index}{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}"
10527 )))
10528 }
10529 Err(rustfs_lock::LockError::QuorumNotReached { required, achieved }) => Err(Error::NamespaceLockQuorumUnavailable {
10530 mode: "write",
10531 bucket: RUSTFS_META_BUCKET.to_string(),
10532 object,
10533 required,
10534 achieved,
10535 }),
10536 Err(err) => Err(Error::Lock(err)),
10537 }
10538 }
10539
10540 async fn acquire_decommission_capacity_owner_target_guard(
10541 &self,
10542 owner: DecommissionCapacityOwner,
10543 target_pool_index: usize,
10544 ) -> Result<Option<rustfs_lock::NamespaceLockGuard>> {
10545 let model_version = self
10546 .pool_meta
10547 .read()
10548 .await
10549 .pools
10550 .get(owner.source_pool_index)
10551 .and_then(|pool| pool.decommission.as_ref())
10552 .and_then(|info| info.capacity_reservation.as_ref())
10553 .filter(|reservation| {
10554 reservation.active()
10555 && reservation.source_pool_index == owner.source_pool_index
10556 && reservation.operation_id == owner.operation_id
10557 && reservation.generation == owner.generation
10558 })
10559 .map(|reservation| reservation.model_version)
10560 .ok_or_else(|| decommission_capacity_blocked_error("decommission capacity owner is stale"))?;
10561 match model_version {
10562 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION => Ok(None),
10563 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION => {
10564 discard_decommission_capacity_target_permit_except(self.id, owner, Some(target_pool_index));
10565 match take_decommission_capacity_target_permit(self.id, target_pool_index, owner) {
10566 Some(guard) => Ok(Some(guard)),
10567 None => self
10568 .acquire_decommission_capacity_target_guard(target_pool_index)
10569 .await
10570 .map(Some),
10571 }
10572 }
10573 version => Err(Error::DecommissionCapacity(format!(
10574 "decommission capacity owner uses unsupported lock model {version}"
10575 ))),
10576 }
10577 }
10578
10579 async fn acquire_decommission_capacity_terminal_guards(
10580 &self,
10581 source_pool_index: usize,
10582 plan: Option<&DecommissionCapacityTerminalFencePlan>,
10583 ) -> Result<Vec<rustfs_lock::NamespaceLockGuard>> {
10584 let Some(plan) = plan else {
10585 return Ok(Vec::new());
10586 };
10587 match plan.model_version {
10588 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION => return Ok(Vec::new()),
10589 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION => {}
10590 version => {
10591 return Err(Error::DecommissionCapacity(format!(
10592 "decommission terminal transition found unsupported capacity lock model {version}"
10593 )));
10594 }
10595 }
10596 let pool = self.pools.first().cloned().ok_or_else(|| {
10599 Error::InvalidArgument(
10600 "decommission-capacity".to_string(),
10601 "storage-pools".to_string(),
10602 "no storage pools available".to_string(),
10603 )
10604 })?;
10605 let mut attempt = 1;
10608 'acquire_targets: loop {
10609 let mut guards = Vec::with_capacity(plan.target_pool_indices.len());
10610 for &target_pool_index in &plan.target_pool_indices {
10611 let object = format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}");
10612 let target_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
10613 let started = std::time::Instant::now();
10614 match target_lock.get_write_lock(get_lock_acquire_timeout()).await {
10615 Ok(guard) => guards.push(guard),
10616 Err(err @ rustfs_lock::LockError::Timeout { .. }) => {
10617 drop(guards);
10620 #[cfg(test)]
10621 pause_decommission_cancel_target_timeout(self.id).await;
10622 if attempt >= DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS {
10623 return Err(Error::Lock(err));
10624 }
10625 warn!(
10626 event = EVENT_DECOMMISSION_STATE,
10627 component = LOG_COMPONENT_ECSTORE,
10628 subsystem = LOG_SUBSYSTEM_POOLS,
10629 state = "cancel_target_fence_retry",
10630 pool_index = source_pool_index,
10631 target_pool_index,
10632 operation_id = %plan.operation_id,
10633 generation = plan.generation,
10634 owner_nonce = %plan.owner_nonce,
10635 attempt,
10636 max_attempts = DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS,
10637 wait_ms = %started.elapsed().as_millis(),
10638 error = %err,
10639 "Decommission cancel will retry target capacity fencing"
10640 );
10641 tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await;
10642 let save_guard = self.pool_meta_save_gate.lock().await;
10643 let (_read_guard, snapshot) = self
10644 .acquire_pool_meta_read_guard(&save_guard, "decommission cancel fence retry failed")
10645 .await?;
10646 if decommission_capacity_terminal_fence_plan(&snapshot, source_pool_index)?.as_ref() != Some(plan) {
10647 return Err(decommission_capacity_blocked_error(
10648 "decommission capacity owner or target cohort changed while retrying terminal fences",
10649 ));
10650 }
10651 attempt += 1;
10652 continue 'acquire_targets;
10653 }
10654 Err(rustfs_lock::LockError::QuorumNotReached { required, achieved }) => {
10655 return Err(Error::NamespaceLockQuorumUnavailable {
10656 mode: "write",
10657 bucket: RUSTFS_META_BUCKET.to_string(),
10658 object,
10659 required,
10660 achieved,
10661 });
10662 }
10663 Err(err) => return Err(Error::Lock(err)),
10664 }
10665 }
10666 return Ok(guards);
10667 }
10668 }
10669
10670 pub(crate) async fn acquire_external_decommission_capacity_fence(
10671 &self,
10672 target_pool_indices: &[usize],
10673 admission: DecommissionCapacityAdmission,
10674 ) -> Result<rustfs_lock::NamespaceLockGuard> {
10675 Ok(self
10676 .acquire_external_decommission_capacity_fence_with_active_source(target_pool_indices, admission)
10677 .await?
10678 .0)
10679 }
10680
10681 pub(crate) async fn acquire_external_decommission_capacity_fence_with_active_source(
10682 &self,
10683 target_pool_indices: &[usize],
10684 admission: DecommissionCapacityAdmission,
10685 ) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
10686 let save_guard = self.pool_meta_save_gate.lock().await;
10687 let (pool_meta_guard, snapshot) = self
10688 .acquire_pool_meta_read_guard(&save_guard, "target capacity admission failed")
10689 .await?;
10690 for target_pool_index in target_pool_indices.iter().copied() {
10691 ensure_external_decommission_target_admission(&snapshot, target_pool_index, admission)?;
10692 }
10693 let has_active_source = pool_meta_has_active_decommission(&snapshot);
10694 drop(save_guard);
10695 Ok((pool_meta_guard, has_active_source))
10696 }
10697
10698 pub(crate) async fn acquire_pool_meta_object_heal_fence(
10703 &self,
10704 target_pool_indices: &[usize],
10705 ) -> Result<(rustfs_lock::NamespaceLockGuard, Vec<Result<()>>)> {
10706 let mut save_guard = self.pool_meta_save_gate.lock().await;
10707 let (pool_meta_guard, snapshot) = self
10708 .acquire_pool_meta_write_guard_with_lock_error(&mut save_guard, "pool metadata heal admission failed", Error::from)
10709 .await?;
10710 let admissions = target_pool_indices
10711 .iter()
10712 .copied()
10713 .map(|target_pool_index| {
10714 ensure_external_decommission_target_admission(&snapshot, target_pool_index, DecommissionCapacityAdmission::Heal)
10715 })
10716 .collect();
10717 drop(save_guard);
10718 Ok((pool_meta_guard, admissions))
10719 }
10720
10721 pub(crate) async fn acquire_decommission_capacity_release_fence_with_active_source(
10722 &self,
10723 ) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
10724 let save_guard = self.pool_meta_save_gate.lock().await;
10725 let (pool_meta_guard, snapshot) = self
10726 .acquire_pool_meta_read_guard(&save_guard, "capacity release fence failed")
10727 .await?;
10728 let has_active_source = pool_meta_has_active_decommission(&snapshot);
10729 drop(save_guard);
10730 Ok((pool_meta_guard, has_active_source))
10731 }
10732
10733 pub(crate) async fn run_decommission_capacity_admitted_mutation<T, F, Fut>(
10734 &self,
10735 target_pool_index: usize,
10736 capacity_owner: Option<DecommissionCapacityOwner>,
10737 expected_data_bytes: Option<usize>,
10738 operation: F,
10739 ) -> Result<T>
10740 where
10741 F: FnOnce() -> Fut,
10742 Fut: std::future::Future<Output = Result<T>>,
10743 {
10744 self.run_decommission_capacity_mutation(
10745 target_pool_index,
10746 capacity_owner,
10747 expected_data_bytes,
10748 DecommissionCapacityMutationMode::Durable,
10749 |_| false,
10750 |_| operation(),
10751 )
10752 .await
10753 }
10754
10755 pub(crate) async fn run_decommission_capacity_admitted_mutation_with_capacity_lease<T, F, Fut>(
10756 &self,
10757 target_pool_index: usize,
10758 capacity_owner: Option<DecommissionCapacityOwner>,
10759 expected_data_bytes: Option<usize>,
10760 operation: F,
10761 ) -> Result<T>
10762 where
10763 F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
10764 Fut: std::future::Future<Output = Result<T>>,
10765 {
10766 self.run_decommission_capacity_mutation(
10767 target_pool_index,
10768 capacity_owner,
10769 expected_data_bytes,
10770 DecommissionCapacityMutationMode::Durable,
10771 |_| false,
10772 operation,
10773 )
10774 .await
10775 }
10776
10777 pub(crate) async fn reconcile_decommission_capacity_before_exact_delete(
10778 &self,
10779 bucket: &str,
10780 object: &str,
10781 opts: &ObjectOptions,
10782 exact: &ObjectInfo,
10783 ) -> Result<()> {
10784 if exact.bucket != bucket || exact.name != decode_dir_object(object) {
10785 return Err(decommission_capacity_blocked_error(
10786 "exact delete object identity changed before capacity reconciliation",
10787 ));
10788 }
10789
10790 let (reconciliations, model_version) = {
10791 let save_guard = self.pool_meta_save_gate.lock().await;
10792 let (_read_guard, snapshot) = self
10793 .acquire_pool_meta_read_guard(&save_guard, "exact delete capacity reconciliation failed")
10794 .await?;
10795 let reconciliations = plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)?;
10796 let model_version = active_decommission_capacity_model(&snapshot)?;
10797 (reconciliations, model_version)
10798 };
10799 if reconciliations.is_empty() {
10800 return Ok(());
10801 }
10802 ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
10803
10804 let mut target_guards = Vec::new();
10805 let mut legacy_write_fence = None;
10806 match model_version {
10807 Some(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION) => {
10808 let mut save_guard = self.pool_meta_save_gate.lock().await;
10809 let (write_guard, snapshot) = self
10810 .acquire_pool_meta_write_guard(&mut save_guard, "exact delete capacity reconciliation failed")
10811 .await?;
10812 if active_decommission_capacity_model(&snapshot)? != model_version
10813 || plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)? != reconciliations
10814 {
10815 return Err(decommission_capacity_blocked_error(
10816 "pending capacity changed before exact target evidence could be verified",
10817 ));
10818 }
10819 legacy_write_fence = Some((save_guard, write_guard, snapshot));
10820 }
10821 Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) => {
10822 let mut target_pool_indices = reconciliations
10823 .iter()
10824 .map(|reconciliation| reconciliation.target_pool_index)
10825 .collect::<Vec<_>>();
10826 target_pool_indices.sort_unstable();
10827 target_pool_indices.dedup();
10828 target_guards.reserve(target_pool_indices.len());
10829 for target_pool_index in target_pool_indices {
10830 let guard = self.acquire_decommission_capacity_target_guard(target_pool_index).await?;
10831 ensure_decommission_capacity_target_fence(&guard, target_pool_index, "exact delete evidence")?;
10832 target_guards.push((target_pool_index, guard));
10833 }
10834 }
10835 Some(version) => {
10836 return Err(Error::DecommissionCapacity(format!(
10837 "exact delete capacity reconciliation found unsupported lock model {version}"
10838 )));
10839 }
10840 None => {
10841 return Err(decommission_capacity_blocked_error(
10842 "exact delete capacity reconciliation has no active reservation lock model",
10843 ));
10844 }
10845 }
10846
10847 let target_lookup_options = ObjectOptions {
10848 versioned: opts.versioned,
10849 version_suspended: opts.version_suspended,
10850 version_id: opts.version_id.clone(),
10851 metadata_chg: opts.version_id.is_some(),
10852 no_lock: true,
10853 ..Default::default()
10854 };
10855 for reconciliation in &reconciliations {
10856 let target_pool = self.pools.get(reconciliation.target_pool_index).ok_or_else(|| {
10857 decommission_capacity_blocked_error(format!(
10858 "source pool {} exact-delete capacity target pool {} is out of range",
10859 reconciliation.source_pool_index, reconciliation.target_pool_index
10860 ))
10861 })?;
10862 let target = target_pool
10863 .get_object_info(bucket, object, &target_lookup_options)
10864 .await
10865 .map_err(|err| {
10866 decommission_capacity_blocked_error(format!(
10867 "source pool {} target pool {} exact object evidence could not be read: {err}",
10868 reconciliation.source_pool_index, reconciliation.target_pool_index
10869 ))
10870 })?;
10871 if !Self::is_equivalent_decommission_capacity_target(exact, &target) {
10872 return Err(decommission_capacity_blocked_error(format!(
10873 "source pool {} target pool {} does not contain an equivalent exact object for its pending capacity intent",
10874 reconciliation.source_pool_index, reconciliation.target_pool_index
10875 )));
10876 }
10877 }
10878 ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
10879
10880 let (mut save_guard, write_guard, mut snapshot) = if let Some(fence) = legacy_write_fence {
10881 fence
10882 } else {
10883 let mut save_guard = self.pool_meta_save_gate.lock().await;
10884 let (write_guard, snapshot) = self
10885 .acquire_pool_meta_write_guard(&mut save_guard, "exact delete capacity reconciliation failed")
10886 .await?;
10887 (save_guard, write_guard, snapshot)
10888 };
10889 if active_decommission_capacity_model(&snapshot)? != model_version {
10890 return Err(decommission_capacity_blocked_error(
10891 "capacity lock model changed while exact target evidence was being verified",
10892 ));
10893 }
10894 let current_reconciliations = plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)?;
10895 if current_reconciliations != reconciliations {
10896 return Err(decommission_capacity_blocked_error(
10897 "pending capacity changed while exact target evidence was being verified",
10898 ));
10899 }
10900 ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
10901 for (target_pool_index, target_guard) in &target_guards {
10902 ensure_decommission_capacity_target_fence(target_guard, *target_pool_index, "exact delete capacity finalize")?;
10903 }
10904
10905 let now = OffsetDateTime::now_utc();
10906 let mut source_pool_indices = Vec::with_capacity(current_reconciliations.len());
10907 for reconciliation in current_reconciliations {
10908 resolve_decommission_target_pending(
10909 &mut snapshot,
10910 reconciliation.source_pool_index,
10911 reconciliation.target_pool_index,
10912 reconciliation.expected_target_physical_bytes,
10913 reconciliation.mutation_id,
10914 )?;
10915 record_decommission_target_consumption(
10916 &mut snapshot,
10917 reconciliation.source_pool_index,
10918 reconciliation.target_pool_index,
10919 DecommissionTargetConsumption {
10920 committed_data_bytes: reconciliation.expected_data_bytes,
10921 target_physical_bytes: reconciliation.expected_target_physical_bytes,
10922 observed_physical_bytes: 0,
10923 },
10924 reconciliation.mutation_id,
10925 now,
10926 )?;
10927 source_pool_indices.push(reconciliation.source_pool_index);
10928 }
10929 source_pool_indices.sort_unstable();
10930 source_pool_indices.dedup();
10931 ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
10932
10933 let outcome = snapshot
10934 .save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &source_pool_indices)
10935 .await?;
10936 ensure_pool_meta_write_fence(&write_guard, "exact delete capacity reconciliation save failed")?;
10937 for (target_pool_index, target_guard) in &target_guards {
10938 ensure_decommission_capacity_target_fence(target_guard, *target_pool_index, "exact delete capacity save")?;
10939 }
10940 {
10941 let mut pool_meta = self.pool_meta.write().await;
10942 for source_pool_index in source_pool_indices {
10943 publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, source_pool_index)?;
10944 }
10945 }
10946 ensure_pool_meta_write_fence(&write_guard, "exact delete capacity reconciliation save failed")?;
10947 for (target_pool_index, target_guard) in &target_guards {
10948 ensure_decommission_capacity_target_fence(target_guard, *target_pool_index, "exact delete capacity publication")?;
10949 }
10950 outcome.disarm();
10951 Ok(())
10952 }
10953
10954 pub(crate) async fn reconcile_decommission_capacity_after_equivalent_target(
10955 &self,
10956 owner: DecommissionCapacityOwner,
10957 target_pool_index: usize,
10958 expected_data_bytes: usize,
10959 ) -> Result<()> {
10960 let target_guard = self
10961 .acquire_decommission_capacity_owner_target_guard(owner, target_pool_index)
10962 .await?;
10963 if let Some(target_guard) = target_guard.as_ref() {
10964 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target reconciliation")?;
10965 }
10966 let mut save_guard = self.pool_meta_save_gate.lock().await;
10967 let (pool_meta_guard, mut snapshot) = self
10968 .acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent target reconciliation failed")
10969 .await?;
10970 let source_pool_index = owner.source_pool_index;
10971 let (target_layout, target_pending_physical_bytes, target_consumed_physical_bytes) = {
10972 let reservation = snapshot
10973 .pools
10974 .get(source_pool_index)
10975 .and_then(|pool| pool.decommission.as_ref())
10976 .and_then(|info| info.capacity_reservation.as_ref())
10977 .filter(|reservation| {
10978 reservation.active()
10979 && reservation.source_pool_index == owner.source_pool_index
10980 && reservation.operation_id == owner.operation_id
10981 && reservation.generation == owner.generation
10982 && reservation.owner_nonce == owner.owner_nonce
10983 })
10984 .ok_or_else(|| decommission_capacity_blocked_error("equivalent target reconciliation owner is stale"))?;
10985 let target = reservation
10986 .targets
10987 .iter()
10988 .find(|target| target.pool_index == target_pool_index)
10989 .ok_or_else(|| decommission_capacity_blocked_error("equivalent target reconciliation allocation is missing"))?;
10990 (target.layout, target.pending_physical_bytes, target.consumed_physical_bytes)
10991 };
10992 let mutation_id = owner.mutation_id.ok_or_else(|| {
10993 decommission_capacity_blocked_error("equivalent target reconciliation mutation identity is missing")
10994 })?;
10995 let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
10996 if target_pending_physical_bytes == 0 {
10997 if target_consumed_physical_bytes >= expected_target_physical_bytes {
10998 let mut pool_meta = self.pool_meta.write().await;
10999 ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent target idempotent publication")?;
11000 publish_decommission_capacity_update(&mut pool_meta, &snapshot, source_pool_index)?;
11001 if let Some(target_guard) = target_guard.as_ref() {
11002 ensure_decommission_capacity_target_fence(
11003 target_guard,
11004 target_pool_index,
11005 "equivalent target idempotent publication",
11006 )?;
11007 }
11008 return Ok(());
11009 }
11010 return Err(decommission_capacity_blocked_error(
11011 "equivalent target has no pending capacity intent to reconcile",
11012 ));
11013 }
11014 if target_pending_physical_bytes < expected_target_physical_bytes {
11015 return Err(decommission_capacity_blocked_error(
11016 "equivalent target pending capacity is smaller than the committed object",
11017 ));
11018 }
11019
11020 resolve_decommission_target_pending(
11021 &mut snapshot,
11022 source_pool_index,
11023 target_pool_index,
11024 expected_target_physical_bytes,
11025 mutation_id,
11026 )?;
11027 record_decommission_target_consumption(
11028 &mut snapshot,
11029 source_pool_index,
11030 target_pool_index,
11031 DecommissionTargetConsumption {
11032 committed_data_bytes: expected_data_bytes,
11033 target_physical_bytes: expected_target_physical_bytes,
11034 observed_physical_bytes: 0,
11035 },
11036 mutation_id,
11037 OffsetDateTime::now_utc(),
11038 )?;
11039 let outcome = snapshot
11040 .save_no_lock_armed(
11041 self.pools.clone(),
11042 &mut save_guard,
11043 pool_meta_guard.lock_lost_signal(),
11044 &[source_pool_index],
11045 )
11046 .await?;
11047 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission equivalent target reconciliation save failed")?;
11048 if let Some(target_guard) = target_guard.as_ref() {
11049 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target capacity save")?;
11050 }
11051 {
11052 let mut pool_meta = self.pool_meta.write().await;
11053 publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, source_pool_index)?;
11054 }
11055 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission equivalent target reconciliation save failed")?;
11056 if let Some(target_guard) = target_guard.as_ref() {
11057 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target capacity publication")?;
11058 }
11059 outcome.disarm();
11060 Ok(())
11061 }
11062
11063 pub(crate) async fn run_decommission_capacity_temporary_mutation<T, F, Fut>(
11064 &self,
11065 target_pool_index: usize,
11066 capacity_owner: Option<DecommissionCapacityOwner>,
11067 expected_data_bytes: Option<usize>,
11068 operation: F,
11069 ) -> Result<T>
11070 where
11071 F: FnOnce() -> Fut,
11072 Fut: std::future::Future<Output = Result<T>>,
11073 {
11074 self.run_decommission_capacity_mutation(
11075 target_pool_index,
11076 capacity_owner,
11077 expected_data_bytes,
11078 DecommissionCapacityMutationMode::Temporary,
11079 |_| false,
11080 |_| operation(),
11081 )
11082 .await
11083 }
11084
11085 pub(crate) async fn run_decommission_capacity_temporary_mutation_with_capacity_lease<T, F, Fut>(
11086 &self,
11087 target_pool_index: usize,
11088 capacity_owner: Option<DecommissionCapacityOwner>,
11089 expected_data_bytes: Option<usize>,
11090 operation: F,
11091 ) -> Result<T>
11092 where
11093 F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11094 Fut: std::future::Future<Output = Result<T>>,
11095 {
11096 self.run_decommission_capacity_mutation(
11097 target_pool_index,
11098 capacity_owner,
11099 expected_data_bytes,
11100 DecommissionCapacityMutationMode::Temporary,
11101 |_| false,
11102 operation,
11103 )
11104 .await
11105 }
11106
11107 pub(crate) async fn run_decommission_capacity_non_growing_replacement_with_capacity_lease<T, F, Fut>(
11112 &self,
11113 target_pool_index: usize,
11114 capacity_owner: Option<DecommissionCapacityOwner>,
11115 expected_data_bytes: Option<usize>,
11116 operation: F,
11117 ) -> Result<T>
11118 where
11119 F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11120 Fut: std::future::Future<Output = Result<T>>,
11121 {
11122 self.run_decommission_capacity_mutation(
11123 target_pool_index,
11124 capacity_owner,
11125 expected_data_bytes,
11126 DecommissionCapacityMutationMode::NonGrowingReplacement,
11127 |_| false,
11128 operation,
11129 )
11130 .await
11131 }
11132
11133 pub(crate) async fn reconcile_decommission_capacity_after_equivalent_temporary_target(
11139 &self,
11140 owner: DecommissionCapacityOwner,
11141 target_pool_index: usize,
11142 expected_data_bytes: usize,
11143 ) -> Result<()> {
11144 let target_guard = self
11145 .acquire_decommission_capacity_owner_target_guard(owner, target_pool_index)
11146 .await?;
11147 if let Some(target_guard) = target_guard.as_ref() {
11148 ensure_decommission_capacity_target_fence(
11149 target_guard,
11150 target_pool_index,
11151 "equivalent temporary target reconciliation",
11152 )?;
11153 }
11154 let mut save_guard = self.pool_meta_save_gate.lock().await;
11155 let (pool_meta_guard, mut snapshot) = self
11156 .acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent temporary target reconciliation failed")
11157 .await?;
11158 let source_pool_index = owner.source_pool_index;
11159 let mutation_id = owner
11160 .mutation_id
11161 .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target mutation identity is missing"))?;
11162 let (target_layout, pending_physical_bytes, pending_mutation_id, has_temporary_mutation) = {
11163 let reservation = snapshot
11164 .pools
11165 .get(source_pool_index)
11166 .and_then(|pool| pool.decommission.as_ref())
11167 .and_then(|info| info.capacity_reservation.as_ref())
11168 .filter(|reservation| reservation.admits_cleanup_owner(owner))
11169 .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target owner is stale"))?;
11170 let target = reservation
11171 .targets
11172 .iter()
11173 .find(|target| target.pool_index == target_pool_index)
11174 .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target allocation is missing"))?;
11175 (
11176 target.layout,
11177 target.pending_physical_bytes,
11178 target.pending_mutation_id,
11179 target
11180 .temporary_mutations
11181 .iter()
11182 .any(|mutation| mutation.mutation_id == mutation_id),
11183 )
11184 };
11185 if pending_physical_bytes == 0 && !has_temporary_mutation {
11186 ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation fence failed")?;
11188 if let Some(target_guard) = target_guard.as_ref() {
11189 ensure_decommission_capacity_target_fence(
11190 target_guard,
11191 target_pool_index,
11192 "equivalent temporary target idempotent reconciliation",
11193 )?;
11194 }
11195 return Ok(());
11196 }
11197 if pending_physical_bytes > 0 && pending_mutation_id != Some(mutation_id) {
11198 return Err(decommission_capacity_blocked_error(
11199 "equivalent temporary target pending intent belongs to another mutation",
11200 ));
11201 }
11202 let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
11203 if pending_physical_bytes > 0 && pending_physical_bytes != expected_target_physical_bytes {
11204 return Err(decommission_capacity_blocked_error(
11205 "equivalent temporary target pending capacity does not match the committed checkpoint",
11206 ));
11207 }
11208 if pending_physical_bytes > 0 {
11209 resolve_decommission_target_pending(
11210 &mut snapshot,
11211 source_pool_index,
11212 target_pool_index,
11213 expected_target_physical_bytes,
11214 mutation_id,
11215 )?;
11216 }
11217 settle_decommission_target_non_growing_replacement(
11222 &mut snapshot,
11223 source_pool_index,
11224 target_pool_index,
11225 mutation_id,
11226 OffsetDateTime::now_utc(),
11227 )?;
11228 let outcome = snapshot
11229 .save_no_lock_armed(
11230 self.pools.clone(),
11231 &mut save_guard,
11232 pool_meta_guard.lock_lost_signal(),
11233 &[source_pool_index],
11234 )
11235 .await?;
11236 ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?;
11237 if let Some(target_guard) = target_guard.as_ref() {
11238 ensure_decommission_capacity_target_fence(
11239 target_guard,
11240 target_pool_index,
11241 "equivalent temporary target capacity save",
11242 )?;
11243 }
11244 {
11245 let mut pool_meta = self.pool_meta.write().await;
11246 publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, source_pool_index)?;
11247 }
11248 ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?;
11249 if let Some(target_guard) = target_guard.as_ref() {
11250 ensure_decommission_capacity_target_fence(
11251 target_guard,
11252 target_pool_index,
11253 "equivalent temporary target capacity publication",
11254 )?;
11255 }
11256 outcome.disarm();
11257 Ok(())
11258 }
11259
11260 pub(crate) async fn has_decommission_capacity_temporary_mutation_state(
11261 &self,
11262 target_pool_index: usize,
11263 owner: DecommissionCapacityOwner,
11264 ) -> bool {
11265 let Some(mutation_id) = owner.mutation_id else {
11266 return false;
11267 };
11268 self.pool_meta
11269 .read()
11270 .await
11271 .pools
11272 .get(owner.source_pool_index)
11273 .and_then(|pool| pool.decommission.as_ref())
11274 .and_then(|info| info.capacity_reservation.as_ref())
11275 .filter(|reservation| reservation.admits_cleanup_owner(owner))
11276 .and_then(|reservation| {
11277 reservation
11278 .targets
11279 .iter()
11280 .find(|target| target.pool_index == target_pool_index)
11281 .map(|target| (reservation.model_version, target))
11282 })
11283 .is_some_and(|(model_version, target)| {
11284 if model_version == DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION {
11285 true
11288 } else {
11289 (target.pending_physical_bytes > 0 && target.pending_mutation_id == Some(mutation_id))
11290 || target
11291 .temporary_mutations
11292 .iter()
11293 .any(|mutation| mutation.mutation_id == mutation_id)
11294 }
11295 })
11296 }
11297
11298 pub(crate) async fn decommission_capacity_cleanup_target_indices(
11299 &self,
11300 owner: DecommissionCapacityOwner,
11301 ) -> Result<Vec<usize>> {
11302 self.pool_meta
11303 .read()
11304 .await
11305 .pools
11306 .get(owner.source_pool_index)
11307 .and_then(|pool| pool.decommission.as_ref())
11308 .and_then(|info| info.capacity_reservation.as_ref())
11309 .filter(|reservation| reservation.admits_cleanup_owner(owner))
11310 .map(|reservation| reservation.targets.iter().map(|target| target.pool_index).collect())
11311 .ok_or_else(|| decommission_capacity_blocked_error("decommission multipart cleanup reservation is stale"))
11312 }
11313
11314 pub(crate) async fn run_decommission_capacity_temporary_release_with_capacity_lease<T, F, Fut>(
11315 &self,
11316 target_pool_index: usize,
11317 capacity_owner: Option<DecommissionCapacityOwner>,
11318 operation: F,
11319 ) -> Result<T>
11320 where
11321 F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11322 Fut: std::future::Future<Output = Result<(T, bool)>>,
11323 {
11324 self.run_decommission_capacity_mutation(
11325 target_pool_index,
11326 capacity_owner,
11327 None,
11328 DecommissionCapacityMutationMode::TemporaryRelease,
11329 |result: &(T, bool)| result.1,
11330 operation,
11331 )
11332 .await
11333 .map(|(result, _)| result)
11334 }
11335
11336 async fn run_decommission_capacity_mutation<T, F, Fut, P>(
11337 &self,
11338 target_pool_index: usize,
11339 capacity_owner: Option<DecommissionCapacityOwner>,
11340 expected_data_bytes: Option<usize>,
11341 mode: DecommissionCapacityMutationMode,
11342 clear_pending_on_temporary_release: P,
11343 operation: F,
11344 ) -> Result<T>
11345 where
11346 F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11347 Fut: std::future::Future<Output = Result<T>>,
11348 P: Fn(&T) -> bool,
11349 {
11350 let temporary = matches!(
11351 mode,
11352 DecommissionCapacityMutationMode::Temporary | DecommissionCapacityMutationMode::NonGrowingReplacement
11353 );
11354 let non_growing_replacement = matches!(mode, DecommissionCapacityMutationMode::NonGrowingReplacement);
11355 let temporary_release = matches!(mode, DecommissionCapacityMutationMode::TemporaryRelease);
11356 let mut operation = Some(operation);
11357 let save_guard = self.pool_meta_save_gate.lock().await;
11358 let (read_guard, snapshot) = self
11359 .acquire_pool_meta_read_guard(&save_guard, "target capacity admission failed")
11360 .await?;
11361 let admission_now = OffsetDateTime::now_utc();
11362 let admitted_owner = capacity_owner.and_then(|owner| {
11363 snapshot
11364 .pools
11365 .get(owner.source_pool_index)
11366 .and_then(|pool| pool.decommission.as_ref())
11367 .and_then(|info| info.capacity_reservation.as_ref())
11368 .filter(|reservation| {
11369 if temporary_release {
11370 reservation.admits_cleanup_owner(owner)
11371 } else {
11372 reservation.admits_owner(owner, admission_now)
11373 }
11374 })
11375 .filter(|reservation| {
11376 reservation
11377 .targets
11378 .iter()
11379 .any(|target| target.pool_index == target_pool_index)
11380 })
11381 .map(|reservation| (owner, reservation.model_version))
11382 });
11383 if capacity_owner.is_some() && admitted_owner.is_none() {
11384 return Err(decommission_capacity_blocked_error(
11385 "decommission target mutation reservation identity is stale",
11386 ));
11387 }
11388 let Some((owner, model_version)) = admitted_owner else {
11389 ensure_external_decommission_target_admission(&snapshot, target_pool_index, DecommissionCapacityAdmission::Mutation)?;
11390 drop(save_guard);
11391 let capacity_lease = read_guard.lock_lost_signal();
11392 return operation.take().expect("capacity-admitted operation should run once")(capacity_lease).await;
11393 };
11394
11395 drop(read_guard);
11398 drop(save_guard);
11399 if model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION {
11400 discard_decommission_capacity_target_permit_except(self.id, owner, Some(target_pool_index));
11401 }
11402 let target_guard = match model_version {
11403 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION => None,
11404 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION => {
11405 Some(match take_decommission_capacity_target_permit(self.id, target_pool_index, owner) {
11406 Some(guard) => guard,
11407 None => self.acquire_decommission_capacity_target_guard(target_pool_index).await?,
11408 })
11409 }
11410 version => {
11411 return Err(Error::DecommissionCapacity(format!(
11412 "decommission target mutation found unsupported capacity lock model {version}"
11413 )));
11414 }
11415 };
11416
11417 #[cfg(test)]
11418 pause_decommission_capacity_before_owner_write(self.id).await;
11419 let mut save_guard = self.pool_meta_save_gate.lock().await;
11420 let (write_guard, mut snapshot) = self
11421 .acquire_pool_meta_write_guard(&mut save_guard, "decommission target capacity admission failed")
11422 .await?;
11423 let mutation_id = owner
11424 .mutation_id
11425 .ok_or_else(|| decommission_capacity_blocked_error("decommission mutation identity is missing"))?;
11426 let source_pool_index = owner.source_pool_index;
11427 let owner_current = snapshot
11428 .pools
11429 .get(source_pool_index)
11430 .and_then(|pool| pool.decommission.as_ref())
11431 .and_then(|info| info.capacity_reservation.as_ref())
11432 .filter(|reservation| {
11433 (if temporary_release {
11434 reservation.admits_cleanup_owner(owner)
11435 } else {
11436 reservation.admits_owner(owner, OffsetDateTime::now_utc())
11437 }) && reservation.model_version == model_version
11438 && reservation
11439 .targets
11440 .iter()
11441 .any(|target| target.pool_index == target_pool_index)
11442 })
11443 .is_some();
11444 if !owner_current {
11445 return Err(decommission_capacity_blocked_error(
11446 "decommission target mutation reservation identity changed before commit",
11447 ));
11448 }
11449 let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
11450 if !temporary_release {
11451 ensure_decommission_capacity_reservations_available(&snapshot, &capacity_infos, "mutation")?;
11452 }
11453 let target_layout = snapshot
11454 .pools
11455 .get(source_pool_index)
11456 .and_then(|pool| pool.decommission.as_ref())
11457 .and_then(|info| info.capacity_reservation.as_ref())
11458 .and_then(|reservation| {
11459 reservation
11460 .targets
11461 .iter()
11462 .find(|target| target.pool_index == target_pool_index)
11463 })
11464 .map(|target| target.layout)
11465 .ok_or_else(|| Error::SlowDown)?;
11466 let expected_target_physical_bytes = if temporary_release {
11467 0
11468 } else {
11469 capacity_target_physical_bytes(expected_data_bytes.unwrap_or(1).max(1), target_layout)?
11470 };
11471 if temporary {
11472 let (remaining, inflight) = snapshot
11473 .pools
11474 .get(source_pool_index)
11475 .and_then(|pool| pool.decommission.as_ref())
11476 .and_then(|info| info.capacity_reservation.as_ref())
11477 .and_then(|reservation| {
11478 reservation
11479 .targets
11480 .iter()
11481 .find(|target| target.pool_index == target_pool_index)
11482 .map(|target| {
11483 (
11484 target.remaining_reserved_physical_bytes(reservation.temporary_copies)
11485 / 1usize.saturating_add(reservation.temporary_copies),
11486 target.inflight_physical_bytes,
11487 )
11488 })
11489 })
11490 .unwrap_or_default();
11491 let available = remaining.saturating_sub(inflight);
11492 if expected_target_physical_bytes > available {
11493 return Err(decommission_capacity_blocked_error(format!(
11494 "source pool {source_pool_index} target pool {target_pool_index} temporary operation requires {expected_target_physical_bytes} physical bytes, but only {available} temporary-copy bytes remain"
11495 )));
11496 }
11497 } else {
11498 ensure_decommission_target_owner_admission(
11499 &snapshot,
11500 owner,
11501 target_pool_index,
11502 expected_target_physical_bytes,
11503 OffsetDateTime::now_utc(),
11504 )?;
11505 }
11506 let pending_added = if temporary_release {
11507 0
11508 } else {
11509 reserve_decommission_target_pending(
11510 &mut snapshot,
11511 source_pool_index,
11512 target_pool_index,
11513 expected_target_physical_bytes,
11514 mutation_id,
11515 OffsetDateTime::now_utc(),
11516 )?
11517 };
11518 if pending_added > 0 {
11519 let outcome = snapshot
11520 .save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &[source_pool_index])
11521 .await?;
11522 ensure_pool_meta_write_fence(&write_guard, "decommission target capacity intent save failed")?;
11523 snapshot = outcome.committed.clone();
11524 {
11525 let mut pool_meta = self.pool_meta.write().await;
11526 publish_decommission_capacity_update(&mut pool_meta, &snapshot, source_pool_index)?;
11527 }
11528 ensure_pool_meta_write_fence(&write_guard, "decommission target capacity intent save failed")?;
11529 outcome.disarm();
11530 }
11531 let mut write_guard = Some(write_guard);
11532 let mut save_guard = Some(save_guard);
11533 let capacity_lease = if let Some(target_guard) = target_guard.as_ref() {
11534 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity intent prepare")?;
11535 let capacity_lease = target_guard.lock_lost_signal();
11536 drop(write_guard.take());
11537 drop(save_guard.take());
11538 capacity_lease
11539 } else {
11540 write_guard
11541 .as_ref()
11542 .ok_or_else(|| Error::other("legacy decommission capacity fence disappeared before target mutation"))?
11543 .lock_lost_signal()
11544 };
11545
11546 let capacity_infos = if pending_added > 0 {
11547 self.get_decommission_all_pool_capacity_infos().await?
11548 } else {
11549 capacity_infos
11550 };
11551 let before_free = capacity_infos
11552 .iter()
11553 .find(|capacity| capacity.pool_index == target_pool_index)
11554 .map(|capacity| capacity.physical_free)
11555 .ok_or_else(|| decommission_capacity_blocked_error("target capacity snapshot is missing before mutation"))?;
11556
11557 let result = operation.take().expect("capacity-admitted operation should run once")(capacity_lease).await;
11558 let clear_pending_on_temporary_release = result.as_ref().ok().is_some_and(&clear_pending_on_temporary_release);
11559 if let Some(target_guard) = target_guard.as_ref() {
11560 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "target mutation")?;
11561 let mut finalize_save_guard = self.pool_meta_save_gate.lock().await;
11562 let (finalize_write_guard, finalize_snapshot) = self
11563 .acquire_pool_meta_write_guard(&mut finalize_save_guard, "decommission target capacity finalize failed")
11564 .await?;
11565 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity finalize")?;
11566 save_guard = Some(finalize_save_guard);
11567 write_guard = Some(finalize_write_guard);
11568 snapshot = finalize_snapshot;
11569 }
11570 let write_guard = write_guard
11571 .as_ref()
11572 .ok_or_else(|| Error::other("decommission capacity write fence disappeared before finalize"))?;
11573 let save_guard = save_guard
11574 .as_mut()
11575 .ok_or_else(|| Error::other("decommission capacity save fence disappeared before finalize"))?;
11576 ensure_pool_meta_write_fence(write_guard, "decommission target capacity finalize failed")?;
11577 ensure_decommission_capacity_mutation_intent_current(
11578 &snapshot,
11579 owner,
11580 target_pool_index,
11581 expected_target_physical_bytes,
11582 mutation_id,
11583 temporary_release,
11584 model_version,
11585 )?;
11586 let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
11587 let after_free = capacity_infos
11588 .iter()
11589 .find(|capacity| capacity.pool_index == target_pool_index)
11590 .map(|capacity| capacity.physical_free)
11591 .ok_or_else(|| decommission_capacity_blocked_error("target capacity snapshot is missing after mutation"))?;
11592 let observed_physical_bytes = before_free.saturating_sub(after_free);
11593 let released_physical_bytes = after_free.saturating_sub(before_free);
11594 let now = OffsetDateTime::now_utc();
11595 let progress_changed = if temporary_release {
11596 if result.is_ok() {
11597 release_decommission_target_inflight(
11598 &mut snapshot,
11599 source_pool_index,
11600 target_pool_index,
11601 released_physical_bytes,
11602 mutation_id,
11603 DecommissionCapacityReleaseProof::confirmed_absence(clear_pending_on_temporary_release),
11604 now,
11605 )?
11606 } else {
11607 false
11608 }
11609 } else if result.is_ok() {
11610 resolve_decommission_target_pending(
11611 &mut snapshot,
11612 source_pool_index,
11613 target_pool_index,
11614 expected_target_physical_bytes,
11615 mutation_id,
11616 )?;
11617 if temporary {
11618 if non_growing_replacement {
11619 settle_decommission_target_non_growing_replacement(
11620 &mut snapshot,
11621 source_pool_index,
11622 target_pool_index,
11623 mutation_id,
11624 now,
11625 )?;
11626 } else {
11627 record_decommission_target_inflight(
11628 &mut snapshot,
11629 source_pool_index,
11630 target_pool_index,
11631 observed_physical_bytes,
11632 mutation_id,
11633 now,
11634 )?;
11635 }
11636 } else {
11637 let consumed_physical_bytes = expected_data_bytes
11638 .map(|_| expected_target_physical_bytes)
11639 .unwrap_or(observed_physical_bytes.max(expected_target_physical_bytes));
11640 let committed_data_bytes =
11641 expected_data_bytes.unwrap_or(capacity_source_data_equivalent(observed_physical_bytes, target_layout)?);
11642 record_decommission_target_consumption(
11643 &mut snapshot,
11644 source_pool_index,
11645 target_pool_index,
11646 DecommissionTargetConsumption {
11647 committed_data_bytes,
11648 target_physical_bytes: consumed_physical_bytes,
11649 observed_physical_bytes,
11650 },
11651 mutation_id,
11652 now,
11653 )?;
11654 }
11655 true
11656 } else if temporary {
11657 record_decommission_target_inflight(
11658 &mut snapshot,
11659 source_pool_index,
11660 target_pool_index,
11661 observed_physical_bytes,
11662 mutation_id,
11663 now,
11664 )?
11665 } else {
11666 record_decommission_target_observation(
11667 &mut snapshot,
11668 source_pool_index,
11669 target_pool_index,
11670 observed_physical_bytes,
11671 now,
11672 )?;
11673 observed_physical_bytes > 0
11674 };
11675 let capacity_result = if temporary_release {
11676 Ok(())
11677 } else {
11678 ensure_decommission_capacity_reservations_available(&snapshot, &capacity_infos, "mutation")
11679 };
11680 if let Err(err) = &capacity_result {
11681 snapshot.mark_decommission_capacity_blocked(source_pool_index, err.to_string(), now)?;
11682 }
11683 if temporary_release && !progress_changed {
11684 ensure_pool_meta_write_fence(write_guard, "decommission target capacity cleanup fence failed")?;
11685 if let Some(target_guard) = target_guard.as_ref() {
11686 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity cleanup")?;
11687 }
11688 capacity_result?;
11689 return result;
11690 }
11691 let outcome = snapshot
11692 .save_no_lock_armed(self.pools.clone(), save_guard, write_guard.lock_lost_signal(), &[source_pool_index])
11693 .await?;
11694 ensure_pool_meta_write_fence(write_guard, "decommission target capacity progress save failed")?;
11695 if let Some(target_guard) = target_guard.as_ref() {
11696 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity progress save")?;
11697 }
11698 {
11699 let mut pool_meta = self.pool_meta.write().await;
11700 publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, source_pool_index)?;
11701 }
11702 ensure_pool_meta_write_fence(write_guard, "decommission target capacity progress save failed")?;
11703 if let Some(target_guard) = target_guard.as_ref() {
11704 ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity progress publication")?;
11705 }
11706 outcome.disarm();
11707 capacity_result?;
11708 result
11709 }
11710
11711 pub(crate) async fn ensure_pool_meta_side_effects_safe(&self, operation: &str) -> Result<()> {
11712 self.pool_meta_save_gate.lock().await.ensure_write_safe(operation)
11713 }
11714
11715 pub async fn pool_meta_writes_ready(&self) -> bool {
11719 self.pool_meta_write_status().await.is_ok()
11720 }
11721
11722 pub async fn pool_meta_write_gate_status(&self) -> PoolMetaWriteGateStatus {
11724 let Ok(write_state) = tokio::time::timeout(std::time::Duration::from_millis(100), self.pool_meta_save_gate.lock()).await
11725 else {
11726 return PoolMetaWriteGateStatus {
11727 writes_ready: false,
11728 check_timed_out: true,
11729 ..PoolMetaWriteGateStatus::default()
11730 };
11731 };
11732 let transaction_aborted = write_state.aborted_transaction.load(Ordering::SeqCst);
11733 let writes_ready = !write_state.write_blocked && !transaction_aborted;
11734 let failure = if writes_ready {
11735 None
11736 } else {
11737 write_state.ensure_write_safe("pool metadata snapshot").err()
11738 };
11739 let context = failure.as_ref().and_then(Error::pool_metadata_failure);
11740 PoolMetaWriteGateStatus {
11741 writes_ready,
11742 check_timed_out: false,
11743 write_blocked: write_state.write_blocked,
11744 transaction_aborted,
11745 pool_meta_absent: write_state.pool_meta_absent,
11746 identity_initialized: write_state.identity_initialized,
11747 identity_needs_repair: write_state.identity_needs_repair,
11748 cluster_epoch: write_state.cluster_epoch,
11749 reason: context.map(|context| context.kind.as_str()),
11750 phase: context.map(|context| context.phase),
11751 since_unix_secs: context.map(|context| context.since.unix_timestamp()),
11752 }
11753 }
11754
11755 pub async fn pool_meta_write_status(&self) -> Result<()> {
11757 let write_state = tokio::time::timeout(std::time::Duration::from_millis(100), self.pool_meta_save_gate.lock())
11758 .await
11759 .map_err(|_| Error::Timeout)?;
11760 write_state.ensure_write_safe("pool metadata readiness")
11761 }
11762
11763 pub(crate) async fn recover_pool_meta_transaction(&self) -> Result<bool> {
11764 {
11765 let Ok(state) = self.pool_meta_save_gate.try_lock() else {
11766 return Ok(false);
11767 };
11768 if state.ensure_write_safe("pool metadata recovery").is_ok() {
11769 return Ok(false);
11770 }
11771 if state.write_blocked
11772 || state.active_transactions.load(Ordering::SeqCst) != 0
11773 || state
11774 .transaction_failure
11775 .lock()
11776 .unwrap_or_else(std::sync::PoisonError::into_inner)
11777 .is_none()
11778 {
11779 return Ok(false);
11780 }
11781 }
11782 let _start_guard = self.start_gate.lock().await;
11783 {
11787 let cancelers = self.decommission_cancelers.read().await;
11788 let mut draining = false;
11789 for canceler in cancelers.iter().flatten().filter(|canceler| canceler.is_active()) {
11790 canceler.cancel();
11791 draining = true;
11792 }
11793 if draining {
11794 return Err(Error::OperationCanceled);
11795 }
11796 }
11797 let movement_gate = self.ctx.data_movement_operation_gate();
11798 let _movement_guard = movement_gate.write().await;
11799 let mut state = self.pool_meta_save_gate.lock().await;
11800 if state.write_blocked || state.active_transactions.load(Ordering::SeqCst) != 0 {
11803 return Ok(false);
11804 }
11805 let Some(blocked) = state
11806 .transaction_failure
11807 .lock()
11808 .unwrap_or_else(std::sync::PoisonError::into_inner)
11809 .clone()
11810 else {
11811 return Ok(false);
11812 };
11813 if self
11814 .rebalance_meta
11815 .read()
11816 .await
11817 .as_ref()
11818 .is_some_and(|meta| meta.cancel.is_some())
11819 {
11820 return Err(Error::other("pool metadata recovery requires rebalance workers to be quiescent"));
11821 }
11822 let pool = self
11823 .pools
11824 .first()
11825 .ok_or_else(|| Error::other("pool metadata recovery has no storage pools"))?;
11826 let lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
11827 let guard = lock.get_write_lock(get_lock_acquire_timeout()).await?;
11828 let fence = PoolMetaPersistenceFence::Distributed(guard.lock_lost_signal());
11829 let mut candidate = PoolMetaWriteState {
11830 aborted_transaction: Arc::new(AtomicBool::new(false)),
11831 transaction_failure: Arc::default(),
11832 active_transactions: Arc::default(),
11833 block_context: None,
11834 recovery_failure: None,
11835 write_blocked: false,
11836 ..state.clone()
11837 };
11838 let selection = load_pool_meta_for_transaction_recovery(self.pools.clone(), &mut candidate).await?;
11839 if selection.meta.pools.len() != self.pools.len()
11840 || selection
11841 .meta
11842 .pools
11843 .iter()
11844 .zip(&self.pools)
11845 .enumerate()
11846 .any(|(index, (meta, pool))| meta.id != index || meta.cmd_line != pool.endpoints.cmd_line)
11847 {
11848 return Err(Error::other(
11849 "pool metadata recovery requires operator reconciliation of the pool topology",
11850 ));
11851 }
11852 let outcome = repair_pool_meta_transaction(self.pools.clone(), &mut candidate, selection, &fence).await?;
11853 let mut pool_meta = self.pool_meta.write().await;
11854 fence.ensure_held()?;
11855 self.ctx.advance_data_movement_operation_epoch();
11856 *pool_meta = outcome.committed.clone();
11857 fence.ensure_held()?;
11858 outcome.disarm();
11859 candidate.aborted_transaction = Arc::clone(&state.aborted_transaction);
11861 candidate.transaction_failure = Arc::clone(&state.transaction_failure);
11862 candidate.active_transactions = Arc::clone(&state.active_transactions);
11863 candidate.block_started_at = Arc::clone(&state.block_started_at);
11864 *state
11865 .block_started_at
11866 .lock()
11867 .unwrap_or_else(std::sync::PoisonError::into_inner) = None;
11868 *state
11869 .transaction_failure
11870 .lock()
11871 .unwrap_or_else(std::sync::PoisonError::into_inner) = None;
11872 state.aborted_transaction.store(false, Ordering::SeqCst);
11873 *state = candidate;
11874 info!(
11875 event = EVENT_DECOMMISSION_STATE,
11876 component = LOG_COMPONENT_ECSTORE,
11877 subsystem = LOG_SUBSYSTEM_POOLS,
11878 state = "pool_metadata_recovered",
11879 phase = blocked.phase,
11880 blocked_since = %blocked.since,
11881 "Pool metadata writes recovered"
11882 );
11883 metrics::counter!("rustfs_pool_metadata_recoveries_total").increment(1);
11884 Ok(true)
11885 }
11886
11887 pub(crate) fn record_pool_meta_recovery_failure(&self, error: Error) {
11888 let Ok(mut state) = self.pool_meta_save_gate.try_lock() else {
11889 return;
11890 };
11891 if state.ensure_write_safe("pool metadata recovery").is_ok() {
11892 return;
11893 }
11894 let classify = |error: &Error| match error.pool_metadata_failure() {
11895 Some(context) => (context.kind.as_str(), context.phase),
11896 None => match error {
11897 Error::OperationCanceled => ("workers_draining", "recovery_quiesce"),
11898 Error::Timeout => ("timeout", "recovery"),
11899 Error::Lock(_) => ("fence_unavailable", "recovery"),
11900 _ => ("unavailable", "recovery"),
11901 },
11902 };
11903 let (reason, phase) = classify(&error);
11904 if state
11905 .recovery_failure
11906 .as_ref()
11907 .is_none_or(|previous| classify(previous) != (reason, phase))
11908 {
11909 warn!(
11910 event = EVENT_DECOMMISSION_STATE,
11911 component = LOG_COMPONENT_ECSTORE,
11912 subsystem = LOG_SUBSYSTEM_POOLS,
11913 state = "pool_metadata_recovery_pending",
11914 reason,
11915 phase,
11916 "Pool metadata recovery remains blocked"
11917 );
11918 }
11919 state.recovery_failure = Some(Arc::new(error));
11920 }
11921
11922 async fn load_runtime_pool_meta_observing(&self, write_state: &mut PoolMetaWriteState, operation: &str) -> Result<PoolMeta> {
11923 write_state.ensure_write_safe(operation)?;
11924 load_pool_meta_identity_observing(self.pools.clone(), write_state).await?;
11925 let mut pool_meta = PoolMeta::default();
11926 let replica_state = pool_meta
11927 .load_no_lock_from_replicas_observing(self.pools.clone(), write_state)
11928 .await?;
11929 write_state.observe_replicas(replica_state);
11930 write_state.ensure_missing_metadata_can_initialize()?;
11931 write_state.ensure_write_safe(operation)?;
11932 Ok(pool_meta)
11933 }
11934
11935 pub(crate) async fn load_runtime_pool_meta_under_activation_fence(
11936 &self,
11937 write_state: &mut PoolMetaWriteState,
11938 activation_fence: &PoolRebalanceActivationFence,
11939 operation: &str,
11940 ) -> Result<PoolMeta> {
11941 activation_fence.ensure_held()?;
11942 let pool_meta = self.load_runtime_pool_meta_observing(write_state, operation).await?;
11943 activation_fence.ensure_held()?;
11944 Ok(pool_meta)
11945 }
11946
11947 pub(crate) async fn load_runtime_pool_meta(&self, operation: &str) -> Result<PoolMeta> {
11948 let mut write_state = self.pool_meta_save_gate.lock().await;
11949 write_state.ensure_write_safe(operation)?;
11950 let pool = self
11951 .pools
11952 .first()
11953 .cloned()
11954 .ok_or_else(|| Error::other(format!("{operation}: no storage pools available")))?;
11955 let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
11956 let _pool_meta_guard = pool_meta_lock.get_read_lock(get_lock_acquire_timeout()).await?;
11957 self.load_runtime_pool_meta_observing(&mut write_state, operation).await
11958 }
11959
11960 async fn run_guarded_decommission_side_effect<T, E, F, Fut>(
11961 &self,
11962 rx: &CancellationToken,
11963 operation_gate: &Arc<tokio::sync::RwLock<()>>,
11964 operation: F,
11965 ) -> std::result::Result<T, E>
11966 where
11967 F: FnOnce() -> Fut,
11968 Fut: std::future::Future<Output = std::result::Result<T, E>>,
11969 E: From<Error>,
11970 {
11971 let _operation_guard = tokio::select! {
11972 biased;
11973 _ = rx.cancelled() => return Err(Error::OperationCanceled.into()),
11974 guard = operation_gate.read() => guard,
11975 };
11976
11977 if rx.is_cancelled() {
11978 return Err(Error::OperationCanceled.into());
11979 }
11980 self.ensure_pool_meta_side_effects_safe("decommission side effect blocked because pool metadata requires recovery")
11981 .await
11982 .map_err(E::from)?;
11983 if rx.is_cancelled() {
11984 return Err(Error::OperationCanceled.into());
11985 }
11986
11987 let result = operation().await;
11988 if rx.is_cancelled() {
11989 return Err(Error::OperationCanceled.into());
11990 }
11991 result
11992 }
11993
11994 async fn save_current_pool_meta(&self, indices: &[usize]) -> Result<()> {
11995 let mut save_guard = self.pool_meta_save_gate.lock().await;
11996 let (pool_meta_guard, mut snapshot) = self
11997 .acquire_pool_meta_write_guard(&mut save_guard, "pool metadata save failed")
11998 .await?;
11999 {
12000 let pool_meta = self.pool_meta.read().await;
12001 merge_pool_meta_updates_for_save(&mut snapshot, &pool_meta, indices, "pool metadata save failed")?;
12002 }
12003 let outcome = snapshot
12004 .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), indices)
12005 .await?;
12006 let mut pool_meta = self.pool_meta.write().await;
12007 ensure_pool_meta_write_fence(&pool_meta_guard, "pool metadata save failed")?;
12008 publish_pool_meta_updates(&mut pool_meta, &outcome.committed, indices);
12009 ensure_pool_meta_write_fence(&pool_meta_guard, "pool metadata save failed")?;
12010 drop(pool_meta);
12011 outcome.disarm();
12012 Ok(())
12013 }
12014
12015 #[cfg(test)]
12016 pub(crate) async fn save_current_pool_meta_for_test(&self, indices: &[usize]) -> Result<()> {
12017 self.save_current_pool_meta(indices).await
12018 }
12019
12020 async fn persist_decommission_unresolved_entry(
12021 &self,
12022 idx: usize,
12023 generation: OffsetDateTime,
12024 entry: DecommissionUnresolvedEntry,
12025 ) -> Result<()> {
12026 {
12027 let rebalance_meta = self.rebalance_meta.read().await.clone();
12028 let mut pool_meta = self.pool_meta.write().await;
12029 ensure_decommission_ledger_persistence_supported(&pool_meta)?;
12030 record_decommission_unresolved_entry(
12031 &mut pool_meta,
12032 idx,
12033 generation,
12034 entry,
12035 OffsetDateTime::now_utc(),
12036 rebalance_meta.as_ref(),
12037 )?;
12038 }
12039 self.save_current_pool_meta(&[idx])
12040 .await
12041 .map_err(|err| Error::other(format!("decommission unresolved entry ledger save failed: {err}")))
12042 }
12043
12044 async fn save_decommission_progress_checkpoint(&self, idx: usize, generation: OffsetDateTime) -> Result<bool> {
12045 self.save_decommission_progress_checkpoint_at(idx, generation, OffsetDateTime::now_utc())
12046 .await
12047 }
12048
12049 async fn save_decommission_progress_checkpoint_at(
12050 &self,
12051 idx: usize,
12052 generation: OffsetDateTime,
12053 now: OffsetDateTime,
12054 ) -> Result<bool> {
12055 let mut save_guard = self.pool_meta_save_gate.lock().await;
12059 let (pool_meta_guard, mut snapshot) = self
12060 .acquire_pool_meta_write_guard(&mut save_guard, "decommission progress save failed")
12061 .await?;
12062 let (snapshot, checkpoint) = {
12063 let rebalance_meta = self.rebalance_meta.read().await.clone();
12064 let pool_meta = self.pool_meta.read().await;
12065 ensure_decommission_generation(&pool_meta, idx, generation)?;
12066 let Some(checkpoint) = pool_meta.decommission_progress_checkpoint(
12067 idx,
12068 DECOMMISSION_PROGRESS_SAVE_INTERVAL,
12069 now,
12070 rebalance_meta.as_ref(),
12071 )?
12072 else {
12073 return Ok(false);
12074 };
12075
12076 let mut current = pool_meta.clone();
12077 let current_count = current.pools.len();
12078 let Some(pool) = current.pools.get_mut(idx) else {
12079 return Err(invalid_decommission_pool_index_error(current_count, idx));
12080 };
12081 pool.last_update = checkpoint.checkpoint_at;
12082 if let (Some(operation_id), Some(owner_nonce), Some(expires_at), Some(reservation)) = (
12083 checkpoint.capacity_operation_id,
12084 checkpoint.capacity_owner_nonce,
12085 checkpoint.capacity_lease_expires_at,
12086 pool.decommission.as_mut().and_then(|info| info.capacity_reservation.as_mut()),
12087 ) && reservation.operation_id == operation_id
12088 && reservation.owner_nonce == owner_nonce
12089 && reservation.active()
12090 {
12091 reservation.renewed_at = checkpoint.checkpoint_at;
12092 reservation.expires_at = expires_at;
12093 }
12094 merge_pool_meta_updates_for_save(&mut snapshot, ¤t, &[idx], "decommission progress save failed")?;
12095 (snapshot, checkpoint)
12096 };
12097
12098 let outcome = match snapshot
12099 .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx])
12100 .await
12101 {
12102 Ok(outcome) => outcome,
12103 Err(err) => {
12104 let retry_after = OffsetDateTime::now_utc() + DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
12105 let mut pool_meta = self.pool_meta.write().await;
12106 pool_meta.defer_decommission_progress_checkpoint(idx, checkpoint, retry_after);
12107 return Err(err);
12108 }
12109 };
12110
12111 let mut pool_meta = self.pool_meta.write().await;
12112 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission progress save failed")?;
12113 pool_meta.version = pool_meta.version.max(outcome.committed.version);
12114 let committed = pool_meta.commit_decommission_progress_checkpoint(idx, checkpoint);
12115 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission progress save failed")?;
12116 drop(pool_meta);
12117 outcome.disarm();
12118 Ok(committed)
12119 }
12120
12121 async fn mark_decommission_bucket_done_and_save(
12122 &self,
12123 idx: usize,
12124 generation: OffsetDateTime,
12125 bucket: &DecomBucketInfo,
12126 ) -> Result<bool> {
12127 let mut save_guard = self.pool_meta_save_gate.lock().await;
12130 let (pool_meta_guard, mut snapshot) = self
12131 .acquire_pool_meta_write_guard(&mut save_guard, "decommission bucket completion save failed")
12132 .await?;
12133 ensure_decommission_generation(&snapshot, idx, generation)?;
12134 let rebalance_meta = self.rebalance_meta.read().await.clone();
12135 let checkpoint = {
12136 let pool_meta = self.pool_meta.read().await;
12137 ensure_decommission_generation(&pool_meta, idx, generation)?;
12138 let mut current = pool_meta.clone();
12139 publish_decommission_capacity_update(&mut current, &snapshot, idx)?;
12140 if !mark_decommission_bucket_done(&mut current, idx, bucket)? {
12141 return Ok(false);
12142 }
12143 let checkpoint_at = current.next_scanner_data_movement_update(OffsetDateTime::now_utc(), rebalance_meta.as_ref());
12144 current.pools[idx].last_update = checkpoint_at;
12145 let info = current.pools[idx]
12146 .decommission
12147 .as_ref()
12148 .ok_or_else(|| decommission_metadata_not_initialized_error("prepare decommission bucket completion"))?;
12149 let checkpoint = DecommissionProgressCheckpoint {
12150 start_time: Some(generation),
12151 queued: false,
12152 counted_items: info.counted_items(),
12153 checkpoint_at,
12154 capacity_operation_id: None,
12155 capacity_owner_nonce: None,
12156 capacity_lease_expires_at: None,
12157 };
12158 merge_pool_meta_updates_for_save(&mut snapshot, ¤t, &[idx], "decommission bucket completion save failed")?;
12159 checkpoint
12160 };
12161
12162 #[cfg(test)]
12163 decommission_test_wrap_result("bucket_completion_before_save", &bucket.name, &bucket.prefix, 0, Ok(()))?;
12164 let outcome = snapshot
12165 .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx])
12166 .await?;
12167 #[cfg(test)]
12168 decommission_test_wrap_result("bucket_completion_after_save", &bucket.name, &bucket.prefix, 0, Ok(()))?;
12169 let mut pool_meta = self.pool_meta.write().await;
12170 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission bucket completion save failed")?;
12171 ensure_decommission_generation(&pool_meta, idx, generation)?;
12172 publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, idx)?;
12173 mark_decommission_bucket_done(&mut pool_meta, idx, bucket)?;
12174 pool_meta.commit_decommission_progress_checkpoint(idx, checkpoint);
12175 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission bucket completion save failed")?;
12176 drop(pool_meta);
12177 outcome.disarm();
12178 Ok(true)
12179 }
12180
12181 pub(crate) async fn save_current_pool_meta_for_decommission_start(
12182 &self,
12183 indices: &[usize],
12184 decom_buckets: Vec<DecomBucketInfo>,
12185 ) -> Result<PoolMeta> {
12186 let mut save_guard = self.pool_meta_save_gate.lock().await;
12187 save_guard.ensure_write_safe("decommission start failed")?;
12188 let rebalance_pool = self
12189 .pools
12190 .first()
12191 .cloned()
12192 .ok_or_else(|| Error::other("decommission start rebalance metadata load failed: no storage pools available"))?;
12193 #[cfg(test)]
12194 observe_pool_activation_start_attempt(PoolActivationStartKind::Decommission);
12195 let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
12196 let mut activation_fence = acquire_pool_rebalance_activation_locks(rebalance_pool.clone(), fleet_proof).await?;
12197
12198 let mut rebalance_meta = RebalanceMeta::new();
12199 match rebalance_meta
12200 .load_with_opts(
12201 rebalance_pool.clone(),
12202 ObjectOptions {
12203 no_lock: true,
12204 ..Default::default()
12205 },
12206 )
12207 .await
12208 {
12209 Ok(()) => ensure_decommission_start_rebalance_meta_allowed(Some(&rebalance_meta))?,
12210 Err(Error::ConfigNotFound) => {}
12211 Err(err) => {
12212 return Err(Error::other(format!(
12213 "rebalance metadata load before decommission start save failed: {err}"
12214 )));
12215 }
12216 }
12217
12218 let current_pool_meta = {
12219 let pool_meta = self.pool_meta.read().await;
12220 pool_meta.clone()
12221 };
12222 let mut latest_pool_meta = PoolMeta::default();
12223 let replica_state = latest_pool_meta
12224 .load_no_lock_from_replicas_observing(self.pools.clone(), &mut save_guard)
12225 .await?;
12226 save_guard.observe_replicas(replica_state);
12227 save_guard.ensure_write_safe("decommission start failed")?;
12228 if latest_pool_meta.pools.is_empty() {
12229 latest_pool_meta = current_pool_meta;
12230 }
12231
12232 ensure_decommission_start_pool_states(&latest_pool_meta, indices)?;
12233 ensure_decommission_ledger_persistence_supported(&latest_pool_meta)?;
12234 ensure_decommission_capacity_writer_supported(&latest_pool_meta)?;
12235 let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
12236 activation_fence.ensure_held()?;
12237 let target_fence_proof = crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof();
12238
12239 let previous_pool_meta = latest_pool_meta.clone();
12240 let capacity_generation = next_decommission_capacity_generation(&latest_pool_meta)?;
12241 let first_idx = indices.first().copied();
12242 let now = OffsetDateTime::now_utc();
12243 for idx in indices.iter().copied() {
12244 let capacity = capacity_infos
12245 .iter()
12246 .find(|capacity| capacity.pool_index == idx)
12247 .ok_or_else(|| Error::DecommissionCapacity(format!("decommission capacity snapshot is missing pool {idx}")))?;
12248 latest_pool_meta.set_decommission_state_at(
12249 idx,
12250 capacity.space,
12251 Some(idx) != first_idx,
12252 now,
12253 Some(&rebalance_meta),
12254 )?;
12255 latest_pool_meta.queue_buckets(idx, decom_buckets.clone());
12256 }
12257 let model_version = select_decommission_capacity_model(&latest_pool_meta, target_fence_proof.is_some())?;
12258 if model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION {
12259 activation_fence.set_decommission_target_fence_proof(target_fence_proof);
12260 }
12261 reserve_decommission_start_target_capacity(
12262 &mut latest_pool_meta,
12263 indices,
12264 &capacity_infos,
12265 uuid::Uuid::new_v4(),
12266 capacity_generation,
12267 now,
12268 model_version,
12269 )?;
12270
12271 activation_fence.ensure_held()?;
12272 let outcome = latest_pool_meta
12273 .save_no_lock_with_activation_fence(self.pools.clone(), &mut save_guard, &activation_fence, indices)
12274 .await?;
12275 activation_fence.ensure_held()?;
12276 {
12277 let mut pool_meta = self.pool_meta.write().await;
12278 publish_pool_meta_updates(&mut pool_meta, &outcome.committed, indices);
12279 }
12280 activation_fence.ensure_held()?;
12281 outcome.disarm();
12282
12283 Ok(previous_pool_meta)
12284 }
12285
12286 async fn rollback_decommission_start_after_reload_failure(
12287 &self,
12288 movement_gate: &Arc<tokio::sync::RwLock<()>>,
12289 previous_pool_meta: &PoolMeta,
12290 indices: &[usize],
12291 ) -> Result<()> {
12292 let _movement_guard = movement_gate.write().await;
12293 let mut save_guard = self.pool_meta_save_gate.lock().await;
12294 let (pool_meta_guard, mut snapshot) = self
12295 .acquire_pool_meta_write_guard(&mut save_guard, "decommission start rollback failed")
12296 .await?;
12297 rollback_start_decommission_pool_meta(&mut snapshot, previous_pool_meta, indices);
12298 let outcome = snapshot
12299 .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), indices)
12300 .await?;
12301 let mut pool_meta = self.pool_meta.write().await;
12302 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission start rollback failed")?;
12303 publish_pool_meta_updates(&mut pool_meta, &outcome.committed, indices);
12304 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission start rollback failed")?;
12305 drop(pool_meta);
12306 outcome.disarm();
12307 self.ctx.advance_data_movement_operation_epoch();
12308 Ok(())
12309 }
12310
12311 async fn ensure_decommission_rebalance_idle_after_refresh(&self) -> Result<()> {
12312 self.load_rebalance_meta().await?;
12313 ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)
12314 }
12315
12316 async fn ensure_decommission_rebalance_idle_after_refresh_under_start_gate(&self) -> Result<()> {
12317 self.load_rebalance_meta_under_start_gate().await?;
12318 ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)
12319 }
12320
12321 pub async fn status(&self, idx: usize) -> Result<PoolStatus> {
12322 let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
12323 let space_info = capacity_infos
12324 .iter()
12325 .find(|capacity| capacity.pool_index == idx)
12326 .map(|capacity| capacity.space)
12327 .ok_or_else(|| invalid_decommission_pool_index_error(self.pools.len(), idx))?;
12328
12329 let pool_meta = self.pool_meta.read().await;
12330 let active_target_reservations = active_decommission_target_reservations(&pool_meta);
12331
12332 let mut pool_info = get_by_index(pool_meta.pools.as_slice(), idx, "fetch decommission status")?.clone();
12333 if let Some(info) = pool_info.decommission.as_mut() {
12334 observe_decommission_capacity_reservation(info, &capacity_infos, &active_target_reservations);
12335 }
12336 Ok(apply_decommission_status_space_info(pool_info, space_info))
12337 }
12338
12339 #[tracing::instrument(skip_all)]
12340 pub async fn refresh_pool_status_meta(&self) -> Result<()> {
12341 let movement_gate = self.ctx.data_movement_operation_gate();
12342 let _movement_guard = movement_gate.write().await;
12343 let persisted = self.load_runtime_pool_meta("refresh pool status metadata failed").await?;
12344
12345 let active_workers = {
12346 let cancelers = self.decommission_cancelers.read().await;
12347 cancelers
12348 .iter()
12349 .map(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
12350 .collect::<Vec<_>>()
12351 };
12352
12353 let mut pool_meta = self.pool_meta.write().await;
12354 if merge_pool_status_refresh(&mut pool_meta, persisted, &active_workers) {
12355 self.ctx.advance_data_movement_operation_epoch();
12356 }
12357 Ok(())
12358 }
12359
12360 async fn get_decommission_pool_capacity_info(&self, idx: usize) -> Result<DecommissionPoolCapacityInfo> {
12361 if let Some(sets) = self.pools.get(idx) {
12362 let mut info = sets.storage_info_snapshot().await;
12363 info.backend = StorageAdminApi::backend_info(self).await;
12364
12365 let total = get_total_usable_capacity(&info.disks, &info);
12366 let free = get_total_usable_capacity_free(&info.disks, &info);
12367 let space = PoolSpaceInfo {
12368 free,
12369 total,
12370 used: total.saturating_sub(free),
12371 };
12372 let layout = DecommissionErasureLayout {
12373 data: info
12374 .backend
12375 .standard_sc_data
12376 .get(idx)
12377 .copied()
12378 .unwrap_or_else(|| sets.set_drive_count.saturating_sub(sets.parity_count)),
12379 parity: info
12380 .backend
12381 .standard_sc_parities
12382 .get(idx)
12383 .copied()
12384 .unwrap_or(sets.parity_count),
12385 };
12386 if !layout.is_valid() {
12387 return Err(Error::DecommissionCapacity(format!(
12388 "failed to read decommission capacity for pool {idx}: invalid erasure layout data={} parity={}",
12389 layout.data, layout.parity
12390 )));
12391 }
12392 let (physical_total, physical_free, physical_used) =
12393 decommission_physical_pool_capacity(&info.disks, idx, layout, space);
12394
12395 Ok(DecommissionPoolCapacityInfo {
12396 pool_index: idx,
12397 space,
12398 layout,
12399 physical_free,
12400 physical_total,
12401 physical_used,
12402 })
12403 } else {
12404 Err(invalid_decommission_pool_index_error(self.pools.len(), idx))
12405 }
12406 }
12407
12408 async fn get_decommission_all_pool_capacity_infos(&self) -> Result<Vec<DecommissionPoolCapacityInfo>> {
12409 #[cfg(any(test, feature = "test-util"))]
12410 if let Some(capacity_infos) = take_decommission_capacity_info_override_for_test(self.id) {
12411 return Ok(capacity_infos);
12412 }
12413
12414 let mut capacity_infos = Vec::with_capacity(self.pools.len());
12415 for idx in 0..self.pools.len() {
12416 capacity_infos.push(self.get_decommission_pool_capacity_info(idx).await?);
12417 }
12418 Ok(capacity_infos)
12419 }
12420
12421 async fn ensure_decommission_runtime_capacity_available(&self, idx: usize, generation: OffsetDateTime) -> Result<()> {
12422 let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
12423 let pool_meta = self.pool_meta.read().await;
12424 ensure_decommission_generation(&pool_meta, idx, generation)?;
12425 #[cfg(test)]
12426 if pool_meta
12427 .pools
12428 .get(idx)
12429 .and_then(|pool| pool.decommission.as_ref())
12430 .is_some_and(|info| info.capacity_reservation.is_none())
12431 {
12432 return Ok(());
12433 }
12434 ensure_decommission_capacity_reservations_available(&pool_meta, &capacity_infos, "migration")
12435 }
12436
12437 async fn decommission_capacity_owner_for_worker(
12438 &self,
12439 idx: usize,
12440 generation: OffsetDateTime,
12441 ) -> Result<Option<DecommissionCapacityOwner>> {
12442 let pool_meta = self.pool_meta.read().await;
12443 ensure_decommission_generation(&pool_meta, idx, generation)?;
12444 let Some(reservation) = pool_meta
12445 .pools
12446 .get(idx)
12447 .and_then(|pool| pool.decommission.as_ref())
12448 .and_then(|info| info.capacity_reservation.as_ref())
12449 .filter(|reservation| reservation.lease_active_at(OffsetDateTime::now_utc()))
12450 else {
12451 return Ok(None);
12452 };
12453 Ok(Some(DecommissionCapacityOwner {
12454 source_pool_index: idx,
12455 operation_id: reservation.operation_id,
12456 generation: reservation.generation,
12457 owner_nonce: reservation.owner_nonce,
12458 mutation_id: None,
12459 }))
12460 }
12461
12462 pub(crate) async fn is_decommission_capacity_target_reserved(
12463 &self,
12464 owner: DecommissionCapacityOwner,
12465 target_pool_index: usize,
12466 ) -> Result<bool> {
12467 let pool_meta = self.pool_meta.read().await;
12468 let reservation = pool_meta
12469 .pools
12470 .get(owner.source_pool_index)
12471 .and_then(|pool| pool.decommission.as_ref())
12472 .and_then(|info| info.capacity_reservation.as_ref())
12473 .filter(|reservation| reservation.admits_owner(owner, OffsetDateTime::now_utc()))
12474 .ok_or_else(|| decommission_capacity_blocked_error("decommission target selection reservation is stale"))?;
12475 Ok(reservation
12476 .targets
12477 .iter()
12478 .any(|target| target.pool_index == target_pool_index))
12479 }
12480
12481 pub(crate) async fn select_decommission_capacity_target_pool(
12482 &self,
12483 owner: DecommissionCapacityOwner,
12484 expected_data_bytes: usize,
12485 ) -> Result<usize> {
12486 let pool_meta = self.pool_meta.read().await;
12487 let reservation = pool_meta
12488 .pools
12489 .get(owner.source_pool_index)
12490 .and_then(|pool| pool.decommission.as_ref())
12491 .and_then(|info| info.capacity_reservation.as_ref())
12492 .filter(|reservation| reservation.admits_owner(owner, OffsetDateTime::now_utc()))
12493 .ok_or_else(|| decommission_capacity_blocked_error("decommission target selection reservation is stale"))?;
12494 let candidate = |target: &DecommissionCapacityTarget| {
12495 let expected_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target.layout).ok()?;
12496 let required_peak = expected_physical_bytes.saturating_mul(1usize.saturating_add(reservation.temporary_copies));
12497 let remaining = target.remaining_reserved_physical_bytes(reservation.temporary_copies);
12498 (required_peak <= remaining).then_some((target.pool_index, remaining))
12499 };
12500 if let Some(permitted_target_pool_index) = decommission_capacity_target_permit_index(self.id, owner) {
12501 if let Some((pool_index, _)) = reservation
12502 .targets
12503 .iter()
12504 .find(|target| target.pool_index == permitted_target_pool_index)
12505 .and_then(&candidate)
12506 {
12507 return Ok(pool_index);
12508 }
12509 discard_decommission_capacity_target_permit_except(self.id, owner, None);
12513 }
12514 reservation
12515 .targets
12516 .iter()
12517 .filter_map(candidate)
12518 .max_by_key(|(_, remaining)| *remaining)
12519 .map(|(pool_index, _)| pool_index)
12520 .ok_or_else(|| {
12521 decommission_capacity_blocked_error(format!(
12522 "source pool {} has no target allocation for {expected_data_bytes} data bytes",
12523 owner.source_pool_index
12524 ))
12525 })
12526 }
12527
12528 pub(crate) async fn next_scanner_data_movement_update(&self, now: OffsetDateTime) -> OffsetDateTime {
12529 let pool_meta = self.pool_meta.read().await;
12530 let rebalance_meta = self.rebalance_meta.read().await;
12531 pool_meta.next_scanner_data_movement_update(now, rebalance_meta.as_ref())
12532 }
12533
12534 #[tracing::instrument(skip(self))]
12535 pub async fn decommission_cancel(self: &Arc<Self>, idx: usize) -> Result<()> {
12536 self.decommission_cancel_with_owner(idx, None).await
12537 }
12538
12539 async fn decommission_cancel_for_operation(self: &Arc<Self>, idx: usize, owner: &DecommissionCanceler) -> Result<()> {
12540 self.decommission_cancel_with_owner(idx, Some(owner)).await
12541 }
12542
12543 #[cfg(test)]
12544 async fn decommission_cancel_with_owner_and_save<Save, SaveFuture>(
12545 self: &Arc<Self>,
12546 idx: usize,
12547 owner: Option<&DecommissionCanceler>,
12548 save_pool_meta: Save,
12549 ) -> Result<()>
12550 where
12551 Save: FnOnce(PoolMeta, Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> SaveFuture + Send + 'static,
12552 SaveFuture: Future<Output = Result<()>> + Send + 'static,
12553 {
12554 let store = self.clone();
12555 let owner = owner.cloned();
12556 tokio::spawn(async move { store.decommission_cancel_transaction(idx, owner, false, save_pool_meta).await })
12559 .await
12560 .map_err(|err| Error::other(format!("decommission cancel transaction task join error: {err}")))?
12561 }
12562
12563 async fn decommission_cancel_transaction<Save, SaveFuture>(
12564 &self,
12565 idx: usize,
12566 owner: Option<DecommissionCanceler>,
12567 acquire_runtime_fence: bool,
12568 save_pool_meta: Save,
12569 ) -> Result<()>
12570 where
12571 Save: FnOnce(PoolMeta, Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> SaveFuture,
12572 SaveFuture: Future<Output = Result<()>>,
12573 {
12574 let owner = owner.as_ref();
12575 ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?;
12576 #[cfg(test)]
12577 if acquire_runtime_fence {
12578 pause_decommission_cancel_before_start_gate(self.id).await;
12579 }
12580 let _start_guard = self.start_gate.lock().await;
12581 let terminal_fence_plan = if acquire_runtime_fence {
12585 let read_save_guard = self.pool_meta_save_gate.lock().await;
12586 let (read_guard, snapshot) = self
12587 .acquire_pool_meta_read_guard(&read_save_guard, "decommission cancel fence planning failed")
12588 .await?;
12589 let plan = decommission_capacity_terminal_fence_plan(&snapshot, idx)?;
12590 drop(read_guard);
12591 drop(read_save_guard);
12592 plan
12593 } else {
12594 None
12595 };
12596 let _capacity_target_guards = if acquire_runtime_fence {
12597 self.acquire_decommission_capacity_terminal_guards(idx, terminal_fence_plan.as_ref())
12598 .await?
12599 } else {
12600 Vec::new()
12601 };
12602 let mut save_guard = self.pool_meta_save_gate.lock().await;
12603 let (_pool_meta_guard, mut persisted_pool_meta) = if acquire_runtime_fence {
12604 let (guard, pool_meta) = self
12605 .acquire_pool_meta_write_guard(&mut save_guard, "decommission cancel failed")
12606 .await?;
12607 (Some(guard), Some(pool_meta))
12608 } else {
12609 save_guard.ensure_write_safe("decommission cancel failed")?;
12610 (None, None)
12611 };
12612 if let Some(persisted_pool_meta) = persisted_pool_meta.as_ref() {
12613 let committed_plan = decommission_capacity_terminal_fence_plan(persisted_pool_meta, idx)?;
12614 if committed_plan != terminal_fence_plan {
12615 return Err(decommission_capacity_blocked_error(
12616 "decommission capacity owner or target cohort changed while acquiring terminal fences",
12617 ));
12618 }
12619 }
12620 let pool_meta_fence = _pool_meta_guard
12621 .as_ref()
12622 .and_then(rustfs_lock::NamespaceLockGuard::lock_lost_signal);
12623
12624 let rebalance_meta = self.rebalance_meta.read().await.clone();
12629 let terminal_at = OffsetDateTime::now_utc();
12630 let mut cancelers = self.decommission_cancelers.write().await;
12631 let mut pool_meta = self.pool_meta.write().await;
12632 if acquire_runtime_fence {
12633 let local_plan = decommission_capacity_terminal_fence_plan(&pool_meta, idx)?;
12634 if local_plan != terminal_fence_plan {
12635 return Err(decommission_capacity_blocked_error(
12636 "local decommission capacity owner differs from the durable terminal fence plan",
12637 ));
12638 }
12639 }
12640 let (pending, should_reload_pool_meta, already_canceled, terminal_canceler, durable_movement_generation) = {
12641 let mut already_canceled = false;
12642 let (pool_present, decommission_present, terminal) = if let Some(pool) = pool_meta.pools.get(idx) {
12643 if let Some(info) = pool.decommission.as_ref() {
12644 already_canceled = info.canceled;
12645 (
12646 true,
12647 info.has_decommission_state(),
12648 should_reject_decommission_cancel_as_terminal(info.complete, info.failed),
12649 )
12650 } else {
12651 (true, false, false)
12652 }
12653 } else {
12654 (false, false, false)
12655 };
12656
12657 ensure_decommission_cancel_allowed(pool_present, decommission_present, terminal)?;
12658 let previous_pool = pool_meta
12659 .pools
12660 .get(idx)
12661 .cloned()
12662 .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), idx))?;
12663 let previous_decommission = previous_pool
12664 .decommission
12665 .as_ref()
12666 .ok_or_else(|| decommission_metadata_not_initialized_error("cancel decommission"))?;
12667 let mut snapshot = pool_meta.clone();
12668 let Some(changed) = update_decommission_for_operation(cancelers.as_slice(), &mut snapshot, idx, owner, |pool_meta| {
12669 pool_meta.decommission_cancel_at(idx, terminal_at, rebalance_meta.as_ref())
12670 }) else {
12671 return Ok(());
12672 };
12673 let pending = if changed {
12674 let canceled_pool = snapshot
12675 .pools
12676 .get(idx)
12677 .cloned()
12678 .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), idx))?;
12679 if let Some(persisted) = persisted_pool_meta.as_mut() {
12680 merge_pool_meta_updates_for_save(persisted, &snapshot, &[idx], "decommission cancel failed")?;
12681 snapshot = persisted.clone();
12682 }
12683 Some((
12684 snapshot,
12685 DecommissionCancelCommit {
12686 previous_start_time: previous_decommission.start_time,
12687 previous_queued: previous_decommission.queued,
12688 previous_last_update: previous_pool.last_update,
12689 canceled_pool,
12690 },
12691 ))
12692 } else {
12693 None
12694 };
12695 let terminal_canceler = if let Some(owner) = owner {
12696 Some(owner.clone())
12697 } else {
12698 cancelers.get(idx).and_then(Option::as_ref).cloned()
12699 };
12700 let durable_movement_generation = pending
12701 .as_ref()
12702 .map(|(_, commit)| crate::store::scanner_data_movement_timestamp_generation(commit.canceled_pool.last_update))
12703 .unwrap_or_default();
12704 (
12705 pending,
12706 should_retry_decommission_cancel_reload(changed, already_canceled),
12707 already_canceled,
12708 terminal_canceler,
12709 durable_movement_generation,
12710 )
12711 };
12712 let active_worker = terminal_canceler.as_ref().is_some_and(DecommissionCanceler::is_active);
12713 if !active_worker && !already_canceled {
12714 warn!(
12715 event = EVENT_DECOMMISSION_STATE,
12716 component = LOG_COMPONENT_ECSTORE,
12717 subsystem = LOG_SUBSYSTEM_POOLS,
12718 pool_index = idx,
12719 state = "cancel_skipped",
12720 reason = "no_active_canceler",
12721 "Decommission cancel skipped"
12722 );
12723 }
12724
12725 let changed = pending.is_some();
12726 let commit_result = if let Some((snapshot, commit)) = pending {
12727 if let Err(err) = save_pool_meta(snapshot, pool_meta_fence).await {
12728 save_guard.block_writes();
12729 return Err(err);
12730 }
12731 if let Some(pool_meta_guard) = _pool_meta_guard.as_ref()
12732 && let Err(err) = ensure_pool_meta_write_fence(pool_meta_guard, "decommission cancel failed")
12733 {
12734 save_guard.block_writes();
12735 return Err(err);
12736 }
12737 commit_decommission_cancel(&mut pool_meta, idx, commit)
12738 } else {
12739 Ok(())
12740 };
12741
12742 commit_result?;
12743 if let Some(pool_meta_guard) = _pool_meta_guard.as_ref()
12744 && let Err(err) = ensure_pool_meta_write_fence(pool_meta_guard, "decommission cancel failed")
12745 {
12746 save_guard.block_writes();
12747 return Err(err);
12748 }
12749 if let Some(canceler) = terminal_canceler.as_ref() {
12750 take_and_cancel_decommission_canceler_for_operation(cancelers.as_mut_slice(), idx, canceler);
12751 }
12752 drop(pool_meta);
12753 drop(cancelers);
12754 drop(_pool_meta_guard);
12755 drop(save_guard);
12756
12757 if changed {
12758 let movement_gate = self.ctx.data_movement_operation_gate();
12762 let _movement_guard = movement_gate.write().await;
12763 self.ctx
12764 .advance_data_movement_operation_epoch_to_durable_generation(durable_movement_generation);
12765 }
12766
12767 if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
12768 let stage = format!("decommission_cancel for pool {idx}");
12769 if let Err(err) =
12770 resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
12771 {
12772 warn!(
12773 event = EVENT_DECOMMISSION_STATE,
12774 component = LOG_COMPONENT_ECSTORE,
12775 subsystem = LOG_SUBSYSTEM_POOLS,
12776 pool_index = idx,
12777 state = "terminal_reload_failed",
12778 error = %err,
12779 "Decommission cancel saved locally but pool meta reload failed"
12780 );
12781 }
12782 }
12783
12784 Ok(())
12785 }
12786
12787 async fn release_decommission_canceler_slot(&self, idx: usize, owner: &DecommissionCanceler) {
12788 let mut cancelers = self.decommission_cancelers.write().await;
12789 take_and_cancel_decommission_canceler_for_operation(cancelers.as_mut_slice(), idx, owner);
12790 }
12791
12792 async fn decommission_terminal_retryable_for_operation(&self, idx: usize, owner: &DecommissionCanceler) -> bool {
12793 let _start_guard = self.start_gate.lock().await;
12794 let mut cancelers = self.decommission_cancelers.write().await;
12795 if !decommission_canceler_is_owned_by(cancelers.as_slice(), idx, owner) {
12796 owner.release();
12797 return false;
12798 }
12799
12800 let retryable = {
12801 let pool_meta = self.pool_meta.read().await;
12802 pool_meta
12803 .pools
12804 .get(idx)
12805 .and_then(|pool| pool.decommission.as_ref())
12806 .is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled)
12807 };
12808 if !retryable {
12809 take_and_cancel_decommission_canceler_for_operation(cancelers.as_mut_slice(), idx, owner);
12810 }
12811 retryable
12812 }
12813
12814 async fn retry_decommission_cancel_for_operation(self: &Arc<Self>, idx: usize, owner: &DecommissionCanceler) {
12815 if !self.decommission_terminal_retryable_for_operation(idx, owner).await {
12816 return;
12817 }
12818
12819 let mut attempt = 0usize;
12820 loop {
12821 if self
12822 .ensure_pool_meta_side_effects_safe("decommission cancel retry paused because pool metadata requires recovery")
12823 .await
12824 .is_err()
12825 {
12826 self.release_decommission_canceler_slot(idx, owner).await;
12827 return;
12828 }
12829 let Err(err) = self.decommission_cancel_for_operation(idx, owner).await else {
12830 return;
12831 };
12832 if !self.decommission_terminal_retryable_for_operation(idx, owner).await {
12833 return;
12834 }
12835 attempt += 1;
12836 warn!(
12837 event = EVENT_DECOMMISSION_STATE,
12838 component = LOG_COMPONENT_ECSTORE,
12839 subsystem = LOG_SUBSYSTEM_POOLS,
12840 pool_index = idx,
12841 state = "terminal_save_retry",
12842 terminal = "canceled",
12843 attempt,
12844 error = %err,
12845 "Decommission terminal save will be retried"
12846 );
12847 tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await;
12848 }
12849 }
12850
12851 async fn retry_decommission_failed_for_operation(&self, idx: usize, owner: &DecommissionCanceler) {
12852 let mut attempt = 0usize;
12853 loop {
12854 if self
12855 .ensure_pool_meta_side_effects_safe("decommission failure retry paused because pool metadata requires recovery")
12856 .await
12857 .is_err()
12858 {
12859 self.release_decommission_canceler_slot(idx, owner).await;
12860 return;
12861 }
12862 let Err(err) = self.decommission_failed_for_operation(idx, owner).await else {
12863 return;
12864 };
12865 if !self.decommission_terminal_retryable_for_operation(idx, owner).await {
12866 return;
12867 }
12868 attempt += 1;
12869 warn!(
12870 event = EVENT_DECOMMISSION_STATE,
12871 component = LOG_COMPONENT_ECSTORE,
12872 subsystem = LOG_SUBSYSTEM_POOLS,
12873 pool_index = idx,
12874 state = "terminal_save_retry",
12875 terminal = "failed",
12876 attempt,
12877 error = %err,
12878 "Decommission terminal save will be retried"
12879 );
12880 tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await;
12881 }
12882 }
12883
12884 async fn decommission_cancel_with_owner(self: &Arc<Self>, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> {
12885 let pools = self.pools.clone();
12886 let store = self.clone();
12887 let owner = owner.cloned();
12888 tokio::spawn(async move {
12889 store
12890 .decommission_cancel_transaction(idx, owner, true, move |snapshot, pool_meta_fence| async move {
12891 snapshot.save_no_lock_with_fence(pools, pool_meta_fence, &[idx]).await
12892 })
12893 .await
12894 })
12895 .await
12896 .map_err(|err| Error::other(format!("decommission cancel transaction task join error: {err}")))?
12897 }
12898
12899 #[cfg(test)]
12900 async fn clear_decommission_with_save<Save, SaveFuture>(self: &Arc<Self>, idx: usize, save_pool_meta: Save) -> Result<()>
12901 where
12902 Save: FnOnce() -> SaveFuture + Send + 'static,
12903 SaveFuture: Future<Output = Result<()>> + Send + 'static,
12904 {
12905 let store = self.clone();
12906 tokio::spawn(async move { store.clear_decommission_transaction(idx, save_pool_meta).await })
12907 .await
12908 .map_err(|err| Error::other(format!("clear decommission transaction task join error: {err}")))?
12909 }
12910
12911 #[tracing::instrument(skip(self))]
12912 pub async fn clear_decommission(self: &Arc<Self>, idx: usize) -> Result<()> {
12913 let store = self.clone();
12914 let save_store = store.clone();
12915 tokio::spawn(async move {
12918 store
12919 .clear_decommission_transaction(idx, move || async move { save_store.save_current_pool_meta(&[idx]).await })
12920 .await
12921 })
12922 .await
12923 .map_err(|err| Error::other(format!("clear decommission transaction task join error: {err}")))?
12924 }
12925
12926 async fn clear_decommission_transaction<Save, SaveFuture>(&self, idx: usize, save_pool_meta: Save) -> Result<()>
12927 where
12928 Save: FnOnce() -> SaveFuture,
12929 SaveFuture: Future<Output = Result<()>>,
12930 {
12931 ensure_decommission_terminal_operation_supported(self.single_pool(), "clear decommission")?;
12932 let _start_guard = self.start_gate.lock().await;
12933
12934 {
12935 let pool_meta = self.pool_meta.read().await;
12936 let pool_count = pool_meta.pools.len();
12937 ensure_valid_decommission_pool_index(pool_count, idx)?;
12938 let Some(pool) = pool_meta.pools.get(idx) else {
12939 return Err(invalid_decommission_pool_index_error(pool_count, idx));
12940 };
12941 let (decommission_present, complete, failed, canceled, unresolved_entries) = pool
12942 .decommission
12943 .as_ref()
12944 .map(|info| {
12945 (
12946 info.has_decommission_state(),
12947 info.complete,
12948 info.failed,
12949 info.canceled,
12950 info.unresolved_entries.len(),
12951 )
12952 })
12953 .unwrap_or((false, false, false, false, 0));
12954 ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled, unresolved_entries)?;
12955 }
12956 self.cancel_decommission_routines(&[idx]).await;
12959 let movement_gate = self.ctx.data_movement_operation_gate();
12960 let _movement_guard = movement_gate.write().await;
12961
12962 let (should_reload_pool_meta, previous_pool_meta) = {
12963 let rebalance_meta = self.rebalance_meta.read().await.clone();
12964 let terminal_at = OffsetDateTime::now_utc();
12965 let mut pool_meta = self.pool_meta.write().await;
12966 let previous_pool_meta = pool_meta.clone();
12967 let changed = pool_meta.clear_decommission_at(idx, terminal_at, rebalance_meta.as_ref())?;
12968 (changed, changed.then_some(previous_pool_meta))
12969 };
12970
12971 if should_reload_pool_meta && let Err(err) = save_pool_meta().await {
12972 if let Some(previous_pool_meta) = previous_pool_meta {
12973 let mut pool_meta = self.pool_meta.write().await;
12974 rollback_decommission_pool_meta(&mut pool_meta, &previous_pool_meta, &[idx]);
12975 }
12976 return Err(err);
12977 }
12978
12979 if should_reload_pool_meta {
12980 self.ctx.advance_data_movement_operation_epoch();
12981 }
12982 drop(_movement_guard);
12983
12984 if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
12985 let stage = format!("clear_decommission for pool {idx}");
12986 if let Err(err) =
12987 resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
12988 {
12989 warn!(
12990 event = EVENT_DECOMMISSION_STATE,
12991 component = LOG_COMPONENT_ECSTORE,
12992 subsystem = LOG_SUBSYSTEM_POOLS,
12993 pool_index = idx,
12994 state = "terminal_reload_failed",
12995 error = %err,
12996 "Decommission clear saved locally but pool meta reload failed"
12997 );
12998 }
12999 }
13000
13001 Ok(())
13002 }
13003
13004 async fn promote_queued_decommission(&self, idx: usize, owner: &DecommissionCanceler) -> Result<OffsetDateTime> {
13005 let (changed, generation, save_error) = {
13007 let _start_guard = self.start_gate.lock().await;
13008 let movement_gate = self.ctx.data_movement_operation_gate();
13009 let _movement_guard = movement_gate.write().await;
13010 let mut save_guard = self.pool_meta_save_gate.lock().await;
13011 let (pool_meta_guard, mut snapshot) = self
13012 .acquire_pool_meta_write_guard(&mut save_guard, "decommission promotion failed")
13013 .await?;
13014 let rebalance_meta = self.rebalance_meta.read().await.clone();
13015 let capacity_infos = if self.pools.is_empty() {
13016 Vec::new()
13017 } else {
13018 self.get_decommission_all_pool_capacity_infos().await?
13019 };
13020 let target_fence_proof = crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof();
13021 let mut pool_meta = self.pool_meta.write().await;
13022 if pool_meta.pools.get(idx).is_none() {
13023 return Err(Error::other("failed to start decommission: target pool was not found"));
13024 }
13025 let capacity_recovery_needed = active_decommission_source_indices(&pool_meta).into_iter().any(|source_idx| {
13026 pool_meta
13027 .pools
13028 .get(source_idx)
13029 .and_then(|pool| pool.decommission.as_ref())
13030 .and_then(|info| info.capacity_reservation.as_ref())
13031 .is_none_or(|reservation| !reservation.active())
13032 });
13033 let capacity_indices = if capacity_infos.is_empty() {
13034 Vec::new()
13035 } else {
13036 recover_decommission_capacity_reservations(
13037 &mut pool_meta,
13038 &capacity_infos,
13039 OffsetDateTime::now_utc(),
13040 target_fence_proof.is_some(),
13041 )?
13042 };
13043 let target_fence_proof_required = capacity_recovery_needed
13044 && active_decommission_capacity_model(&pool_meta)? == Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION);
13045 ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?;
13046 let reconciled = reconcile_decommission_meta_buckets(&mut pool_meta, idx);
13047 let promoted = pool_meta.promote_queued_decommission_at(idx, OffsetDateTime::now_utc(), rebalance_meta.as_ref());
13048 let mut changed_indices = capacity_indices;
13049 let changed = !changed_indices.is_empty() || reconciled || promoted;
13050 if changed {
13051 if !changed_indices.contains(&idx) {
13052 changed_indices.push(idx);
13053 }
13054 merge_pool_meta_updates_for_save(&mut snapshot, &pool_meta, &changed_indices, "decommission promotion failed")?;
13055 }
13056 drop(pool_meta);
13057
13058 let (save_outcome, save_error) = if changed {
13059 ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?;
13060 match snapshot
13061 .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &changed_indices)
13062 .await
13063 {
13064 Ok(outcome) => (Some(outcome), None),
13065 Err(err) => (None, Some(err)),
13066 }
13067 } else {
13068 (None, None)
13069 };
13070 let generation = self.active_decommission_generation(idx).await?;
13071 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission promotion failed")?;
13072 ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?;
13073 if let Some(outcome) = save_outcome {
13074 let mut pool_meta = self.pool_meta.write().await;
13075 pool_meta.version = pool_meta.version.max(outcome.committed.version);
13076 drop(pool_meta);
13077 outcome.disarm();
13078 }
13079 (changed, generation, save_error)
13080 };
13081
13082 if let Some(err) = save_error {
13083 resolve_decommission_terminal_mark_after_error_result(
13084 self.decommission_failed_for_operation(idx, owner).await,
13085 idx,
13086 &err,
13087 )?;
13088 return Err(err);
13089 }
13090
13091 if changed {
13092 self.ctx.advance_data_movement_operation_epoch();
13093 }
13094 if changed && let Some(notification_sys) = runtime_sources::notification_sys() {
13095 let stage = format!("promote_queued_decommission for pool {idx}");
13096 if let Err(err) =
13097 resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
13098 {
13099 resolve_decommission_terminal_mark_after_error_result(
13100 self.decommission_failed_for_operation(idx, owner).await,
13101 idx,
13102 &err,
13103 )?;
13104 return Err(err);
13105 }
13106 }
13107
13108 Ok(generation)
13109 }
13110
13111 #[cfg(test)]
13112 pub(crate) async fn promote_queued_decommission_for_test(&self, idx: usize) -> Result<()> {
13113 let owner = DecommissionCanceler::new(CancellationToken::new());
13114 self.promote_queued_decommission(idx, &owner).await?;
13115 let mut cancelers = self.decommission_cancelers.write().await;
13116 if let Some(slot) = cancelers.get_mut(idx)
13117 && let Some(previous) = slot.replace(owner)
13118 {
13119 previous.release();
13120 }
13121 Ok(())
13122 }
13123
13124 async fn record_decommission_terminal_reload_failure(&self, idx: usize, stage: &str, err: Error) -> Result<()> {
13125 let movement_gate = self.ctx.data_movement_operation_gate();
13126 let _movement_guard = movement_gate.write().await;
13127 let changed = {
13128 let rebalance_meta = self.rebalance_meta.read().await.clone();
13129 let mut pool_meta = self.pool_meta.write().await;
13130 pool_meta.record_decommission_terminal_reload_failure_at(
13131 idx,
13132 stage,
13133 err.to_string(),
13134 OffsetDateTime::now_utc(),
13135 rebalance_meta.as_ref(),
13136 )?
13137 };
13138
13139 if changed {
13140 self.save_current_pool_meta(&[idx]).await?;
13141 }
13142
13143 Ok(())
13144 }
13145
13146 async fn pause_decommission_for_capacity(&self, idx: usize, err: &Error) -> Result<()> {
13147 let mut save_guard = self.pool_meta_save_gate.lock().await;
13148 let (pool_meta_guard, mut snapshot) = self
13149 .acquire_pool_meta_write_guard(&mut save_guard, "decommission capacity pause failed")
13150 .await?;
13151 snapshot.mark_decommission_capacity_blocked(idx, err.to_string(), OffsetDateTime::now_utc())?;
13152 let outcome = snapshot
13153 .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx])
13154 .await?;
13155 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission capacity pause failed")?;
13156 {
13157 let mut pool_meta = self.pool_meta.write().await;
13158 publish_pool_meta_updates(&mut pool_meta, &outcome.committed, &[idx]);
13159 }
13160 ensure_pool_meta_write_fence(&pool_meta_guard, "decommission capacity pause failed")?;
13161 outcome.disarm();
13162 Ok(())
13163 }
13164
13165 pub async fn is_decommission_running(&self) -> bool {
13166 {
13167 let cancelers = self.decommission_cancelers.read().await;
13168 if has_active_decommission_canceler(cancelers.as_slice()) {
13169 return true;
13170 }
13171 }
13172
13173 let pool_meta = self.pool_meta.read().await;
13174 for pool in pool_meta.pools.iter() {
13175 if let Some(ref info) = pool.decommission
13176 && info.has_decommission_state()
13177 && !info.complete
13178 && !info.failed
13179 && !info.canceled
13180 {
13181 return true;
13182 }
13183 }
13184
13185 false
13186 }
13187
13188 async fn decommission_cancel_requested(&self, idx: usize, rx: &CancellationToken) -> bool {
13189 let pool_meta = self.pool_meta.read().await;
13190 is_decommission_cancel_requested(rx.is_cancelled(), pool_meta.pools.get(idx))
13191 }
13192
13193 #[cfg(test)]
13194 async fn cancel_decommission_routines_and_wait(&self, indices: &[usize]) {
13195 self.cancel_decommission_routines(indices).await;
13196 let movement_gate = self.ctx.data_movement_operation_gate();
13197 let _movement_guard = movement_gate.write().await;
13198 }
13199
13200 async fn cancel_decommission_routines(&self, indices: &[usize]) {
13201 {
13202 let mut cancelers = self.decommission_cancelers.write().await;
13203 for idx in indices {
13204 take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), *idx);
13205 }
13206 }
13207 }
13208
13209 async fn quiesce_decommission_worker_after_join_error(&self, canceler: &DecommissionCanceler) {
13210 canceler.cancel();
13211 let movement_gate = self.ctx.data_movement_operation_gate();
13212 let _movement_guard = movement_gate.write().await;
13213 }
13214
13215 async fn reserve_decommission_routines(
13216 &self,
13217 rx: &CancellationToken,
13218 indices: &[usize],
13219 ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
13220 let indices = dedup_indices(indices);
13221 if indices.is_empty() {
13222 return Ok(Vec::new());
13223 }
13224
13225 let _start_guard = self.start_gate.lock().await;
13226 let save_guard = self.pool_meta_save_gate.lock().await;
13227 save_guard.ensure_write_safe("decommission cannot be scheduled while pool metadata requires recovery")?;
13228 let indices = {
13229 let pool_meta = self.pool_meta.read().await;
13230 let indices = resumable_decommission_queue_indices(&pool_meta)
13231 .into_iter()
13232 .filter(|idx| indices.contains(idx))
13233 .collect::<Vec<_>>();
13234 if !indices.is_empty() {
13235 ensure_decommission_ledger_persistence_supported(&pool_meta)?;
13236 }
13237 indices
13238 };
13239 if indices.is_empty() {
13240 return Ok(Vec::new());
13241 }
13242
13243 let index_cancelers = {
13244 let mut cancelers = self.decommission_cancelers.write().await;
13245 let missing = missing_decommission_worker_prefix(indices.as_slice(), cancelers.as_slice());
13246 if missing.is_empty() {
13247 return Ok(Vec::new());
13248 }
13249 let bound = bind_missing_decommission_cancelers(missing.as_slice(), rx, cancelers.as_mut_slice());
13250 let guards = guard_decommission_cancelers(bound);
13251 ensure_decommission_routines_scheduled(guards.len(), missing.len())?;
13252 guards
13253 };
13254 Ok(index_cancelers)
13255 }
13256
13257 async fn reserve_missing_local_decommission_routines(
13258 &self,
13259 rx: &CancellationToken,
13260 endpoints: &EndpointServerPools,
13261 ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
13262 let indices = {
13263 let pool_meta = self.pool_meta.read().await;
13264 resumable_decommission_queue_indices(&pool_meta)
13265 };
13266 let indices = local_decommission_queue_prefix(endpoints, &indices)?;
13267 self.reserve_decommission_routines(rx, indices.as_slice()).await
13268 }
13269
13270 pub async fn spawn_missing_local_decommission_routines(self: &Arc<Self>) -> Result<()> {
13271 self.spawn_missing_local_decommission_routines_with_token(CancellationToken::new())
13272 .await
13273 }
13274
13275 pub(crate) async fn has_active_local_decommission_worker(&self) -> bool {
13276 let cancelers = self.decommission_cancelers.read().await;
13277 has_active_decommission_canceler(cancelers.as_slice())
13278 }
13279
13280 pub(crate) async fn spawn_missing_local_decommission_routines_with_token(
13281 self: &Arc<Self>,
13282 rx: CancellationToken,
13283 ) -> Result<()> {
13284 #[cfg(test)]
13285 let endpoints = self.instance_endpoints().unwrap_or_else(|| self.endpoints());
13286 #[cfg(not(test))]
13287 let endpoints = self.endpoints();
13288 let index_cancelers = self.reserve_missing_local_decommission_routines(&rx, &endpoints).await?;
13289 if index_cancelers.is_empty() {
13290 return Ok(());
13291 }
13292
13293 drop(spawn_decommission_index_cancelers(
13294 self.clone(),
13295 rx,
13296 index_cancelers,
13297 Arc::new(Semaphore::new(decommission_entry_concurrency_limit())),
13298 ));
13299 Ok(())
13300 }
13301
13302 #[tracing::instrument(skip(self, rx))]
13303 pub async fn decommission(&self, rx: CancellationToken, indices: Vec<usize>) -> Result<()> {
13304 let indices = dedup_indices(&indices);
13305
13306 info!(
13307 event = EVENT_DECOMMISSION_STATE,
13308 component = LOG_COMPONENT_ECSTORE,
13309 subsystem = LOG_SUBSYSTEM_POOLS,
13310 pool_indices = ?indices,
13311 state = "requested",
13312 "Decommission requested"
13313 );
13314 validate_start_decommission_request(&indices, self.single_pool())?;
13315
13316 self.ensure_decommission_rebalance_idle_after_refresh().await?;
13317
13318 let store = require_decommission_store(runtime_sources::object_store_handle(), "start decommission")?;
13319 let local_indices = local_decommission_queue_prefix(&self.endpoints(), &indices)?;
13320 let index_cancelers = self
13321 .start_decommission_with_routines(indices, &rx, local_indices.as_slice())
13322 .await?;
13323 drop(spawn_decommission_index_cancelers(
13324 store,
13325 rx,
13326 index_cancelers,
13327 Arc::new(Semaphore::new(decommission_entry_concurrency_limit())),
13328 ));
13329
13330 Ok(())
13331 }
13332
13333 async fn active_decommission_generation(&self, idx: usize) -> Result<OffsetDateTime> {
13334 let pool_meta = self.pool_meta.read().await;
13335 let Some(pool) = pool_meta.pools.get(idx) else {
13336 return Err(invalid_decommission_pool_index_error(pool_meta.pools.len(), idx));
13337 };
13338 let Some(info) = pool.decommission.as_ref() else {
13339 return Err(decommission_metadata_not_initialized_error("load decommission generation"));
13340 };
13341 let Some(generation) = info.start_time else {
13342 return Err(Error::OperationCanceled);
13343 };
13344 ensure_decommission_generation(&pool_meta, idx, generation)?;
13345 Ok(generation)
13346 }
13347
13348 async fn ensure_decommission_generation_current(&self, idx: usize, generation: OffsetDateTime) -> Result<()> {
13349 let pool_meta = self.pool_meta.read().await;
13350 ensure_decommission_generation(&pool_meta, idx, generation)
13351 }
13352
13353 #[allow(clippy::too_many_arguments)]
13354 async fn decommission_entry_worker(
13355 self: Arc<Self>,
13356 rx: CancellationToken,
13357 idx: usize,
13358 set_idx: usize,
13359 generation: OffsetDateTime,
13360 bucket: String,
13361 set: Arc<SetDisks>,
13362 lifecycle_config: Option<BucketLifecycleConfiguration>,
13363 object_lock_config: Option<ObjectLockConfiguration>,
13364 replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
13365 expected_bucket_incarnation_id: Option<uuid::Uuid>,
13366 source_changed_exhaustions: Arc<AtomicUsize>,
13367 entry_budget: Arc<Semaphore>,
13368 queue: Arc<tokio::sync::Mutex<mpsc::Receiver<QueuedDecommissionEntry>>>,
13369 entry_error: Arc<tokio::sync::Mutex<Option<Error>>>,
13370 ) {
13371 loop {
13372 let queued = tokio::select! {
13373 biased;
13374 _ = rx.cancelled() => return,
13375 item = async {
13376 let mut queue = queue.lock().await;
13377 queue.recv().await
13378 } => item,
13379 };
13380 let Some(QueuedDecommissionEntry { entry, queue_permit }) = queued else {
13381 return;
13382 };
13383 let object_name = entry.name.clone();
13384
13385 if entry_error.lock().await.is_some() {
13386 drop(queue_permit);
13387 continue;
13388 }
13389
13390 if let Err(err) = self.ensure_decommission_generation_current(idx, generation).await {
13391 if matches!(err, Error::OperationCanceled) {
13392 rx.cancel();
13393 } else {
13394 record_decommission_entry_error(&entry_error, &rx, err).await;
13395 }
13396 return;
13397 }
13398
13399 if let Err(err) = backpressure::wait_for_data_movement_admission(DataMovementOperation::Decommission, idx, &rx).await
13400 {
13401 if matches!(err, Error::OperationCanceled) {
13402 return;
13403 }
13404 error!(
13405 event = EVENT_DECOMMISSION_ENTRY,
13406 component = LOG_COMPONENT_ECSTORE,
13407 subsystem = LOG_SUBSYSTEM_POOLS,
13408 pool_index = idx,
13409 set_index = set_idx,
13410 bucket = %bucket,
13411 object = %object_name,
13412 state = "entry_admission_failed",
13413 error = %err,
13414 "Decommission entry admission failed"
13415 );
13416 record_decommission_entry_error(&entry_error, &rx, err).await;
13417 return;
13418 }
13419
13420 let entry_budget_permit = match tokio::select! {
13421 biased;
13422 _ = rx.cancelled() => return,
13423 permit = entry_budget.clone().acquire_owned() => permit,
13424 } {
13425 Ok(permit) => permit,
13426 Err(err) => {
13427 let err = Error::other(format!("decommission entry budget permit acquire failed: {err}"));
13428 error!(
13429 event = EVENT_DECOMMISSION_ENTRY,
13430 component = LOG_COMPONENT_ECSTORE,
13431 subsystem = LOG_SUBSYSTEM_POOLS,
13432 pool_index = idx,
13433 set_index = set_idx,
13434 bucket = %bucket,
13435 object = %object_name,
13436 state = "entry_budget_acquire_failed",
13437 error = %err,
13438 "Decommission entry budget permit acquire failed"
13439 );
13440 record_decommission_entry_error(&entry_error, &rx, err).await;
13441 return;
13442 }
13443 };
13444
13445 let result = self
13446 .decommission_entry(
13447 rx.clone(),
13448 idx,
13449 generation,
13450 entry,
13451 bucket.clone(),
13452 set.clone(),
13453 lifecycle_config.clone(),
13454 object_lock_config.clone(),
13455 replication_config.clone(),
13456 expected_bucket_incarnation_id,
13457 Arc::clone(&source_changed_exhaustions),
13458 )
13459 .await;
13460 drop(entry_budget_permit);
13461 drop(queue_permit);
13462
13463 if let Err(err) = result {
13464 error!(
13465 event = EVENT_DECOMMISSION_ENTRY,
13466 component = LOG_COMPONENT_ECSTORE,
13467 subsystem = LOG_SUBSYSTEM_POOLS,
13468 pool_index = idx,
13469 set_index = set_idx,
13470 bucket = %bucket,
13471 object = %object_name,
13472 state = "entry_failed",
13473 error = %err,
13474 "Decommission entry failed"
13475 );
13476 record_decommission_entry_error(&entry_error, &rx, err).await;
13477 return;
13478 }
13479 }
13480 }
13481
13482 #[allow(clippy::too_many_arguments)]
13483 async fn decommission_set(
13484 self: Arc<Self>,
13485 rx: CancellationToken,
13486 idx: usize,
13487 set_idx: usize,
13488 generation: OffsetDateTime,
13489 set: Arc<SetDisks>,
13490 bi: DecomBucketInfo,
13491 lifecycle_config: Option<BucketLifecycleConfiguration>,
13492 object_lock_config: Option<ObjectLockConfiguration>,
13493 replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
13494 expected_bucket_incarnation_id: Option<uuid::Uuid>,
13495 source_changed_exhaustions: Arc<AtomicUsize>,
13496 entry_budget: Arc<Semaphore>,
13497 entry_error: Arc<tokio::sync::Mutex<Option<Error>>>,
13498 ) -> Result<()> {
13499 let worker_count = DECOMMISSION_ENTRY_WORKERS_PER_SET;
13500 let queue_capacity = decommission_entry_queue_capacity(worker_count);
13501 let outstanding_capacity = queue_capacity.saturating_add(worker_count);
13502 let outstanding = Arc::new(Semaphore::new(outstanding_capacity));
13503 let (tx, rx_queue) = mpsc::channel(queue_capacity);
13504 let queue = Arc::new(tokio::sync::Mutex::new(rx_queue));
13505
13506 let mut entry_workers = tokio::task::JoinSet::new();
13507 for _ in 0..worker_count {
13508 let this = self.clone();
13509 let rx = rx.clone();
13510 let bucket = bi.name.clone();
13511 let set = set.clone();
13512 let lifecycle_config = lifecycle_config.clone();
13513 let object_lock_config = object_lock_config.clone();
13514 let replication_config = replication_config.clone();
13515 let source_changed_exhaustions = Arc::clone(&source_changed_exhaustions);
13516 let queue = queue.clone();
13517 let entry_budget = entry_budget.clone();
13518 let entry_error = entry_error.clone();
13519 entry_workers.spawn(async move {
13520 this.decommission_entry_worker(
13521 rx,
13522 idx,
13523 set_idx,
13524 generation,
13525 bucket,
13526 set,
13527 lifecycle_config,
13528 object_lock_config,
13529 replication_config,
13530 expected_bucket_incarnation_id,
13531 source_changed_exhaustions,
13532 entry_budget,
13533 queue,
13534 entry_error,
13535 )
13536 .await;
13537 });
13538 }
13539
13540 let callback: ListCallback = Arc::new({
13541 let tx = tx.clone();
13542 let outstanding = outstanding.clone();
13543 let callback_rx = rx.clone();
13544 let entry_error = entry_error.clone();
13545 let bucket = bi.name.clone();
13546 move |entry: MetaCacheEntry| {
13547 let tx = tx.clone();
13548 let outstanding = outstanding.clone();
13549 let callback_rx = callback_rx.clone();
13550 let entry_error = entry_error.clone();
13551 let bucket = bucket.clone();
13552 Box::pin(async move {
13553 if callback_rx.is_cancelled() || entry_error.lock().await.is_some() {
13554 return;
13555 }
13556
13557 if matches!(
13558 enqueue_decommission_entry(&callback_rx, &outstanding, &tx, entry).await,
13559 DecommissionEntryEnqueueResult::Closed
13560 ) {
13561 let err = Error::other("decommission entry queue closed");
13562 error!(
13563 event = EVENT_DECOMMISSION_ENTRY,
13564 component = LOG_COMPONENT_ECSTORE,
13565 subsystem = LOG_SUBSYSTEM_POOLS,
13566 pool_index = idx,
13567 set_index = set_idx,
13568 bucket = %bucket,
13569 state = "entry_queue_closed",
13570 error = %err,
13571 "Decommission entry queue closed"
13572 );
13573 record_decommission_entry_error(&entry_error, &callback_rx, err).await;
13574 }
13575 })
13576 }
13577 });
13578
13579 let list_set = set.clone();
13580 let list_rx = rx.clone();
13581 let list_rx_for_list = list_rx.clone();
13582 let list_rx_for_drain = list_rx.clone();
13583 let list_bi = bi.clone();
13584 let list_outstanding = outstanding.clone();
13585 let list_entry_error = entry_error.clone();
13586 let list_store = self.clone();
13587 let mut listing = tokio::spawn(async move {
13588 run_decommission_listing_with_retry_and_drain(
13589 list_rx.clone(),
13590 list_bi.name.clone(),
13591 callback,
13592 idx,
13593 set_idx,
13594 DECOMMISSION_LISTING_MAX_ATTEMPTS,
13595 move |callback| {
13596 let set = list_set.clone();
13597 let rx = list_rx_for_list.clone();
13598 let bucket = list_bi.clone();
13599 let entry_error = list_entry_error.clone();
13600 let store = list_store.clone();
13601 async move {
13602 set.list_objects_to_decommission(
13603 store,
13604 rx,
13605 bucket,
13606 callback,
13607 entry_error,
13608 idx,
13609 set_idx,
13610 generation,
13611 false,
13612 )
13613 .await
13614 }
13615 },
13616 move || {
13617 let rx = list_rx_for_drain.clone();
13618 let outstanding = list_outstanding.clone();
13619 async move { drain_decommission_entry_queue(&rx, &outstanding, outstanding_capacity).await }
13620 },
13621 )
13622 .await
13623 });
13624
13625 let mut listing_result = None;
13626 let mut workers_left = worker_count;
13627 let mut sender = Some(tx);
13628 while listing_result.is_none() || workers_left > 0 {
13629 tokio::select! {
13630 biased;
13631 result = &mut listing, if listing_result.is_none() => {
13632 let result = resolve_decommission_listing_worker_result(set_idx, result);
13633 if result.is_err() {
13634 rx.cancel();
13635 }
13636 listing_result = Some(result);
13637 drop(sender.take());
13638 }
13639 worker_result = entry_workers.join_next(), if workers_left > 0 => {
13640 workers_left -= 1;
13641 if let Some(Err(err)) = worker_result {
13642 let err = Error::other(format!("decommission entry worker {set_idx} task join error: {err}"));
13643 error!(
13644 event = EVENT_DECOMMISSION_ENTRY,
13645 component = LOG_COMPONENT_ECSTORE,
13646 subsystem = LOG_SUBSYSTEM_POOLS,
13647 pool_index = idx,
13648 set_index = set_idx,
13649 bucket = %bi.name,
13650 state = "entry_worker_join_failed",
13651 error = %err,
13652 "Decommission entry worker task failed"
13653 );
13654 record_decommission_entry_error(&entry_error, &rx, err).await;
13655 }
13656 }
13657 }
13658 }
13659
13660 let listing_result = listing_result.unwrap_or_else(|| Err(Error::other("decommission listing task did not complete")));
13661 if let Some(err) = entry_error.lock().await.clone() {
13662 return Err(err);
13663 }
13664 listing_result
13665 }
13666
13667 async fn track_decommission_entry_progress_stage(
13668 &self,
13669 idx: usize,
13670 generation: OffsetDateTime,
13671 bucket: &str,
13672 object: &str,
13673 stage: &'static str,
13674 ) -> Result<()> {
13675 {
13676 let mut pool_meta = self.pool_meta.write().await;
13677 ensure_decommission_generation(&pool_meta, idx, generation)?;
13678 track_decommission_current_object_stage(&mut pool_meta, idx, bucket, object, stage)
13679 .map_err(|err| with_decommission_entry_context(stage, bucket, object, err))?;
13680 }
13681
13682 Ok(())
13683 }
13684
13685 #[allow(clippy::too_many_arguments)]
13686 async fn wait_decommission_target_gate_retry(
13687 &self,
13688 rx: &CancellationToken,
13689 idx: usize,
13690 generation: OffsetDateTime,
13691 target_pool_index: usize,
13692 capacity_owner: Option<DecommissionCapacityOwner>,
13693 set: &SetDisks,
13694 entry: &MetaCacheEntry,
13695 bucket: &str,
13696 expected_version: &FileInfo,
13697 target_busy_attempt: usize,
13698 ) -> Result<Option<DecommissionCapacityTargetPermit>> {
13699 #[cfg(test)]
13700 notify_decommission_target_gate_retry(self.id);
13701 let mut wait_attempt = target_busy_attempt;
13702 let target_guard = loop {
13703 let retry_delay = decommission_retry_backoff_delay(
13704 DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY,
13705 wait_attempt.min(DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS),
13706 );
13707 if wait_decommission_retry_backoff(rx, retry_delay).await {
13708 decommission_cancel_signal_result(rx.is_cancelled())?;
13709 }
13710 if self.decommission_cancel_requested(idx, rx).await {
13711 rx.cancel();
13712 }
13713 decommission_cancel_signal_result(rx.is_cancelled())?;
13714 self.ensure_decommission_generation_current(idx, generation).await?;
13715
13716 match self.acquire_decommission_capacity_target_guard(target_pool_index).await {
13717 Ok(guard) => break guard,
13718 Err(err) if is_decommission_capacity_target_gate_busy(&err) => {
13719 wait_attempt = wait_attempt.saturating_add(1);
13720 }
13721 Err(err) => return Err(err),
13722 }
13723 };
13724
13725 #[cfg(test)]
13726 notify_decommission_target_gate_exact_reload(self.id);
13727 let current = load_decommission_entry_exact_versions(set, entry, bucket, "target_gate_retry").await?;
13728 ensure_decommission_capacity_target_fence(&target_guard, target_pool_index, "source identity revalidation")?;
13729 let expected_identity = data_movement::source_cleanup_version_identity(expected_version);
13730 let identity_current = current
13731 .versions
13732 .iter()
13733 .any(|version| data_movement::source_cleanup_version_identity(version) == expected_identity);
13734 if !identity_current {
13735 return Ok(None);
13736 }
13737 let owner = capacity_owner.ok_or_else(|| Error::other("target-gate retry is missing its decommission capacity owner"))?;
13738 let owner = owner.with_mutation_id(decommission_capacity_version_mutation_id(owner, bucket, expected_version));
13739 install_decommission_capacity_target_permit(self.id, target_pool_index, owner, target_guard).map(Some)
13740 }
13741
13742 #[cfg(feature = "test-util")]
13743 pub async fn prepare_scanner_pause_backlog_retirement_for_test(
13744 &self,
13745 source_pool_index: usize,
13746 source_bytes: usize,
13747 ) -> Result<()> {
13748 let source_bytes = source_bytes.max(1);
13749 let total = source_bytes.saturating_mul(8).saturating_add(64 * 1024);
13750 let capacities = (0..self.pools.len())
13751 .map(|pool_index| {
13752 DecommissionPoolCapacityInfo::for_test(
13753 pool_index,
13754 DecommissionErasureLayout { data: 1, parity: 0 },
13755 if pool_index == source_pool_index { 0 } else { total },
13756 total,
13757 if pool_index == source_pool_index { source_bytes } else { 0 },
13758 )
13759 })
13760 .collect();
13761 set_decommission_capacity_info_overrides_for_test(self.id, vec![capacities]);
13762 self.save_current_pool_meta_for_decommission_start(&[source_pool_index], Vec::new())
13763 .await
13764 .map(|_| ())
13765 }
13766
13767 #[cfg(feature = "test-util")]
13768 pub async fn retire_scanner_pause_backlog_for_test(
13769 self: &Arc<Self>,
13770 source_pool_index: usize,
13771 source_set_index: usize,
13772 ) -> Result<()> {
13773 let set = self
13774 .pools
13775 .get(source_pool_index)
13776 .and_then(|pool| pool.disk_set.get(source_set_index))
13777 .cloned()
13778 .ok_or_else(|| Error::other("scanner retirement test requested an unknown set"))?;
13779 let generation = self.active_decommission_generation(source_pool_index).await?;
13780 let owner = self
13781 .decommission_capacity_owner_for_worker(source_pool_index, generation)
13782 .await?;
13783 let expected = set
13784 .load_file_info_versions_exact(RUSTFS_META_BUCKET, data_movement::scanner_backlog::SCANNER_PAUSE_BACKLOG_PATH)
13785 .await?
13786 .unwrap_or_default();
13787 match self
13788 .retire_scanner_pause_backlog_entry(CancellationToken::new(), source_pool_index, generation, set, expected, owner)
13789 .await?
13790 {
13791 DecommissionEntryAttemptOutcome::Complete => Ok(()),
13792 DecommissionEntryAttemptOutcome::SourceChanged => Err(Error::other("scanner retirement source changed")),
13793 }
13794 }
13795
13796 #[cfg(feature = "test-util")]
13797 pub async fn stage_scanner_pause_backlog_retirement_intent_for_test(
13798 &self,
13799 source_pool_index: usize,
13800 source_set_index: usize,
13801 ) -> Result<()> {
13802 let generation = self.active_decommission_generation(source_pool_index).await?;
13803 let owner = self
13804 .decommission_capacity_owner_for_worker(source_pool_index, generation)
13805 .await?
13806 .ok_or_else(|| Error::other("scanner retirement test has no capacity owner"))?;
13807 let versions = self.pools[source_pool_index].disk_set[source_set_index]
13808 .load_file_info_versions_exact(RUSTFS_META_BUCKET, data_movement::scanner_backlog::SCANNER_PAUSE_BACKLOG_PATH)
13809 .await?
13810 .ok_or_else(|| Error::other("scanner retirement test source is missing"))?;
13811 let version = versions
13812 .versions
13813 .first()
13814 .ok_or_else(|| Error::other("scanner retirement test source is empty"))?;
13815 let owner = owner.with_mutation_id(decommission_capacity_version_mutation_id(owner, RUSTFS_META_BUCKET, version));
13816 let size = usize::try_from(version.size).map_err(|_| Error::other("scanner retirement test source size is invalid"))?;
13817 let target = self.select_decommission_capacity_target_pool(owner, size).await?;
13818 let failed: Result<()> = self
13819 .run_decommission_capacity_admitted_mutation(target, Some(owner), Some(size), || async {
13820 Err(Error::other("injected scanner retirement target failure"))
13821 })
13822 .await;
13823 match failed {
13824 Err(err) if err.to_string().contains("injected scanner retirement target failure") => Ok(()),
13825 Err(err) => Err(err),
13826 Ok(()) => Err(Error::other("scanner retirement test did not retain its target intent")),
13827 }
13828 }
13829
13830 #[allow(clippy::too_many_arguments)]
13831 async fn retire_scanner_pause_backlog_entry(
13832 self: &Arc<Self>,
13833 rx: CancellationToken,
13834 idx: usize,
13835 generation: OffsetDateTime,
13836 set: Arc<SetDisks>,
13837 expected: FileInfoVersions,
13838 capacity_owner: Option<DecommissionCapacityOwner>,
13839 ) -> Result<DecommissionEntryAttemptOutcome> {
13840 let store = Arc::clone(self);
13841 tokio::spawn(async move {
13844 let operation_gate = store.ctx.data_movement_operation_gate();
13845 store
13846 .run_guarded_decommission_side_effect(&rx, &operation_gate, || {
13847 store.retire_scanner_pause_backlog_entry_inner(idx, generation, set, &expected, capacity_owner)
13848 })
13849 .await
13850 })
13851 .await
13852 .map_err(Error::from)?
13853 }
13854
13855 async fn retire_scanner_pause_backlog_entry_inner(
13856 &self,
13857 idx: usize,
13858 generation: OffsetDateTime,
13859 set: Arc<SetDisks>,
13860 expected: &FileInfoVersions,
13861 capacity_owner: Option<DecommissionCapacityOwner>,
13862 ) -> Result<DecommissionEntryAttemptOutcome> {
13863 use data_movement::scanner_backlog::{
13864 SCANNER_PAUSE_BACKLOG_PATH, persist_native_scanner_pause_backlog_replica, plan_scanner_pause_backlog_retirement,
13865 read_scanner_pause_backlog_retirement_replicas,
13866 };
13867
13868 if expected.versions.is_empty() && expected.free_versions.is_empty() {
13869 return Ok(DecommissionEntryAttemptOutcome::Complete);
13872 }
13873 let [version] = expected.versions.as_slice() else {
13874 return Err(Error::other("scanner pause backlog retirement requires exactly one source version"));
13875 };
13876 if version.version_id.is_some_and(|version| !version.is_nil())
13877 || version.deleted
13878 || version.tier_free_version()
13879 || version.is_remote()
13880 || !expected.free_versions.is_empty()
13881 {
13882 return Err(Error::other(
13883 "scanner pause backlog retirement requires an unversioned local source record",
13884 ));
13885 }
13886 let object_fence = self
13887 .acquire_decommission_source_cleanup_fence(RUSTFS_META_BUCKET, SCANNER_PAUSE_BACKLOG_PATH, set.as_ref())
13888 .await?;
13889 let save_guard = self.pool_meta_save_gate.lock().await;
13892 let (pool_meta_guard, snapshot) = self
13893 .acquire_pool_meta_read_guard(&save_guard, "scanner pause backlog retirement admission failed")
13894 .await?;
13895 ensure_decommission_generation(&snapshot, idx, generation)?;
13896 let owner = capacity_owner.ok_or_else(|| {
13897 decommission_capacity_blocked_error("scanner pause backlog retirement has no active capacity owner")
13898 })?;
13899 let reservation = snapshot.pools[idx]
13900 .decommission
13901 .as_ref()
13902 .and_then(|info| info.capacity_reservation.as_ref())
13903 .filter(|reservation| reservation.admits_owner(owner, OffsetDateTime::now_utc()))
13904 .ok_or_else(|| decommission_capacity_blocked_error("scanner pause backlog retirement reservation is stale"))?;
13905 let mutation_id = decommission_capacity_version_mutation_id(owner, RUSTFS_META_BUCKET, version);
13906 if reservation.targets.iter().any(|target| {
13907 target.pending_mutation_id == Some(mutation_id)
13908 || target
13909 .temporary_mutations
13910 .iter()
13911 .any(|mutation| mutation.mutation_id == mutation_id)
13912 }) {
13913 return Err(decommission_capacity_blocked_error(
13914 "scanner pause backlog retirement has an unresolved target capacity intent",
13915 ));
13916 }
13917 if snapshot.scanner_pause_backlog_pool_writable(idx) {
13918 return Err(Error::other("scanner pause backlog source still belongs to writable membership"));
13919 }
13920 let sets: Vec<_> = self
13921 .pools
13922 .iter()
13923 .enumerate()
13924 .filter(|(pool_index, _)| *pool_index == idx || snapshot.scanner_pause_backlog_pool_writable(*pool_index))
13925 .flat_map(|(_, pool)| pool.disk_set.iter().cloned())
13926 .collect();
13927 let mut replicas = read_scanner_pause_backlog_retirement_replicas(idx, set.set_index, sets.clone()).await?;
13928 if let Some(plan) = plan_scanner_pause_backlog_retirement(idx, &replicas)? {
13929 let expected_stable_record = plan.stable_record.clone();
13930 let mut phases = Vec::with_capacity(3);
13931 if let Some(record) = plan.seed_record {
13932 phases.push(("seed", record));
13933 }
13934 phases.push(("commit", plan.commit_record));
13935 phases.push(("stabilize", plan.stable_record));
13936 for (phase, record) in phases {
13937 if object_fence.is_lock_lost()
13938 || pool_meta_guard.is_lock_lost()
13939 || !reservation.admits_owner(owner, OffsetDateTime::now_utc())
13940 {
13941 return Err(decommission_capacity_blocked_error(
13942 "scanner pause backlog retirement fence expired during native repair",
13943 ));
13944 }
13945 for read in replicas.iter().filter(|read| read.replica.pool_index != idx) {
13946 ensure_external_decommission_target_admission(
13947 &snapshot,
13948 read.replica.pool_index,
13949 DecommissionCapacityAdmission::ScannerBacklog,
13950 )?;
13951 }
13952 let results = join_all(replicas.iter().filter(|read| read.replica.pool_index != idx).map(|read| {
13953 let target = Arc::clone(&self.pools[read.replica.pool_index].disk_set[read.replica.set_index]);
13954 let record = record.clone();
13955 let object_fence = &object_fence;
13956 let pool_meta_guard = &pool_meta_guard;
13957 async move {
13958 let mut opts = ObjectOptions {
13959 no_lock: self.pools[0].disk_set[0].shares_namespace_lock_domain(&target).await,
13960 ..Default::default()
13961 };
13962 object_fence.add_namespace_lock_fence(&mut opts);
13963 opts.add_namespace_lock_guard(pool_meta_guard);
13964 persist_native_scanner_pause_backlog_replica(target, record, read.preconditions(), opts, phase).await
13965 }
13966 }))
13967 .await;
13968 for result in results {
13971 result?;
13972 }
13973 replicas = read_scanner_pause_backlog_retirement_replicas(idx, set.set_index, sets.clone()).await?;
13974 if replicas
13975 .iter()
13976 .filter(|read| read.replica.pool_index != idx)
13977 .any(|read| read.replica.data.as_deref() != Some(record.as_slice()))
13978 {
13979 return Err(Error::other(
13980 "scanner pause backlog native repair did not persist every surviving replica",
13981 ));
13982 }
13983 if let Some(next) = plan_scanner_pause_backlog_retirement(idx, &replicas)?
13984 && next.stable_record != expected_stable_record
13985 {
13986 return Err(Error::other("scanner pause backlog native authority changed during repair"));
13987 }
13988 }
13989 if plan_scanner_pause_backlog_retirement(idx, &replicas)?.is_some() {
13990 return Err(Error::other("scanner pause backlog native repair is not fully committed and stable"));
13991 }
13992 }
13993 if object_fence.is_lock_lost()
13994 || pool_meta_guard.is_lock_lost()
13995 || !reservation.admits_owner(owner, OffsetDateTime::now_utc())
13996 {
13997 return Err(decommission_capacity_blocked_error(
13998 "scanner pause backlog retirement fence expired before cleanup",
13999 ));
14000 }
14001 let result = data_movement::cleanup_source_entry_if_unchanged(
14002 set,
14003 RUSTFS_META_BUCKET,
14004 SCANNER_PAUSE_BACKLOG_PATH,
14005 expected,
14006 &[],
14007 data_movement::SourceCleanupBucketFence {
14008 expected_incarnation_id: None,
14009 lifecycle_guard: None,
14010 namespace_lock_lost_signal: pool_meta_guard.lock_lost_signal(),
14011 object_mutation_fence: Some(&object_fence),
14012 },
14013 "scanner pause backlog retirement",
14014 )
14015 .await;
14016 match result {
14017 Ok(_) => Ok(DecommissionEntryAttemptOutcome::Complete),
14018 Err(data_movement::SourceCleanupError::SourceChanged) => Ok(DecommissionEntryAttemptOutcome::SourceChanged),
14019 Err(data_movement::SourceCleanupError::Storage(err)) => Err(err),
14020 }
14021 }
14022
14023 #[allow(clippy::too_many_arguments)]
14024 #[tracing::instrument(
14025 level = "trace",
14026 skip_all,
14027 fields(
14028 event = EVENT_DECOMMISSION_ENTRY,
14029 component = LOG_COMPONENT_ECSTORE,
14030 subsystem = LOG_SUBSYSTEM_POOLS,
14031 state = "processing",
14032 pool_index = idx,
14033 bucket = %bucket,
14034 object = %entry.name,
14035 generation = %generation,
14036 )
14037 )]
14038 async fn decommission_entry(
14039 self: &Arc<Self>,
14040 rx: CancellationToken,
14041 idx: usize,
14042 generation: OffsetDateTime,
14043 entry: MetaCacheEntry,
14044 bucket: String,
14045 set: Arc<SetDisks>,
14046 lifecycle_config: Option<BucketLifecycleConfiguration>,
14047 object_lock_config: Option<ObjectLockConfiguration>,
14048 replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
14049 expected_bucket_incarnation_id: Option<uuid::Uuid>,
14050 source_changed_exhaustions: Arc<AtomicUsize>,
14051 ) -> Result<()> {
14052 let mut counted_versions = HashSet::new();
14053
14054 for entry_attempt in 1..=DECOMMISSION_ENTRY_MAX_ATTEMPTS {
14055 let attempt_result = {
14056 let mut conflict_attempt = 0;
14057 loop {
14058 let result = self
14059 .decommission_entry_attempt(
14060 rx.clone(),
14061 idx,
14062 generation,
14063 entry.clone(),
14064 bucket.clone(),
14065 Arc::clone(&set),
14066 lifecycle_config.clone(),
14067 object_lock_config.clone(),
14068 replication_config.clone(),
14069 expected_bucket_incarnation_id,
14070 entry_attempt,
14071 source_changed_exhaustions.as_ref(),
14072 &mut counted_versions,
14073 )
14074 .await;
14075 let retry = result
14076 .as_ref()
14077 .err()
14078 .and_then(|err| decommission_capacity_retry_kind(err, conflict_attempt));
14079 let retry_attempt = match retry {
14080 Some(DecommissionCapacityRetryKind::IntentConflict) => {
14081 conflict_attempt += 1;
14082 conflict_attempt
14083 }
14084 None => break result,
14085 };
14086 let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, retry_attempt);
14087 if wait_decommission_retry_backoff(&rx, retry_delay).await {
14088 decommission_cancel_signal_result(rx.is_cancelled())?;
14089 }
14090 }
14091 };
14092 match attempt_result {
14093 Ok(DecommissionEntryAttemptOutcome::Complete) => return Ok(()),
14094 Ok(DecommissionEntryAttemptOutcome::SourceChanged) => {
14095 let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, entry_attempt);
14096 warn!(
14097 event = EVENT_DECOMMISSION_ENTRY,
14098 component = LOG_COMPONENT_ECSTORE,
14099 subsystem = LOG_SUBSYSTEM_POOLS,
14100 state = "source_changed_retry",
14101 pool_index = idx,
14102 bucket = %bucket,
14103 object = %entry.name,
14104 attempt = entry_attempt,
14105 max_attempts = DECOMMISSION_ENTRY_MAX_ATTEMPTS,
14106 retry_delay_ms = retry_delay.as_millis(),
14107 "Decommission source changed during cleanup preflight; retrying entry"
14108 );
14109 if wait_decommission_retry_backoff(&rx, retry_delay).await {
14110 decommission_cancel_signal_result(rx.is_cancelled())?;
14111 }
14112 }
14113 Err(err) => return Err(err),
14114 }
14115 }
14116
14117 Err(Error::other(format!(
14118 "decommission entry retry loop ended without a terminal result for {bucket}/{}",
14119 entry.name
14120 )))
14121 }
14122
14123 #[allow(unused_assignments, clippy::too_many_arguments)]
14124 async fn decommission_entry_attempt(
14125 self: &Arc<Self>,
14126 rx: CancellationToken,
14127 idx: usize,
14128 generation: OffsetDateTime,
14129 entry: MetaCacheEntry,
14130 bucket: String,
14131 set: Arc<SetDisks>,
14132 lifecycle_config: Option<BucketLifecycleConfiguration>,
14133 object_lock_config: Option<ObjectLockConfiguration>,
14134 replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
14135 expected_bucket_incarnation_id: Option<uuid::Uuid>,
14136 entry_attempt: usize,
14137 source_changed_exhaustions: &AtomicUsize,
14138 counted_versions: &mut HashSet<(Option<uuid::Uuid>, bool)>,
14139 ) -> Result<DecommissionEntryAttemptOutcome> {
14140 debug!(
14141 event = EVENT_DECOMMISSION_ENTRY,
14142 component = LOG_COMPONENT_ECSTORE,
14143 subsystem = LOG_SUBSYSTEM_POOLS,
14144 state = "started",
14145 pool_index = idx,
14146 bucket = %bucket,
14147 object = %entry.name,
14148 attempt = entry_attempt,
14149 max_attempts = DECOMMISSION_ENTRY_MAX_ATTEMPTS,
14150 "Decommission entry started"
14151 );
14152 if entry.is_dir() {
14153 debug!(
14154 event = EVENT_DECOMMISSION_ENTRY,
14155 component = LOG_COMPONENT_ECSTORE,
14156 subsystem = LOG_SUBSYSTEM_POOLS,
14157 pool_index = idx,
14158 bucket = %bucket,
14159 object = %entry.name,
14160 state = "skipped_directory",
14161 "Decommission entry skipped directory"
14162 );
14163 return Ok(DecommissionEntryAttemptOutcome::Complete);
14164 }
14165 if is_decommission_set_local_usage_cache(&bucket, &entry.name) {
14169 return Ok(DecommissionEntryAttemptOutcome::Complete);
14170 }
14171 let durable_ilm_record = if bucket == RUSTFS_META_BUCKET {
14172 classify_durable_ilm_record(&entry.name)
14173 .map_err(|err| with_decommission_entry_context("durable_ilm_namespace", &bucket, &entry.name, err))?
14174 } else {
14175 None
14176 };
14177 if self.decommission_cancel_requested(idx, &rx).await {
14178 rx.cancel();
14179 }
14180 decommission_cancel_signal_result(rx.is_cancelled())?;
14181 self.ensure_decommission_generation_current(idx, generation).await?;
14182 self.ensure_decommission_runtime_capacity_available(idx, generation).await?;
14183 let capacity_owner = self.decommission_capacity_owner_for_worker(idx, generation).await?;
14184 let operation_gate = self.ctx.data_movement_operation_gate();
14185
14186 let bucket_incarnation_fence = match expected_bucket_incarnation_id {
14187 Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
14188 None => None,
14189 };
14190
14191 let mut fivs = load_decommission_entry_exact_versions(&set, &entry, &bucket, "file_info_versions").await?;
14192
14193 if data_movement::scanner_backlog::is_scanner_pause_backlog(&bucket, &entry.name) {
14194 let outcome = self
14195 .retire_scanner_pause_backlog_entry(rx, idx, generation, Arc::clone(&set), fivs.clone(), capacity_owner)
14196 .await?;
14197 if matches!(outcome, DecommissionEntryAttemptOutcome::Complete) {
14198 let mut pool_meta = self.pool_meta.write().await;
14199 ensure_decommission_generation(&pool_meta, idx, generation)?;
14200 if let Some(version) = fivs.versions.first()
14201 && counted_versions.insert((version.version_id, false))
14202 {
14203 count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), false)?;
14204 }
14205 track_decommission_current_object(&mut pool_meta, idx, &bucket, &entry.name)?;
14206 drop(pool_meta);
14207 self.track_decommission_entry_progress_stage(
14208 idx,
14209 generation,
14210 &bucket,
14211 &entry.name,
14212 DECOMMISSION_STAGE_ENTRY_FINISHED,
14213 )
14214 .await?;
14215 }
14216 return Ok(outcome);
14217 }
14218
14219 let pending_mutations = if let Some(owner) = capacity_owner {
14220 self.pool_meta
14221 .read()
14222 .await
14223 .pools
14224 .get(owner.source_pool_index)
14225 .and_then(|pool| pool.decommission.as_ref())
14226 .and_then(|info| info.capacity_reservation.as_ref())
14227 .filter(|reservation| reservation.admits_cleanup_owner(owner))
14228 .map(|reservation| {
14229 reservation
14230 .targets
14231 .iter()
14232 .filter_map(|target| target.pending_mutation_id)
14233 .collect::<HashSet<_>>()
14234 })
14235 .unwrap_or_default()
14236 } else {
14237 HashSet::new()
14238 };
14239 fivs.versions.sort_by_key(|version| {
14240 let mutation_id = capacity_owner.map(|owner| decommission_capacity_version_mutation_id(owner, &bucket, version));
14241 (
14242 mutation_id.is_none_or(|mutation_id| !pending_mutations.contains(&mutation_id)),
14243 version.mod_time.is_none(),
14244 std::cmp::Reverse(version.mod_time),
14245 )
14246 });
14247
14248 let mut decommissioned: usize = 0;
14249 let mut expired: usize = 0;
14250 let mut free_version_disposition = DecommissionFreeVersionDisposition::default();
14251 let mut cleanup_preflight_allowed_missing = Vec::new();
14252 let mut entry_blocked = false;
14253
14254 for version in fivs.versions.iter() {
14255 if self.decommission_cancel_requested(idx, &rx).await {
14256 rx.cancel();
14257 }
14258 decommission_cancel_signal_result(rx.is_cancelled())?;
14259
14260 if version.tier_free_version() {
14261 let version_id = version.version_id.map(|v| v.to_string());
14262 let mut migration_error = None;
14263 let mut migrated = false;
14264 let mut consumed = false;
14265 let mut capacity_failure = false;
14266 let mut version_attempt = 1;
14267 let mut target_busy_attempt: usize = 0;
14268 let mut target_permit = None;
14269 while version_attempt <= DECOMMISSION_VERSION_COPY_ATTEMPTS {
14270 let result = self
14271 .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
14272 self.decommission_tiered_object(
14273 bucket.as_str(),
14274 &version.name,
14275 version,
14276 &decommission_capacity_owned_opts(
14277 decommission_remote_tiered_opts(
14278 version,
14279 version_id.clone(),
14280 idx,
14281 expected_bucket_incarnation_id,
14282 ),
14283 capacity_owner,
14284 ),
14285 )
14286 .await
14287 })
14288 .await;
14289 drop(target_permit.take());
14290 if let Some(target_pool_index) = result.as_ref().err().and_then(decommission_capacity_target_gate_busy_index)
14291 {
14292 target_busy_attempt = target_busy_attempt.saturating_add(1);
14293 let Some(permit) = self
14294 .wait_decommission_target_gate_retry(
14295 &rx,
14296 idx,
14297 generation,
14298 target_pool_index,
14299 capacity_owner,
14300 set.as_ref(),
14301 &entry,
14302 &bucket,
14303 version,
14304 target_busy_attempt,
14305 )
14306 .await?
14307 else {
14308 return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
14309 };
14310 target_permit = Some(permit);
14311 continue;
14312 }
14313
14314 match classify_decommission_free_version_attempt(result) {
14315 DecommissionFreeVersionAttempt::Migrated => {
14316 migrated = true;
14317 migration_error = None;
14318 break;
14319 }
14320 DecommissionFreeVersionAttempt::Consumed => {
14321 consumed = true;
14322 migration_error = None;
14323 break;
14324 }
14325 DecommissionFreeVersionAttempt::CapacityFailure(err) => {
14326 capacity_failure = true;
14327 migration_error = Some(err);
14328 break;
14329 }
14330 DecommissionFreeVersionAttempt::Retry(err) => {
14331 migration_error = Some(err);
14332 version_attempt += 1;
14333 }
14334 }
14335 }
14336
14337 if counted_versions.insert((version.version_id, version.deleted)) {
14338 let mut pool_meta = self.pool_meta.write().await;
14339 ensure_decommission_generation(&pool_meta, idx, generation)?;
14340 if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, !migrated && !consumed) {
14341 return Err(with_decommission_entry_context(
14342 "count_decommission_item",
14343 bucket.as_str(),
14344 entry.name.as_str(),
14345 err,
14346 ));
14347 }
14348 }
14349
14350 if migrated || consumed {
14351 decommissioned += 1;
14352 cleanup_preflight_allowed_missing.push(data_movement::source_cleanup_version_identity(version));
14353 }
14354 if migrated {
14355 free_version_disposition.record_migrated();
14356 } else if consumed {
14357 free_version_disposition.record_consumed();
14358 } else {
14359 free_version_disposition.record_retained();
14360 }
14361
14362 debug!(
14363 event = EVENT_DECOMMISSION_ENTRY,
14364 component = LOG_COMPONENT_ECSTORE,
14365 subsystem = LOG_SUBSYSTEM_POOLS,
14366 pool_index = idx,
14367 bucket = %bucket,
14368 object = %version.name,
14369 version_id = ?version_id,
14370 result = ?migration_error,
14371 reason = if migrated {
14372 DECOMMISSION_FREE_VERSION_MIGRATED_REASON
14373 } else if consumed {
14374 DECOMMISSION_FREE_VERSION_CONSUMED_REASON
14375 } else {
14376 DECOMMISSION_FREE_VERSION_RETAINED_REASON
14377 },
14378 state = if migrated {
14379 "free_version_migrated"
14380 } else if consumed {
14381 "free_version_consumed"
14382 } else {
14383 "free_version_retained"
14384 },
14385 "Decommission free-version disposition recorded"
14386 );
14387
14388 if capacity_failure {
14389 return Err(with_decommission_entry_context(
14390 "decommission_tier_free_version",
14391 bucket.as_str(),
14392 version.name.as_str(),
14393 migration_error.expect("capacity failure must retain its error"),
14394 ));
14395 }
14396
14397 if !migrated && !consumed {
14398 break;
14399 }
14400 continue;
14401 }
14402
14403 if self
14404 .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
14405 should_skip_lifecycle_for_data_movement(
14406 self.clone(),
14407 &bucket,
14408 version,
14409 lifecycle_config.as_ref(),
14410 object_lock_config.as_ref(),
14411 true,
14412 &LcEventSrc::Decom,
14413 None,
14414 )
14415 .await
14416 })
14417 .await
14418 .map_err(|err| with_decommission_entry_context("lifecycle_expiry", bucket.as_str(), version.name.as_str(), err))?
14419 {
14420 expired += 1;
14421 cleanup_preflight_allowed_missing.push(data_movement::source_cleanup_version_identity(version));
14422 continue;
14423 }
14424
14425 let remaining_versions = decommission_remaining_version_count(&fivs.versions, expired);
14426 if should_skip_decommission_delete_marker(version, remaining_versions, replication_config.is_some()) {
14427 decommissioned += 1;
14429 debug!(
14430 event = EVENT_DECOMMISSION_ENTRY,
14431 component = LOG_COMPONENT_ECSTORE,
14432 subsystem = LOG_SUBSYSTEM_POOLS,
14433 pool_index = idx,
14434 bucket = %bucket,
14435 object = %version.name,
14436 state = "skipped_delete_marker",
14437 "Decommission delete marker skipped"
14438 );
14439 continue;
14440 }
14441
14442 let version_id = version.version_id.map(|v| v.to_string());
14443
14444 let mut ignore = false;
14445 let mut cleanup_ignored = false;
14446 let mut failure = false;
14447 let mut error = None;
14448 if version.deleted {
14449 let mut version_attempt = 1;
14450 let mut target_busy_attempt: usize = 0;
14451 let mut target_permit = None;
14452 while version_attempt <= DECOMMISSION_VERSION_COPY_ATTEMPTS {
14453 let result = self
14454 .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
14455 self.delete_object(
14456 bucket.as_str(),
14457 &version.name,
14458 decommission_capacity_owned_opts(
14459 decommission_delete_marker_opts(
14460 version,
14461 version_id.clone(),
14462 idx,
14463 expected_bucket_incarnation_id,
14464 ),
14465 capacity_owner,
14466 ),
14467 )
14468 .await
14469 })
14470 .await;
14471 drop(target_permit.take());
14472 if let Some(target_pool_index) = result.as_ref().err().and_then(decommission_capacity_target_gate_busy_index)
14473 {
14474 target_busy_attempt = target_busy_attempt.saturating_add(1);
14475 let Some(permit) = self
14476 .wait_decommission_target_gate_retry(
14477 &rx,
14478 idx,
14479 generation,
14480 target_pool_index,
14481 capacity_owner,
14482 set.as_ref(),
14483 &entry,
14484 &bucket,
14485 version,
14486 target_busy_attempt,
14487 )
14488 .await?
14489 else {
14490 return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
14491 };
14492 target_permit = Some(permit);
14493 continue;
14494 }
14495 #[cfg(test)]
14496 let result = decommission_test_wrap_result(
14497 "delete_marker_copy",
14498 bucket.as_str(),
14499 version.name.as_str(),
14500 version_attempt,
14501 result,
14502 );
14503
14504 match result {
14505 Ok(_) => {
14506 failure = false;
14507 error = None;
14508 break;
14509 }
14510 Err(err) if is_decommission_copy_cleanup_safe_error(&err) => {
14511 warn!(
14512 event = EVENT_DECOMMISSION_ENTRY,
14513 component = LOG_COMPONENT_ECSTORE,
14514 subsystem = LOG_SUBSYSTEM_POOLS,
14515 state = "ignored_delete_marker_copy",
14516 pool_index = idx,
14517 bucket = %bucket,
14518 object = %version.name,
14519 version_id = ?version_id,
14520 error = ?err,
14521 "Decommission delete marker copy ignored"
14522 );
14523 ignore = true;
14524 cleanup_ignored = true;
14525 break;
14526 }
14527 Err(err) if is_decommission_target_capacity_error(&err) => {
14528 return Err(with_decommission_entry_context(
14529 "delete_marker_copy",
14530 bucket.as_str(),
14531 version.name.as_str(),
14532 err,
14533 ));
14534 }
14535 Err(err) => {
14536 failure = true;
14537 if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS {
14538 error!(
14539 event = EVENT_DECOMMISSION_ENTRY,
14540 component = LOG_COMPONENT_ECSTORE,
14541 subsystem = LOG_SUBSYSTEM_POOLS,
14542 state = "delete_marker_copy_failed",
14543 pool_index = idx,
14544 bucket = %bucket,
14545 object = %version.name,
14546 version_id = ?version_id,
14547 attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
14548 error = ?err,
14549 "Decommission delete marker copy failed"
14550 );
14551 error = Some(err);
14552 break;
14553 }
14554
14555 let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt);
14556 warn!(
14557 event = EVENT_DECOMMISSION_ENTRY,
14558 component = LOG_COMPONENT_ECSTORE,
14559 subsystem = LOG_SUBSYSTEM_POOLS,
14560 state = "delete_marker_copy_retry",
14561 pool_index = idx,
14562 bucket = %bucket,
14563 object = %version.name,
14564 version_id = ?version_id,
14565 attempt = version_attempt,
14566 max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
14567 retry_delay_ms = retry_delay.as_millis(),
14568 error = ?err,
14569 "Decommission delete marker copy failed; retrying"
14570 );
14571 error = Some(err);
14572 if wait_decommission_retry_backoff(&rx, retry_delay).await {
14573 decommission_cancel_signal_result(rx.is_cancelled())?;
14574 }
14575 version_attempt += 1;
14576 }
14577 }
14578 }
14579
14580 if ignore {
14581 if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
14582 decommissioned += 1;
14583 }
14584 debug!(
14585 event = EVENT_DECOMMISSION_ENTRY,
14586 component = LOG_COMPONENT_ECSTORE,
14587 subsystem = LOG_SUBSYSTEM_POOLS,
14588 pool_index = idx,
14589 bucket = %bucket,
14590 object = %version.name,
14591 state = "ignored",
14592 "Decommission entry ignored"
14593 );
14594 continue;
14595 }
14596
14597 if counted_versions.insert((version.version_id, version.deleted)) {
14598 let mut pool_meta = self.pool_meta.write().await;
14599 ensure_decommission_generation(&pool_meta, idx, generation)?;
14600 if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, failure) {
14601 return Err(with_decommission_entry_context(
14602 "count_decommission_item",
14603 bucket.as_str(),
14604 entry.name.as_str(),
14605 err,
14606 ));
14607 }
14608 }
14609
14610 if !failure {
14611 decommissioned += 1;
14612 }
14613
14614 if !failure {
14615 debug!(
14616 event = EVENT_DECOMMISSION_ENTRY,
14617 component = LOG_COMPONENT_ECSTORE,
14618 subsystem = LOG_SUBSYSTEM_POOLS,
14619 state = "delete_marker_copied",
14620 pool_index = idx,
14621 bucket = %bucket,
14622 object = %version.name,
14623 version_id = ?version_id,
14624 result = ?error,
14625 "Decommission delete marker copied"
14626 );
14627 }
14628 continue;
14629 }
14630
14631 let mut version_attempt = 1;
14632 let mut target_busy_attempt: usize = 0;
14633 let mut target_permit = None;
14634 while version_attempt <= DECOMMISSION_VERSION_COPY_ATTEMPTS {
14635 if version.is_remote() {
14636 let result = self
14637 .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
14638 self.decommission_tiered_object(
14639 bucket.as_str(),
14640 &version.name,
14641 version,
14642 &decommission_capacity_owned_opts(
14643 decommission_remote_tiered_opts(
14644 version,
14645 version_id.clone(),
14646 idx,
14647 expected_bucket_incarnation_id,
14648 ),
14649 capacity_owner,
14650 ),
14651 )
14652 .await
14653 })
14654 .await;
14655 drop(target_permit.take());
14656 if let Some(target_pool_index) = result.as_ref().err().and_then(decommission_capacity_target_gate_busy_index)
14657 {
14658 target_busy_attempt = target_busy_attempt.saturating_add(1);
14659 let Some(permit) = self
14660 .wait_decommission_target_gate_retry(
14661 &rx,
14662 idx,
14663 generation,
14664 target_pool_index,
14665 capacity_owner,
14666 set.as_ref(),
14667 &entry,
14668 &bucket,
14669 version,
14670 target_busy_attempt,
14671 )
14672 .await?
14673 else {
14674 return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
14675 };
14676 target_permit = Some(permit);
14677 continue;
14678 }
14679 #[cfg(test)]
14680 let result = decommission_test_wrap_result(
14681 "decommission_tiered_object",
14682 bucket.as_str(),
14683 version.name.as_str(),
14684 version_attempt,
14685 result,
14686 );
14687
14688 match result {
14689 Ok(_) => {
14690 failure = false;
14691 error = None;
14692 }
14693 Err(err) if is_decommission_copy_cleanup_safe_error(&err) => {
14694 ignore = true;
14695 cleanup_ignored = true;
14696 }
14697 Err(err) if is_decommission_target_capacity_error(&err) => {
14698 return Err(with_decommission_entry_context(
14699 "decommission_tiered_object",
14700 bucket.as_str(),
14701 version.name.as_str(),
14702 err,
14703 ));
14704 }
14705 Err(err) => {
14706 failure = true;
14707 if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS {
14708 error!(
14709 event = EVENT_DECOMMISSION_ENTRY,
14710 component = LOG_COMPONENT_ECSTORE,
14711 subsystem = LOG_SUBSYSTEM_POOLS,
14712 state = "tiered_copy_failed",
14713 pool_index = idx,
14714 bucket = %bucket,
14715 object = %version.name,
14716 version_id = ?version_id,
14717 attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
14718 error = ?err,
14719 "Decommission tiered version copy failed"
14720 );
14721 error = Some(err);
14722 } else {
14723 let retry_delay =
14724 decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt);
14725 warn!(
14726 event = EVENT_DECOMMISSION_ENTRY,
14727 component = LOG_COMPONENT_ECSTORE,
14728 subsystem = LOG_SUBSYSTEM_POOLS,
14729 state = "tiered_copy_retry",
14730 pool_index = idx,
14731 bucket = %bucket,
14732 object = %version.name,
14733 version_id = ?version_id,
14734 attempt = version_attempt,
14735 max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
14736 retry_delay_ms = retry_delay.as_millis(),
14737 error = ?err,
14738 "Decommission tiered version copy failed; retrying"
14739 );
14740 error = Some(err);
14741 if wait_decommission_retry_backoff(&rx, retry_delay).await {
14742 decommission_cancel_signal_result(rx.is_cancelled())?;
14743 }
14744 version_attempt += 1;
14745 continue;
14746 }
14747 }
14748 }
14749 break;
14750 }
14751
14752 let bucket = bucket.clone();
14753
14754 let read_result = set
14755 .get_object_reader(
14756 bucket.as_str(),
14757 &encode_dir_object(&version.name),
14758 None,
14759 HeaderMap::new(),
14760 &decommission_object_migration_read_opts(version_id.clone()),
14761 )
14762 .await;
14763 #[cfg(test)]
14764 let read_result =
14765 decommission_test_wrap_result("object_read", &bucket, &version.name, version_attempt, read_result);
14766 let rd = match read_result {
14767 Ok(rd) => rd,
14768 Err(err) => {
14769 if is_err_object_not_found(&err) || is_err_version_not_found(&err) {
14770 ignore = true;
14771 cleanup_ignored = true;
14772 break;
14773 }
14774
14775 failure = true;
14776 if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS {
14777 error!(
14778 event = EVENT_DECOMMISSION_ENTRY,
14779 component = LOG_COMPONENT_ECSTORE,
14780 subsystem = LOG_SUBSYSTEM_POOLS,
14781 state = "object_read_failed",
14782 pool_index = idx,
14783 bucket = %bucket,
14784 object = %version.name,
14785 version_id = ?version_id,
14786 attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
14787 error = ?err,
14788 "Decommission source object read failed"
14789 );
14790 error = Some(err);
14791 break;
14792 }
14793
14794 let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt);
14795 warn!(
14796 event = EVENT_DECOMMISSION_ENTRY,
14797 component = LOG_COMPONENT_ECSTORE,
14798 subsystem = LOG_SUBSYSTEM_POOLS,
14799 state = "object_read_retry",
14800 pool_index = idx,
14801 bucket = %bucket,
14802 object = %version.name,
14803 version_id = ?version_id,
14804 attempt = version_attempt,
14805 max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
14806 retry_delay_ms = retry_delay.as_millis(),
14807 error = ?err,
14808 "Decommission source object read failed; retrying"
14809 );
14810 error = Some(err);
14811 if wait_decommission_retry_backoff(&rx, retry_delay).await {
14812 decommission_cancel_signal_result(rx.is_cancelled())?;
14813 }
14814 version_attempt += 1;
14815 continue;
14816 }
14817 };
14818
14819 let bucket_name = bucket.clone();
14820 let object_name = rd.object_info.name.clone();
14821
14822 self.track_decommission_entry_progress_stage(
14823 idx,
14824 generation,
14825 bucket_name.as_str(),
14826 object_name.as_str(),
14827 DECOMMISSION_STAGE_MIGRATE_OBJECT,
14828 )
14829 .await?;
14830
14831 let migrate_result = self
14832 .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
14833 self.clone()
14834 .decommission_object(idx, bucket, rd, expected_bucket_incarnation_id, capacity_owner)
14835 .await
14836 })
14837 .await;
14838 drop(target_permit.take());
14839 if let Some(target_pool_index) = migrate_result
14840 .as_ref()
14841 .err()
14842 .and_then(decommission_capacity_target_gate_busy_index)
14843 {
14844 target_busy_attempt = target_busy_attempt.saturating_add(1);
14845 let Some(permit) = self
14846 .wait_decommission_target_gate_retry(
14847 &rx,
14848 idx,
14849 generation,
14850 target_pool_index,
14851 capacity_owner,
14852 set.as_ref(),
14853 &entry,
14854 &bucket_name,
14855 version,
14856 target_busy_attempt,
14857 )
14858 .await?
14859 else {
14860 return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
14861 };
14862 target_permit = Some(permit);
14863 continue;
14864 }
14865 #[cfg(test)]
14866 let migrate_result = decommission_test_wrap_result(
14867 DECOMMISSION_STAGE_MIGRATE_OBJECT,
14868 bucket_name.as_str(),
14869 object_name.as_str(),
14870 version_attempt,
14871 migrate_result,
14872 );
14873
14874 if let Err(err) = migrate_result {
14875 if is_decommission_copy_cleanup_safe_error(&err) {
14876 ignore = true;
14877 cleanup_ignored = true;
14878 break;
14879 }
14880
14881 if is_decommission_target_capacity_error(&err) {
14882 return Err(with_decommission_entry_context(
14883 DECOMMISSION_STAGE_MIGRATE_OBJECT,
14884 bucket_name.as_str(),
14885 object_name.as_str(),
14886 err,
14887 ));
14888 }
14889
14890 failure = true;
14891 if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS {
14892 error!(
14893 event = EVENT_DECOMMISSION_ENTRY,
14894 component = LOG_COMPONENT_ECSTORE,
14895 subsystem = LOG_SUBSYSTEM_POOLS,
14896 state = "object_migration_failed",
14897 pool_index = idx,
14898 bucket = %bucket_name,
14899 object = %object_name,
14900 version = %version.name,
14901 attempt = version_attempt,
14902 max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
14903 error = ?err,
14904 "Decommission object migration failed"
14905 );
14906 error = Some(err);
14907 break;
14908 }
14909
14910 let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt);
14911 warn!(
14912 event = EVENT_DECOMMISSION_ENTRY,
14913 component = LOG_COMPONENT_ECSTORE,
14914 subsystem = LOG_SUBSYSTEM_POOLS,
14915 state = "object_migration_retry",
14916 pool_index = idx,
14917 bucket = %bucket_name,
14918 object = %object_name,
14919 version = %version.name,
14920 attempt = version_attempt,
14921 max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
14922 retry_delay_ms = retry_delay.as_millis(),
14923 error = ?err,
14924 "Decommission object migration failed; retrying"
14925 );
14926 error = Some(err);
14927 if wait_decommission_retry_backoff(&rx, retry_delay).await {
14928 decommission_cancel_signal_result(rx.is_cancelled())?;
14929 }
14930 version_attempt += 1;
14931 continue;
14932 }
14933
14934 warn!(
14935 event = EVENT_DECOMMISSION_ENTRY,
14936 component = LOG_COMPONENT_ECSTORE,
14937 subsystem = LOG_SUBSYSTEM_POOLS,
14938 pool_index = idx,
14939 bucket = %bucket_name,
14940 object = %object_name,
14941 version = %version.name,
14942 state = "object_migrated",
14943 "Decommission object migrated"
14944 );
14945
14946 failure = false;
14947 break;
14948 }
14949
14950 if ignore {
14951 if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
14952 decommissioned += 1;
14953 }
14954 debug!(
14955 event = EVENT_DECOMMISSION_ENTRY,
14956 component = LOG_COMPONENT_ECSTORE,
14957 subsystem = LOG_SUBSYSTEM_POOLS,
14958 pool_index = idx,
14959 bucket = %bucket,
14960 object = %version.name,
14961 state = "ignored",
14962 "Decommission entry ignored"
14963 );
14964 continue;
14965 }
14966
14967 if counted_versions.insert((version.version_id, version.deleted)) {
14968 let mut pool_meta = self.pool_meta.write().await;
14969 ensure_decommission_generation(&pool_meta, idx, generation)?;
14970 if let Err(err) = count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), failure) {
14971 return Err(with_decommission_entry_context(
14972 "count_decommission_item",
14973 bucket.as_str(),
14974 entry.name.as_str(),
14975 err,
14976 ));
14977 }
14978 }
14979
14980 if failure {
14981 break;
14982 }
14983
14984 if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
14985 decommissioned += 1;
14986 }
14987 }
14988
14989 if free_version_disposition.total() > 0 {
14990 debug!(
14991 event = EVENT_DECOMMISSION_ENTRY,
14992 component = LOG_COMPONENT_ECSTORE,
14993 subsystem = LOG_SUBSYSTEM_POOLS,
14994 pool_index = idx,
14995 bucket = %bucket,
14996 object = %entry.name,
14997 free_versions_migrated = free_version_disposition.migrated,
14998 free_versions_consumed = free_version_disposition.consumed,
14999 free_versions_retained = free_version_disposition.retained,
15000 free_versions_total = free_version_disposition.total(),
15001 reason = DECOMMISSION_FREE_VERSION_DISPOSITION_REASON,
15002 state = "free_version_disposition",
15003 "Decommission free-version disposition summary"
15004 );
15005 }
15006
15007 if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) && durable_ilm_record.is_none()
15008 {
15009 if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
15010 return Err(Error::other("decommission bucket incarnation fence was lost before source cleanup"));
15011 }
15012 decommission_cancel_signal_result(rx.is_cancelled())?;
15013 self.ensure_decommission_generation_current(idx, generation).await?;
15014
15015 self.track_decommission_entry_progress_stage(
15016 idx,
15017 generation,
15018 bucket.as_str(),
15019 entry.name.as_str(),
15020 DECOMMISSION_STAGE_CLEANUP_PREFLIGHT,
15021 )
15022 .await?;
15023
15024 self.track_decommission_entry_progress_stage(
15025 idx,
15026 generation,
15027 bucket.as_str(),
15028 entry.name.as_str(),
15029 DECOMMISSION_STAGE_SOURCE_CLEANUP,
15030 )
15031 .await?;
15032
15033 #[cfg(test)]
15034 run_decommission_cleanup_mutation_hook(bucket.as_str(), entry.name.as_str(), entry_attempt).await;
15035
15036 let source_cleanup_mutation_fence = self
15037 .acquire_decommission_source_cleanup_fence(bucket.as_str(), entry.name.as_str(), set.as_ref())
15038 .await?;
15039 let cleanup_result = self
15040 .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
15041 data_movement::cleanup_source_entry_if_unchanged(
15042 set.clone(),
15043 bucket.as_str(),
15044 entry.name.as_str(),
15045 &fivs,
15046 &cleanup_preflight_allowed_missing,
15047 data_movement::SourceCleanupBucketFence {
15048 expected_incarnation_id: expected_bucket_incarnation_id,
15049 lifecycle_guard: bucket_incarnation_fence
15050 .as_ref()
15051 .and_then(|guard| guard.namespace_lock_guard()),
15052 namespace_lock_lost_signal: None,
15053 object_mutation_fence: Some(&source_cleanup_mutation_fence),
15054 },
15055 "decommission",
15056 )
15057 .await
15058 })
15059 .await;
15060 match cleanup_result {
15061 Ok(_) => {}
15062 Err(data_movement::SourceCleanupError::Storage(err)) => {
15063 resolve_decommission_entry_cleanup_delete_result(
15064 Err::<(), Error>(err),
15065 bucket.as_str(),
15066 entry.name.as_str(),
15067 )?;
15068 }
15069 Err(data_movement::SourceCleanupError::SourceChanged) if entry_attempt < DECOMMISSION_ENTRY_MAX_ATTEMPTS => {
15070 return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
15071 }
15072 Err(data_movement::SourceCleanupError::SourceChanged) => {
15073 let exhausted_entries = source_changed_exhaustions.fetch_add(1, Ordering::Relaxed) + 1;
15074 if should_fail_decommission_pool_after_exhausted_source_changed(exhausted_entries) {
15075 return Err(Error::other(format!(
15076 "decommission source cleanup retries exhausted for {}/{} on all {} attempts; exhausted entries exceed pool limit {}",
15077 bucket, entry.name, DECOMMISSION_ENTRY_MAX_ATTEMPTS, DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT
15078 )));
15079 }
15080
15081 {
15082 let mut pool_meta = self.pool_meta.write().await;
15083 ensure_decommission_generation(&pool_meta, idx, generation)?;
15084 count_decommission_item(&mut pool_meta, idx, 0, true).map_err(|err| {
15085 with_decommission_entry_context(
15086 "count_source_changed_exhaustion",
15087 bucket.as_str(),
15088 entry.name.as_str(),
15089 err,
15090 )
15091 })?;
15092 }
15093
15094 error!(
15095 event = EVENT_DECOMMISSION_ENTRY,
15096 component = LOG_COMPONENT_ECSTORE,
15097 subsystem = LOG_SUBSYSTEM_POOLS,
15098 state = "source_cleanup_exhausted",
15099 pool_index = idx,
15100 bucket = %bucket,
15101 object = %entry.name,
15102 attempts = DECOMMISSION_ENTRY_MAX_ATTEMPTS,
15103 exhausted_entries,
15104 exhaustion_limit = DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT,
15105 "Decommission source cleanup retries exhausted; source retained and entry marked failed"
15106 );
15107 entry_blocked = true;
15108 }
15109 }
15110 } else if durable_ilm_record.is_some() {
15111 debug!(
15112 event = EVENT_DECOMMISSION_ENTRY,
15113 component = LOG_COMPONENT_ECSTORE,
15114 subsystem = LOG_SUBSYSTEM_POOLS,
15115 pool_index = idx,
15116 bucket = %bucket,
15117 object = %entry.name,
15118 state = "retained_for_final_verification",
15119 "Decommission durable ILM source retained for final verification"
15120 );
15121 } else if decommissioned != fivs.versions.len() || expired > 0 {
15122 warn!(
15123 event = EVENT_DECOMMISSION_ENTRY,
15124 component = LOG_COMPONENT_ECSTORE,
15125 subsystem = LOG_SUBSYSTEM_POOLS,
15126 pool_index = idx,
15127 bucket = %bucket,
15128 object = %entry.name,
15129 decommissioned,
15130 total_versions = fivs.versions.len(),
15131 expired,
15132 state = "source_retained",
15133 "Decommission source object retained"
15134 );
15135 }
15136
15137 let should_save_progress = {
15138 let mut pool_meta = self.pool_meta.write().await;
15139 ensure_decommission_generation(&pool_meta, idx, generation)?;
15140
15141 if let Err(err) = track_decommission_current_object(&mut pool_meta, idx, bucket.as_str(), entry.name.as_str()) {
15142 return Err(with_decommission_entry_context(
15143 "track_decommission_current_object",
15144 bucket.as_str(),
15145 entry.name.as_str(),
15146 err,
15147 ));
15148 }
15149
15150 match resolve_decommission_update_after_result(pool_meta.update_after(idx, DECOMMISSION_PROGRESS_SAVE_INTERVAL)) {
15151 Ok(ok) => ok,
15152 Err(err) => {
15153 return Err(with_decommission_entry_context("update_after", bucket.as_str(), entry.name.as_str(), err));
15154 }
15155 }
15156 };
15157
15158 self.track_decommission_entry_progress_stage(
15159 idx,
15160 generation,
15161 bucket.as_str(),
15162 entry.name.as_str(),
15163 DECOMMISSION_STAGE_ENTRY_FINISHED,
15164 )
15165 .await?;
15166
15167 if should_save_progress {
15168 match self.save_decommission_progress_checkpoint(idx, generation).await {
15169 Ok(true) => {
15170 if let Some(notification_sys) = runtime_sources::notification_sys()
15171 && let Err(err) = resolve_decommission_entry_reload_result(
15172 notification_sys.reload_pool_meta().await,
15173 bucket.as_str(),
15174 entry.name.as_str(),
15175 )
15176 {
15177 warn!("{err}");
15178 }
15179 }
15180 Ok(false) => {}
15181 Err(err) => {
15182 if let Some(err) = resolve_decommission_progress_save_result(Err(err)) {
15183 warn!(
15184 event = EVENT_DECOMMISSION_ENTRY,
15185 component = LOG_COMPONENT_ECSTORE,
15186 subsystem = LOG_SUBSYSTEM_POOLS,
15187 pool_index = idx,
15188 bucket = %bucket,
15189 object = %entry.name,
15190 state = "progress_save_failed",
15191 error = %err,
15192 "Decommission progress save failed; continuing and will retry at the next checkpoint"
15193 );
15194 }
15195 }
15196 }
15197 }
15198
15199 debug!(
15200 event = EVENT_DECOMMISSION_ENTRY,
15201 component = LOG_COMPONENT_ECSTORE,
15202 subsystem = LOG_SUBSYSTEM_POOLS,
15203 state = if entry_blocked { "blocked" } else { "completed" },
15204 pool_index = idx,
15205 bucket = %bucket,
15206 object = %entry.name,
15207 "Decommission entry finished"
15208 );
15209 Ok(DecommissionEntryAttemptOutcome::Complete)
15210 }
15211
15212 #[cfg(test)]
15213 pub(crate) async fn decommission_entry_for_test(
15214 self: &Arc<Self>,
15215 idx: usize,
15216 entry: MetaCacheEntry,
15217 bucket: String,
15218 set: Arc<SetDisks>,
15219 ) -> Result<()> {
15220 self.decommission_entry_with_retry_state_for_test(
15221 CancellationToken::new(),
15222 idx,
15223 entry,
15224 bucket,
15225 set,
15226 None,
15227 Arc::new(AtomicUsize::new(0)),
15228 )
15229 .await
15230 }
15231
15232 #[cfg(test)]
15233 #[allow(clippy::too_many_arguments)]
15234 pub(crate) async fn decommission_entry_with_retry_state_for_test(
15235 self: &Arc<Self>,
15236 rx: CancellationToken,
15237 idx: usize,
15238 entry: MetaCacheEntry,
15239 bucket: String,
15240 set: Arc<SetDisks>,
15241 expected_bucket_incarnation_id: Option<uuid::Uuid>,
15242 source_changed_exhaustions: Arc<AtomicUsize>,
15243 ) -> Result<()> {
15244 {
15245 let mut cancelers = self.decommission_cancelers.write().await;
15246 if cancelers.get(idx).and_then(Option::as_ref).is_none()
15247 && let Some(slot) = cancelers.get_mut(idx)
15248 {
15249 *slot = Some(DecommissionCanceler::new(CancellationToken::new()));
15250 }
15251 }
15252 let needs_capacity_reservation = {
15253 let pool_meta = self.pool_meta.read().await;
15254 pool_meta
15255 .pools
15256 .get(idx)
15257 .and_then(|pool| pool.decommission.as_ref())
15258 .and_then(|info| info.capacity_reservation.as_ref())
15259 .is_some_and(|reservation| !reservation.active())
15260 };
15261 if needs_capacity_reservation {
15262 let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
15263 let mut pool_meta = self.pool_meta.write().await;
15264 let version = pool_meta.version;
15265 pool_meta.version = POOL_META_VERSION;
15266 recover_decommission_capacity_reservations(
15267 &mut pool_meta,
15268 &capacity_infos,
15269 OffsetDateTime::now_utc(),
15270 crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof().is_some(),
15271 )?;
15272 pool_meta.version = version;
15273 }
15274 let generation = self.active_decommission_generation(idx).await?;
15275 self.decommission_entry(
15276 rx,
15277 idx,
15278 generation,
15279 entry,
15280 bucket,
15281 set,
15282 None,
15283 None,
15284 None,
15285 expected_bucket_incarnation_id,
15286 source_changed_exhaustions,
15287 )
15288 .await
15289 }
15290
15291 #[cfg(all(test, feature = "test-util"))]
15292 pub(crate) async fn decommission_entry_for_test_with_bucket_incarnation(
15293 self: &Arc<Self>,
15294 idx: usize,
15295 entry: MetaCacheEntry,
15296 bucket: String,
15297 set: Arc<SetDisks>,
15298 ) -> Result<()> {
15299 let expected_bucket_incarnation_id = if is_meta_bucketname(&bucket) {
15300 None
15301 } else {
15302 Some(self.bucket_incarnation_id_from_disk(&bucket).await?)
15303 };
15304 let generation = self.active_decommission_generation(idx).await?;
15305 self.decommission_entry(
15306 CancellationToken::new(),
15307 idx,
15308 generation,
15309 entry,
15310 bucket,
15311 set,
15312 None,
15313 None,
15314 None,
15315 expected_bucket_incarnation_id,
15316 Arc::new(AtomicUsize::new(0)),
15317 )
15318 .await
15319 }
15320
15321 #[tracing::instrument(skip(self, rx))]
15322 async fn decommission_pool(
15323 self: &Arc<Self>,
15324 rx: CancellationToken,
15325 idx: usize,
15326 pool: Arc<Sets>,
15327 bi: DecomBucketInfo,
15328 entry_budget: Arc<Semaphore>,
15329 source_changed_exhaustions: Arc<AtomicUsize>,
15330 ) -> Result<()> {
15331 let entry_error = Arc::new(tokio::sync::Mutex::new(None::<Error>));
15332 let generation = self.active_decommission_generation(idx).await?;
15333 let mut listing_workers = Vec::with_capacity(pool.disk_set.len());
15334
15335 let mut lifecycle_config = None;
15336 let mut object_lock_config = None;
15337 let mut replication_config = None;
15338 let expected_bucket_incarnation_id = if bi.name == RUSTFS_META_BUCKET {
15339 None
15340 } else {
15341 Some(self.bucket_incarnation_id_from_disk(&bi.name).await?)
15342 };
15343
15344 if bi.name != RUSTFS_META_BUCKET {
15345 let _ = resolve_decommission_optional_bucket_config_result(
15346 &bi.name,
15347 "versioning",
15348 BucketVersioningSys::get_in(&self.ctx, &bi.name).await,
15349 )?;
15350 let expiry_configs = get_expiry_configs(self, &bi.name).await?;
15351 lifecycle_config = expiry_configs.lifecycle.map(|config| (*config).clone());
15352 object_lock_config = expiry_configs.object_lock.map(|config| (*config).clone());
15353 replication_config = resolve_decommission_optional_bucket_config_result(
15354 &bi.name,
15355 "replication",
15356 metadata_sys::get_replication_config_in(&self.ctx, &bi.name).await,
15357 )?;
15358 }
15359
15360 for (set_idx, set) in pool.disk_set.iter().enumerate() {
15361 debug!(
15362 event = EVENT_DECOMMISSION_BUCKET,
15363 component = LOG_COMPONENT_ECSTORE,
15364 subsystem = LOG_SUBSYSTEM_POOLS,
15365 pool_index = idx,
15366 set_index = set_idx,
15367 bucket = %bi.name,
15368 state = "listing_worker_started",
15369 "Decommission listing worker started"
15370 );
15371
15372 let set = set.clone();
15373 let store = Arc::clone(self);
15374 let rx_clone = rx.clone();
15375 let bi_clone = bi.clone();
15376 let lifecycle_config = lifecycle_config.clone();
15377 let object_lock_config = object_lock_config.clone();
15378 let replication_config = replication_config.clone();
15379 let entry_budget = entry_budget.clone();
15380 let source_changed_exhaustions = Arc::clone(&source_changed_exhaustions);
15381 let entry_error = entry_error.clone();
15382 let worker = tokio::spawn(async move {
15383 store
15384 .decommission_set(
15385 rx_clone,
15386 idx,
15387 set_idx,
15388 generation,
15389 set,
15390 bi_clone,
15391 lifecycle_config,
15392 object_lock_config,
15393 replication_config,
15394 expected_bucket_incarnation_id,
15395 source_changed_exhaustions,
15396 entry_budget,
15397 entry_error,
15398 )
15399 .await
15400 });
15401 listing_workers.push((set_idx, worker));
15402 }
15403
15404 debug!(
15405 event = EVENT_DECOMMISSION_BUCKET,
15406 component = LOG_COMPONENT_ECSTORE,
15407 subsystem = LOG_SUBSYSTEM_POOLS,
15408 pool_index = idx,
15409 bucket = %bi.name,
15410 state = "waiting_for_workers",
15411 "Decommission waiting for workers"
15412 );
15413
15414 let mut listing_worker_error = None;
15415 for (set_id, worker) in listing_workers {
15416 if let Err(err) = resolve_decommission_listing_worker_result(set_id, worker.await) {
15417 rx.cancel();
15418 if listing_worker_error.is_none() {
15419 listing_worker_error = Some(err);
15420 }
15421 }
15422 }
15423
15424 if let Some(err) = listing_worker_error {
15425 return Err(err);
15426 }
15427
15428 if let Some(err) = entry_error.lock().await.clone() {
15429 return Err(err);
15430 }
15431
15432 if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) {
15433 warn!(
15434 event = EVENT_DECOMMISSION_BUCKET,
15435 component = LOG_COMPONENT_ECSTORE,
15436 subsystem = LOG_SUBSYSTEM_POOLS,
15437 pool_index = idx,
15438 bucket = %bi.name,
15439 state = "cancelled_after_wait",
15440 error = %err,
15441 "Decommission bucket cancelled after wait"
15442 );
15443 return Err(err);
15444 }
15445
15446 debug!(
15447 event = EVENT_DECOMMISSION_BUCKET,
15448 component = LOG_COMPONENT_ECSTORE,
15449 subsystem = LOG_SUBSYSTEM_POOLS,
15450 pool_index = idx,
15451 bucket = %bi.name,
15452 state = "completed",
15453 "Decommission bucket completed"
15454 );
15455
15456 Ok(())
15457 }
15458
15459 #[cfg(all(test, feature = "test-util"))]
15460 pub(crate) async fn decommission_pool_for_test(
15461 self: &Arc<Self>,
15462 rx: CancellationToken,
15463 idx: usize,
15464 pool: Arc<Sets>,
15465 bucket: DecomBucketInfo,
15466 ) -> Result<()> {
15467 self.decommission_pool(
15468 rx,
15469 idx,
15470 pool,
15471 bucket,
15472 Arc::new(Semaphore::new(decommission_entry_concurrency_limit())),
15473 Arc::new(AtomicUsize::new(0)),
15474 )
15475 .await
15476 }
15477
15478 #[tracing::instrument(skip(self, canceler))]
15479 pub async fn do_decommission_in_routine(
15480 self: &Arc<Self>,
15481 canceler: DecommissionCanceler,
15482 idx: usize,
15483 entry_budget: Arc<Semaphore>,
15484 ) -> Result<()> {
15485 let rx = canceler.token().clone();
15486 self.run_decommission_in_routine(rx, idx, &canceler, entry_budget).await
15487 }
15488
15489 async fn run_decommission_in_routine(
15490 self: &Arc<Self>,
15491 rx: CancellationToken,
15492 idx: usize,
15493 canceler: &DecommissionCanceler,
15494 entry_budget: Arc<Semaphore>,
15495 ) -> Result<()> {
15496 self.ensure_pool_meta_side_effects_safe("decommission cannot run while pool metadata requires recovery")
15497 .await?;
15498 let generation = match self.promote_queued_decommission(idx, canceler).await {
15499 Ok(generation) => generation,
15500 Err(Error::OperationCanceled) => return Ok(()),
15501 Err(err) if is_decommission_capacity_blocked_error(&err) || is_decommission_target_capacity_error(&err) => {
15502 if let Err(pause_err) = self.pause_decommission_for_capacity(idx, &err).await {
15503 return Err(decommission_capacity_blocked_error(format!(
15504 "failed to persist paused state for pool {idx}: {pause_err}"
15505 )));
15506 }
15507 return Ok(());
15508 }
15509 Err(err) => {
15510 resolve_decommission_terminal_mark_after_error_result(
15511 self.decommission_failed_for_operation(idx, canceler).await,
15512 idx,
15513 &err,
15514 )?;
15515 return Err(err);
15516 }
15517 };
15518 if rx.is_cancelled() {
15519 let already_canceled = {
15520 let pool_meta = self.pool_meta.read().await;
15521 should_skip_canceled_decommission_routine(true, pool_meta.pools.get(idx))
15522 };
15523 if already_canceled {
15524 warn!(
15525 event = EVENT_DECOMMISSION_STATE,
15526 component = LOG_COMPONENT_ECSTORE,
15527 subsystem = LOG_SUBSYSTEM_POOLS,
15528 pool_index = idx,
15529 state = "canceled_preserved",
15530 "Decommission routine skipped because pool is already canceled"
15531 );
15532 return Ok(());
15533 }
15534 if let Err(err) = self.decommission_cancel_for_operation(idx, canceler).await {
15535 resolve_decommission_terminal_mark_after_error_result(
15536 self.decommission_failed_for_operation(idx, canceler).await,
15537 idx,
15538 &err,
15539 )?;
15540 return Err(err);
15541 }
15542 return Ok(());
15543 }
15544 let result = self
15545 .decommission_in_background(rx.clone(), idx, generation, entry_budget)
15546 .await;
15547
15548 if let Err(err) = &result
15549 && (is_decommission_capacity_blocked_error(err) || is_decommission_target_capacity_error(err))
15550 {
15551 if let Err(pause_err) = self.pause_decommission_for_capacity(idx, err).await {
15552 return Err(decommission_capacity_blocked_error(format!(
15553 "failed to persist paused state for pool {idx}: {pause_err}"
15554 )));
15555 }
15556 return Ok(());
15557 }
15558
15559 let (final_state, canceled, cmd_line) = {
15560 let pool_meta = self.pool_meta.read().await;
15561 let Some(pool) = pool_meta.pools.get(idx) else {
15562 error!(
15563 event = EVENT_DECOMMISSION_STATE,
15564 component = LOG_COMPONENT_ECSTORE,
15565 subsystem = LOG_SUBSYSTEM_POOLS,
15566 pool_index = idx,
15567 state = "pool_metadata_missing",
15568 "Decommission pool metadata missing"
15569 );
15570 return Err(Error::other(format!(
15571 "failed to resolve decommission final state: pool metadata missing for idx {idx}"
15572 )));
15573 };
15574
15575 let (final_state, canceled) = if let Some(info) = &pool.decommission {
15576 (
15577 determine_decommission_final_state(info.items_decommission_failed, info.canceled),
15578 info.canceled,
15579 )
15580 } else {
15581 (DecommissionFinalState::Failed, false)
15582 };
15583 let cmd_line = pool.cmd_line.clone();
15584 (final_state, canceled, cmd_line)
15585 };
15586
15587 if let Err(err) = result {
15588 error!(
15589 event = EVENT_DECOMMISSION_STATE,
15590 component = LOG_COMPONENT_ECSTORE,
15591 subsystem = LOG_SUBSYSTEM_POOLS,
15592 pool_index = idx,
15593 state = "background_failed",
15594 error = ?err,
15595 "Decommission background routine failed"
15596 );
15597
15598 if should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
15599 warn!(
15600 event = EVENT_DECOMMISSION_STATE,
15601 component = LOG_COMPONENT_ECSTORE,
15602 subsystem = LOG_SUBSYSTEM_POOLS,
15603 pool_index = idx,
15604 cmd_line = %cmd_line,
15605 state = "cancelled_preserved",
15606 "Decommission cancelled; preserving canceled state"
15607 );
15608 return Ok(());
15609 }
15610
15611 resolve_decommission_terminal_mark_after_error_result(
15612 self.decommission_failed_for_operation(idx, canceler).await,
15613 idx,
15614 &err,
15615 )?;
15616 warn!(
15617 event = EVENT_DECOMMISSION_STATE,
15618 component = LOG_COMPONENT_ECSTORE,
15619 subsystem = LOG_SUBSYSTEM_POOLS,
15620 pool_index = idx,
15621 state = "marked_failed",
15622 "Decommission marked failed"
15623 );
15624
15625 return Ok(());
15626 }
15627
15628 debug!(
15629 event = EVENT_DECOMMISSION_STATE,
15630 component = LOG_COMPONENT_ECSTORE,
15631 subsystem = LOG_SUBSYSTEM_POOLS,
15632 pool_index = idx,
15633 state = "background_complete",
15634 "Decommission background routine completed"
15635 );
15636
15637 if should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
15638 warn!(
15639 event = EVENT_DECOMMISSION_STATE,
15640 component = LOG_COMPONENT_ECSTORE,
15641 subsystem = LOG_SUBSYSTEM_POOLS,
15642 pool_index = idx,
15643 cmd_line = %cmd_line,
15644 state = "terminal_state_preserved",
15645 "Decommission terminal state preserved after cancellation"
15646 );
15647 return Ok(());
15648 }
15649
15650 match final_state {
15651 DecommissionFinalState::Complete => {
15652 debug!(
15653 event = EVENT_DECOMMISSION_STATE,
15654 component = LOG_COMPONENT_ECSTORE,
15655 subsystem = LOG_SUBSYSTEM_POOLS,
15656 pool_index = idx,
15657 cmd_line = %cmd_line,
15658 state = "verifying_completion",
15659 "Decommission completion verification started"
15660 );
15661 let verified_unresolved_entries = match self.check_after_decommission(idx, &rx, generation).await {
15662 Ok(verified_entries) => verified_entries,
15663 Err(err) => {
15664 if is_err_operation_canceled(&err) {
15665 return Err(err);
15666 }
15667 resolve_decommission_terminal_mark_result(
15668 self.decommission_failed_for_operation(idx, canceler).await,
15669 "failed",
15670 &cmd_line,
15671 )?;
15672 return Err(Error::other(format!(
15673 "failed to finalize decommission for pool {cmd_line}: post-check failed: {err}"
15674 )));
15675 }
15676 };
15677 info!(
15678 event = EVENT_DECOMMISSION_STATE,
15679 component = LOG_COMPONENT_ECSTORE,
15680 subsystem = LOG_SUBSYSTEM_POOLS,
15681 pool_index = idx,
15682 cmd_line = %cmd_line,
15683 state = "marking_completed",
15684 "Decommission marking completed state"
15685 );
15686 if let Err(err) = self
15687 .complete_decommission_for_operation(idx, canceler, generation, verified_unresolved_entries)
15688 .await
15689 {
15690 resolve_decommission_terminal_mark_result(
15691 self.decommission_failed_for_operation(idx, canceler).await,
15692 "failed",
15693 &cmd_line,
15694 )?;
15695 return Err(Error::other(format!("failed to finalize decommission for pool {cmd_line}: {err}")));
15696 }
15697 }
15698 DecommissionFinalState::Failed => {
15699 warn!(
15700 event = EVENT_DECOMMISSION_STATE,
15701 component = LOG_COMPONENT_ECSTORE,
15702 subsystem = LOG_SUBSYSTEM_POOLS,
15703 pool_index = idx,
15704 cmd_line = %cmd_line,
15705 state = "marking_failed",
15706 "Decommission marking failed state"
15707 );
15708 resolve_decommission_terminal_mark_result(
15709 self.decommission_failed_for_operation(idx, canceler).await,
15710 "failed",
15711 &cmd_line,
15712 )?;
15713 }
15714 }
15715
15716 info!(
15717 event = EVENT_DECOMMISSION_STATE,
15718 component = LOG_COMPONENT_ECSTORE,
15719 subsystem = LOG_SUBSYSTEM_POOLS,
15720 pool_index = idx,
15721 cmd_line = %cmd_line,
15722 state = "completed",
15723 "Decommission completed"
15724 );
15725 Ok(())
15726 }
15727
15728 #[tracing::instrument(skip(self))]
15729 pub async fn decommission_failed(&self, idx: usize) -> Result<()> {
15730 self.decommission_failed_with_owner(idx, None).await
15731 }
15732
15733 async fn decommission_failed_for_operation(&self, idx: usize, owner: &DecommissionCanceler) -> Result<()> {
15734 self.decommission_failed_with_owner(idx, Some(owner)).await
15735 }
15736
15737 async fn decommission_failed_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> {
15738 self.decommission_failed_with_owner_and_save(idx, owner, self.save_current_pool_meta(&[idx]))
15739 .await
15740 }
15741
15742 async fn decommission_failed_with_owner_and_save<SaveFuture>(
15743 &self,
15744 idx: usize,
15745 owner: Option<&DecommissionCanceler>,
15746 save_pool_meta: SaveFuture,
15747 ) -> Result<()>
15748 where
15749 SaveFuture: Future<Output = Result<()>>,
15750 {
15751 ensure_decommission_terminal_operation_supported(self.single_pool(), "mark decommission failed")?;
15752 let _start_guard = self.start_gate.lock().await;
15753 let movement_gate = self.ctx.data_movement_operation_gate();
15754 let _movement_guard = movement_gate.write().await;
15755
15756 let rebalance_meta = self.rebalance_meta.read().await.clone();
15760 let terminal_at = OffsetDateTime::now_utc();
15761 let (should_reload_pool_meta, previous_pool_meta, terminal_canceler) = {
15762 let cancelers = self.decommission_cancelers.read().await;
15763 let mut pool_meta = self.pool_meta.write().await;
15764 let previous_pool_meta = pool_meta.clone();
15765 let Some(changed) =
15766 update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, idx, owner, |pool_meta| {
15767 pool_meta.decommission_failed_at(idx, terminal_at, rebalance_meta.as_ref())
15768 })
15769 else {
15770 return Ok(());
15771 };
15772 let terminal_canceler = if let Some(owner) = owner {
15773 Some(owner.clone())
15774 } else {
15775 cancelers.get(idx).and_then(Option::as_ref).cloned()
15776 };
15777 (changed, changed.then_some(previous_pool_meta), terminal_canceler)
15778 };
15779
15780 if should_reload_pool_meta && let Err(err) = save_pool_meta.await {
15781 if let Some(previous_pool_meta) = previous_pool_meta {
15782 let mut pool_meta = self.pool_meta.write().await;
15783 rollback_decommission_pool_meta(&mut pool_meta, &previous_pool_meta, &[idx]);
15784 }
15785 return Err(err);
15786 }
15787 if should_reload_pool_meta {
15788 {
15789 let mut pool_meta = self.pool_meta.write().await;
15790 pool_meta.mark_decommission_progress_saved();
15791 }
15792 }
15793 if let Some(canceler) = terminal_canceler.as_ref() {
15794 self.release_decommission_canceler_slot(idx, canceler).await;
15795 }
15796
15797 if should_reload_pool_meta {
15798 self.ctx.advance_data_movement_operation_epoch();
15799 }
15800 drop(_movement_guard);
15801
15802 if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
15803 let stage = format!("decommission_failed for pool {idx}");
15804 if let Some(err) = observe_decommission_terminal_reload_result(
15805 resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()),
15806 stage.as_str(),
15807 ) {
15808 if let Err(record_err) = self
15809 .record_decommission_terminal_reload_failure(idx, stage.as_str(), err.clone())
15810 .await
15811 {
15812 warn!(
15813 event = EVENT_DECOMMISSION_STATE,
15814 component = LOG_COMPONENT_ECSTORE,
15815 subsystem = LOG_SUBSYSTEM_POOLS,
15816 pool_index = idx,
15817 state = "terminal_reload_record_failed",
15818 error = %record_err,
15819 original_error = %err,
15820 "Decommission terminal reload failure record failed"
15821 );
15822 }
15823 warn!(
15824 event = EVENT_DECOMMISSION_STATE,
15825 component = LOG_COMPONENT_ECSTORE,
15826 subsystem = LOG_SUBSYSTEM_POOLS,
15827 pool_index = idx,
15828 state = "terminal_reload_failed",
15829 error = %err,
15830 "Decommission terminal state saved but pool meta reload failed"
15831 );
15832 }
15833 }
15834
15835 Ok(())
15836 }
15837
15838 #[tracing::instrument(skip(self))]
15839 pub async fn complete_decommission(&self, idx: usize) -> Result<()> {
15840 self.complete_decommission_with_owner(idx, None, None, None).await
15841 }
15842
15843 async fn complete_decommission_for_operation(
15844 &self,
15845 idx: usize,
15846 owner: &DecommissionCanceler,
15847 verified_generation: OffsetDateTime,
15848 verified_unresolved_entries: Vec<DecommissionUnresolvedEntry>,
15849 ) -> Result<()> {
15850 self.complete_decommission_with_owner(idx, Some(owner), Some(verified_generation), Some(verified_unresolved_entries))
15851 .await
15852 }
15853
15854 async fn complete_decommission_with_owner(
15855 &self,
15856 idx: usize,
15857 owner: Option<&DecommissionCanceler>,
15858 verified_generation: Option<OffsetDateTime>,
15859 verified_unresolved_entries: Option<Vec<DecommissionUnresolvedEntry>>,
15860 ) -> Result<()> {
15861 ensure_decommission_terminal_operation_supported(self.single_pool(), "complete decommission")?;
15862 ensure_valid_decommission_pool_index(self.pools.len(), idx)?;
15863 if let Some(owner) = owner {
15864 let cancelers = self.decommission_cancelers.read().await;
15865 if !decommission_canceler_is_owned_by(cancelers.as_slice(), idx, owner) {
15866 owner.release();
15867 return Ok(());
15868 }
15869 }
15870 self.verify_decommission_durable_ilm_receipts(idx).await?;
15871 let _start_guard = self.start_gate.lock().await;
15872 let movement_gate = self.ctx.data_movement_operation_gate();
15873 let _movement_guard = movement_gate.write().await;
15874
15875 let rebalance_meta = self.rebalance_meta.read().await.clone();
15879 let terminal_at = OffsetDateTime::now_utc();
15880 let (should_reload_pool_meta, completed, previous_pool_meta, terminal_canceler) = {
15881 let cancelers = self.decommission_cancelers.read().await;
15882 let mut pool_meta = self.pool_meta.write().await;
15883 let previous_pool_meta = pool_meta.clone();
15884 let Some(changed) =
15885 update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, idx, owner, |pool_meta| {
15886 reconcile_decommission_unresolved_entries_for_completion(
15887 pool_meta,
15888 idx,
15889 verified_generation,
15890 verified_unresolved_entries.as_deref(),
15891 )?;
15892 Ok::<bool, Error>(pool_meta.decommission_complete_at(idx, terminal_at, rebalance_meta.as_ref()))
15893 })
15894 else {
15895 return Ok(());
15896 };
15897 let changed = changed?;
15898 let completed = pool_meta
15899 .pools
15900 .get(idx)
15901 .and_then(|pool| pool.decommission.as_ref())
15902 .is_some_and(|decommission| decommission.complete);
15903 let terminal_canceler = if let Some(owner) = owner {
15904 Some(owner.clone())
15905 } else {
15906 cancelers.get(idx).and_then(Option::as_ref).cloned()
15907 };
15908 (changed, completed, changed.then_some(previous_pool_meta), terminal_canceler)
15909 };
15910
15911 if should_reload_pool_meta && let Err(err) = self.save_current_pool_meta(&[idx]).await {
15912 if let Some(previous_pool_meta) = previous_pool_meta {
15913 let mut pool_meta = self.pool_meta.write().await;
15914 rollback_decommission_pool_meta(&mut pool_meta, &previous_pool_meta, &[idx]);
15915 }
15916 return Err(err);
15917 }
15918 if should_reload_pool_meta {
15919 {
15920 let mut pool_meta = self.pool_meta.write().await;
15921 pool_meta.mark_decommission_progress_saved();
15922 }
15923 }
15924 if let Some(canceler) = terminal_canceler.as_ref() {
15925 self.release_decommission_canceler_slot(idx, canceler).await;
15926 }
15927
15928 if should_reload_pool_meta {
15929 self.ctx.advance_data_movement_operation_epoch();
15930 }
15931 drop(_movement_guard);
15932
15933 if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
15934 let stage = format!("complete_decommission for pool {idx}");
15935 if let Some(err) = observe_decommission_terminal_reload_result(
15936 resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()),
15937 stage.as_str(),
15938 ) {
15939 if let Err(record_err) = self
15940 .record_decommission_terminal_reload_failure(idx, stage.as_str(), err.clone())
15941 .await
15942 {
15943 warn!(
15944 event = EVENT_DECOMMISSION_STATE,
15945 component = LOG_COMPONENT_ECSTORE,
15946 subsystem = LOG_SUBSYSTEM_POOLS,
15947 pool_index = idx,
15948 state = "terminal_reload_record_failed",
15949 error = %record_err,
15950 original_error = %err,
15951 "Decommission terminal reload failure record failed"
15952 );
15953 }
15954 warn!(
15955 event = EVENT_DECOMMISSION_STATE,
15956 component = LOG_COMPONENT_ECSTORE,
15957 subsystem = LOG_SUBSYSTEM_POOLS,
15958 pool_index = idx,
15959 state = "terminal_reload_failed",
15960 error = %err,
15961 "Decommission terminal state saved but pool meta reload failed"
15962 );
15963 }
15964 }
15965
15966 if completed && let Err(err) = self.cleanup_decommission_durable_ilm_receipts(idx).await {
15967 warn!(
15968 event = EVENT_DECOMMISSION_STATE,
15969 component = LOG_COMPONENT_ECSTORE,
15970 subsystem = LOG_SUBSYSTEM_POOLS,
15971 pool_index = idx,
15972 state = "receipt_cleanup_failed",
15973 error = %err,
15974 "Decommission durable ILM receipt cleanup failed"
15975 );
15976 }
15977
15978 Ok(())
15979 }
15980
15981 async fn decommission_pending_bucket(
15982 self: &Arc<Self>,
15983 rx: CancellationToken,
15984 idx: usize,
15985 generation: OffsetDateTime,
15986 bucket: DecomBucketInfo,
15987 entry_budget: Arc<Semaphore>,
15988 source_changed_exhaustions: Arc<AtomicUsize>,
15989 ) -> Result<()> {
15990 let is_decommissioned = {
15991 let pool_meta = self.pool_meta.read().await;
15992 ensure_decommission_generation(&pool_meta, idx, generation)?;
15993 resolve_decommission_bucket_state(&pool_meta, idx, &bucket)?
15994 };
15995
15996 if is_decommissioned {
15997 warn!("decommission: already done, moving on {}", bucket.to_string());
15998
15999 self.mark_decommission_bucket_done_and_save(idx, generation, &bucket).await?;
16000 return Ok(());
16001 }
16002
16003 warn!("decommission: currently on bucket {}", &bucket.name);
16004
16005 let pool = get_by_index(self.pools.as_slice(), idx, "load decommission background pool")?.clone();
16006 if let Err(err) = self
16007 .decommission_pool(rx.clone(), idx, pool, bucket.clone(), entry_budget, source_changed_exhaustions)
16008 .await
16009 {
16010 error!("decommission: decommission_pool err {:?}", &err);
16011 return Err(err);
16012 } else {
16013 warn!("decommission: decommission_pool done {}", &bucket.name);
16014 }
16015
16016 if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) {
16017 warn!("decommission: cancellation observed after decommission_pool {}", &bucket.name);
16018 return Err(err);
16019 }
16020
16021 self.mark_decommission_bucket_done_and_save(idx, generation, &bucket).await?;
16022
16023 warn!("decommission: decommission_pool bucket_done {}", &bucket.name);
16024
16025 Ok(())
16026 }
16027
16028 #[tracing::instrument(skip(self, rx))]
16029 async fn decommission_in_background(
16030 self: &Arc<Self>,
16031 rx: CancellationToken,
16032 idx: usize,
16033 generation: OffsetDateTime,
16034 entry_budget: Arc<Semaphore>,
16035 ) -> Result<()> {
16036 self.ensure_decommission_runtime_capacity_available(idx, generation).await?;
16037 let pending = {
16038 let pool_meta = self.pool_meta.read().await;
16039 pool_meta.pending_buckets(idx)
16040 };
16041 let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
16042 let bucket_concurrency = decommission_bucket_concurrency_limit();
16043 let (regular_buckets, meta_buckets) = split_decommission_buckets(pending);
16044 let store = Arc::clone(self);
16045 run_decommission_phases(rx.clone(), regular_buckets, meta_buckets, bucket_concurrency, move |bucket, rx| {
16046 let store = Arc::clone(&store);
16047 let entry_budget = entry_budget.clone();
16048 let source_changed_exhaustions = Arc::clone(&source_changed_exhaustions);
16049 Box::pin(async move {
16050 store
16051 .decommission_pending_bucket(rx, idx, generation, bucket, entry_budget, source_changed_exhaustions)
16052 .await
16053 })
16054 })
16055 .await
16056 }
16057
16058 #[tracing::instrument(skip(self))]
16059 pub async fn start_decommission(&self, indices: Vec<usize>) -> Result<()> {
16060 self.start_decommission_inner(indices, None).await.map(|_| ())
16061 }
16062
16063 async fn start_decommission_with_routines(
16064 &self,
16065 indices: Vec<usize>,
16066 rx: &CancellationToken,
16067 local_indices: &[usize],
16068 ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
16069 self.start_decommission_inner(indices, Some((rx, local_indices))).await
16070 }
16071
16072 async fn start_decommission_inner(
16073 &self,
16074 indices: Vec<usize>,
16075 reservation: Option<(&CancellationToken, &[usize])>,
16076 ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
16077 let indices = dedup_indices(&indices);
16078 validate_start_decommission_request(&indices, self.single_pool())?;
16079
16080 self.ensure_decommission_rebalance_idle_after_refresh().await?;
16081 #[cfg(test)]
16082 let endpoints = self.instance_endpoints().unwrap_or_else(|| self.endpoints());
16083 #[cfg(not(test))]
16084 let endpoints = self.endpoints();
16085 ensure_decommission_start_local_leader(&endpoints, &indices)?;
16086
16087 for idx in indices.iter().copied() {
16088 ensure_valid_decommission_pool_index(self.pools.len(), idx)?;
16089 }
16090
16091 {
16092 let pool_meta = self.pool_meta.read().await;
16093 ensure_decommission_start_pool_states(&pool_meta, &indices)?;
16094 ensure_decommission_ledger_persistence_supported(&pool_meta)?;
16095 ensure_decommission_capacity_writer_supported(&pool_meta)?;
16096 }
16097
16098 #[cfg(test)]
16099 observe_pool_activation_preflight_side_effect_attempt(PoolActivationStartKind::Decommission);
16100 let decom_buckets = self.get_buckets_to_decommission().await?;
16101
16102 let mut healed_buckets = HashSet::with_capacity(decom_buckets.len());
16103 for bk in decom_buckets.iter() {
16104 if healed_buckets.insert(bk.name.as_str()) {
16105 resolve_decommission_preflight_heal_result(&bk.name, self.heal_bucket(&bk.name, &HealOpts::default()).await)?;
16106 }
16107 }
16108
16109 let meta_bucket_opts = decommission_meta_bucket_options();
16110 for prefix in DECOMMISSION_META_PREFIXES {
16111 let bk = path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(prefix)]);
16112 if let Err(err) = self
16113 .make_bucket(bk.to_string_lossy().to_string().as_str(), &meta_bucket_opts)
16114 .await
16115 && !is_err_bucket_exists(&err)
16116 {
16117 error!("decommission: make bucket failed: {err}");
16118 return Err(err);
16119 }
16120 }
16121
16122 let _start_guard = self.start_gate.lock().await;
16123 self.ensure_decommission_rebalance_idle_after_refresh_under_start_gate()
16124 .await?;
16125
16126 let all_capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
16127 let target_fence_proof_available =
16128 crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof().is_some();
16129 self.cancel_decommission_routines(&indices).await;
16132 let movement_gate = self.ctx.data_movement_operation_gate();
16133 let _movement_guard = movement_gate.write().await;
16134
16135 let index_cancelers = if let Some((rx, local_indices)) = reservation {
16136 let mut cancelers = self.decommission_cancelers.write().await;
16140 let pool_meta = self.pool_meta.read().await;
16141 ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_capacity_infos, target_fence_proof_available)?;
16142 reserve_decommission_start_cancelers(&pool_meta, &indices, local_indices, rx, cancelers.as_mut_slice())?
16143 } else {
16144 let pool_meta = self.pool_meta.read().await;
16145 ensure_decommission_start_pool_states(&pool_meta, &indices)?;
16146 ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_capacity_infos, target_fence_proof_available)?;
16147 Vec::new()
16148 };
16149
16150 let previous_pool_meta = self
16151 .save_current_pool_meta_for_decommission_start(&indices, decom_buckets)
16152 .await?;
16153 self.ctx.advance_data_movement_operation_epoch();
16154 drop(_movement_guard);
16157
16158 if let Some(notification_sys) = runtime_sources::notification_sys()
16159 && let Err(err) = resolve_start_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await)
16160 {
16161 warn!(
16162 event = EVENT_DECOMMISSION_STATE,
16163 component = LOG_COMPONENT_ECSTORE,
16164 subsystem = LOG_SUBSYSTEM_POOLS,
16165 state = "start_failed",
16166 stage = "reload_pool_meta",
16167 error = %err,
16168 "Decommission start failed after pool metadata save"
16169 );
16170
16171 let rollback_result = self
16172 .rollback_decommission_start_after_reload_failure(&movement_gate, &previous_pool_meta, &indices)
16173 .await;
16174 if let Err(rollback_save_err) = rollback_result {
16175 error!(
16176 event = EVENT_DECOMMISSION_STATE,
16177 component = LOG_COMPONENT_ECSTORE,
16178 subsystem = LOG_SUBSYSTEM_POOLS,
16179 state = "rollback_failed",
16180 stage = "save_pool_meta",
16181 error = %rollback_save_err,
16182 original_error = %err,
16183 "Decommission rollback failed after pool metadata reload failure"
16184 );
16185 return Err(Error::other(format!(
16186 "{err}; decommission start rollback save failed: {rollback_save_err}"
16187 )));
16188 }
16189
16190 if let Err(rollback_reload_err) = resolve_decommission_pool_meta_reload_result(
16191 notification_sys.reload_pool_meta().await,
16192 "start_decommission_rollback",
16193 ) {
16194 error!(
16195 event = EVENT_DECOMMISSION_STATE,
16196 component = LOG_COMPONENT_ECSTORE,
16197 subsystem = LOG_SUBSYSTEM_POOLS,
16198 state = "rollback_partial",
16199 stage = "reload_pool_meta",
16200 error = %rollback_reload_err,
16201 original_error = %err,
16202 "Decommission rollback metadata reload failed after local rollback save"
16203 );
16204 return Err(Error::other(format!(
16205 "{err}; decommission start rollback saved locally but peer reload failed: {rollback_reload_err}"
16206 )));
16207 }
16208
16209 warn!(
16210 event = EVENT_DECOMMISSION_STATE,
16211 component = LOG_COMPONENT_ECSTORE,
16212 subsystem = LOG_SUBSYSTEM_POOLS,
16213 state = "rollback_success",
16214 original_error = %err,
16215 "Decommission start rolled back after pool metadata reload failure"
16216 );
16217 return Err(Error::other(format!("{err}; decommission start rollback succeeded")));
16218 }
16219
16220 Ok(index_cancelers)
16221 }
16222
16223 async fn get_buckets_to_decommission(&self) -> Result<Vec<DecomBucketInfo>> {
16224 let buckets = self.list_bucket(&BucketOptions::default()).await?;
16225
16226 let mut ret: Vec<DecomBucketInfo> = buckets
16227 .iter()
16228 .map(|v| DecomBucketInfo {
16229 name: v.name.clone(),
16230 ..Default::default()
16231 })
16232 .collect();
16233
16234 ret.extend(decommission_meta_buckets());
16235
16236 Ok(ret)
16237 }
16238
16239 async fn durable_ilm_receipt_run_token(&self, source_pool_idx: usize) -> Result<String> {
16240 let pool_meta = self.pool_meta.read().await;
16241 let pool = pool_meta
16242 .pools
16243 .get(source_pool_idx)
16244 .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), source_pool_idx))?;
16245 let start_time = pool
16246 .decommission
16247 .as_ref()
16248 .and_then(|info| info.start_time)
16249 .ok_or_else(|| Error::other(format!("decommission run identity is missing for pool {source_pool_idx}")))?;
16250 Ok(decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time))
16251 }
16252
16253 async fn durable_ilm_receipt_run_token_for_generation(
16254 &self,
16255 source_pool_idx: usize,
16256 generation: OffsetDateTime,
16257 ) -> Result<String> {
16258 let pool_meta = self.pool_meta.read().await;
16259 ensure_decommission_generation(&pool_meta, source_pool_idx, generation)?;
16260 let pool = pool_meta
16261 .pools
16262 .get(source_pool_idx)
16263 .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), source_pool_idx))?;
16264 Ok(decommission_durable_ilm_receipt_run_token(&pool.cmd_line, generation))
16265 }
16266
16267 async fn load_decommissioned_durable_ilm_target(
16268 &self,
16269 source_pool_idx: usize,
16270 path: &str,
16271 max_record_size: usize,
16272 record_context: &str,
16273 ) -> Result<Option<(usize, Vec<u8>)>> {
16274 let mut target = None::<(usize, Vec<u8>)>;
16275 let mut first_read_error = None;
16276 for (target_pool_idx, pool) in self.pools.iter().enumerate() {
16277 if target_pool_idx == source_pool_idx {
16278 continue;
16279 }
16280 match read_config_limited_preserve_empty(pool.clone(), path, max_record_size).await {
16281 Ok(data) => {
16282 if let Some((existing_pool_idx, existing)) = target.as_ref()
16283 && existing != &data
16284 {
16285 return Err(Error::other(format!(
16286 "divergent target durable ILM records at path `{path}` {record_context} in pools {existing_pool_idx} and {target_pool_idx}"
16287 )));
16288 }
16289 target = Some((target_pool_idx, data));
16290 }
16291 Err(err)
16292 if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
16293 || is_err_object_not_found(&err)
16294 || is_err_version_not_found(&err) => {}
16295 Err(err) => {
16296 first_read_error.get_or_insert_with(|| {
16297 Error::other(format!(
16298 "failed to read target durable ILM record at path `{path}` {record_context} from pool {target_pool_idx}: {err}"
16299 ))
16300 });
16301 }
16302 }
16303 }
16304
16305 if let Some(err) = first_read_error {
16306 return Err(err);
16307 }
16308 Ok(target)
16309 }
16310
16311 async fn list_decommission_durable_ilm_receipt_paths_in_pool(&self, pool_idx: usize, prefix: &str) -> Result<Vec<String>> {
16312 let pool = self
16313 .pools
16314 .get(pool_idx)
16315 .ok_or_else(|| invalid_decommission_pool_index_error(self.pools.len(), pool_idx))?;
16316 let mut receipts = Vec::new();
16317 let mut continuation = None;
16318 loop {
16319 let page = pool
16320 .clone()
16321 .list_objects_v2(RUSTFS_META_BUCKET, prefix, continuation, None, 1000, false, None, false)
16322 .await
16323 .map_err(|err| {
16324 Error::other(format!(
16325 "failed to list durable ILM decommission receipts under `{prefix}` in pool {pool_idx}: {err}"
16326 ))
16327 })?;
16328 receipts.extend(page.objects.into_iter().map(|object| object.name));
16329 if !page.is_truncated {
16330 break;
16331 }
16332 continuation = Some(page.next_continuation_token.ok_or_else(|| {
16333 Error::other(format!(
16334 "durable ILM decommission receipt listing under `{prefix}` in pool {pool_idx} was truncated without a continuation token"
16335 ))
16336 })?);
16337 }
16338 Ok(receipts)
16339 }
16340
16341 async fn list_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<Vec<(usize, String)>> {
16342 let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
16343 self.list_decommission_durable_ilm_receipts_for_run(source_pool_idx, &run_token)
16344 .await
16345 }
16346
16347 async fn list_decommission_durable_ilm_receipts_for_run(
16348 &self,
16349 source_pool_idx: usize,
16350 run_token: &str,
16351 ) -> Result<Vec<(usize, String)>> {
16352 let prefix = decommission_durable_ilm_receipt_run_prefix(run_token);
16353 let mut receipts = Vec::new();
16354 for pool_idx in 0..self.pools.len() {
16355 if pool_idx == source_pool_idx {
16356 continue;
16357 }
16358 for receipt_path in self
16359 .list_decommission_durable_ilm_receipt_paths_in_pool(pool_idx, &prefix)
16360 .await?
16361 {
16362 let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
16363 if locator.run_token != run_token {
16364 return Err(Error::other(format!(
16365 "durable ILM receipt path `{receipt_path}` has an unexpected run token"
16366 )));
16367 }
16368 receipts.push((pool_idx, receipt_path));
16369 }
16370 }
16371 Ok(receipts)
16372 }
16373
16374 async fn list_decommission_durable_ilm_manifest_receipts(&self, source_pool_idx: usize) -> Result<Vec<String>> {
16375 let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
16376 self.list_decommission_durable_ilm_manifest_receipts_for_run(source_pool_idx, &run_token)
16377 .await
16378 }
16379
16380 async fn list_decommission_durable_ilm_manifest_receipts_for_run(
16381 &self,
16382 source_pool_idx: usize,
16383 run_token: &str,
16384 ) -> Result<Vec<String>> {
16385 let prefix = decommission_durable_ilm_receipt_run_prefix(run_token);
16386 let receipt_paths = self
16387 .list_decommission_durable_ilm_receipt_paths_in_pool(source_pool_idx, &prefix)
16388 .await?;
16389 for receipt_path in &receipt_paths {
16390 let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?;
16391 if locator.run_token != run_token {
16392 return Err(Error::other(format!(
16393 "durable ILM expected manifest receipt path `{receipt_path}` has an unexpected run token"
16394 )));
16395 }
16396 }
16397 Ok(receipt_paths)
16398 }
16399
16400 #[cfg(all(test, feature = "test-util"))]
16401 async fn persist_decommission_durable_ilm_manifest(&self, source_pool_idx: usize) -> Result<()> {
16402 let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
16403 self.persist_decommission_durable_ilm_manifest_for_run(source_pool_idx, &run_token)
16404 .await
16405 }
16406
16407 async fn persist_decommission_durable_ilm_manifest_for_run(&self, source_pool_idx: usize, run_token: &str) -> Result<()> {
16408 let receipt_paths = self
16409 .list_decommission_durable_ilm_manifest_receipts_for_run(source_pool_idx, run_token)
16410 .await?;
16411 for receipt_path in &receipt_paths {
16412 self.read_decommission_durable_ilm_receipt(source_pool_idx, receipt_path)
16413 .await?;
16414 }
16415 let manifest = DecommissionDurableIlmManifest::new(run_token, &receipt_paths)?;
16416 let manifest_path = decommission_durable_ilm_manifest_path(run_token);
16417 let encoded = manifest.encode()?;
16418 let mut attempt = 1;
16419 loop {
16420 match read_config_limited_preserve_empty(
16421 self.pools[source_pool_idx].clone(),
16422 &manifest_path,
16423 DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE,
16424 )
16425 .await
16426 {
16427 Ok(existing) => {
16428 DecommissionDurableIlmManifest::decode(&existing, run_token, &receipt_paths).map_err(|err| {
16429 Error::other(format!(
16430 "durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx} is invalid: {err}"
16431 ))
16432 })?;
16433 return Ok(());
16434 }
16435 Err(err)
16436 if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
16437 || is_err_object_not_found(&err)
16438 || is_err_version_not_found(&err) => {}
16439 Err(err) => {
16440 return Err(Error::other(format!(
16441 "failed to read durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}"
16442 )));
16443 }
16444 }
16445 match save_config_with_opts(
16446 self.pools[source_pool_idx].clone(),
16447 &manifest_path,
16448 encoded.clone(),
16449 &ObjectOptions {
16450 max_parity: true,
16451 write_completion: crate::object_api::WriteCompletion::TailDrained,
16452 http_preconditions: Some(HTTPPreconditions {
16453 if_none_match: Some("*".to_string()),
16454 ..Default::default()
16455 }),
16456 ..Default::default()
16457 },
16458 )
16459 .await
16460 {
16461 Ok(()) => return Ok(()),
16462 Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => {
16463 attempt += 1;
16464 }
16465 Err(Error::PreconditionFailed) => {
16466 return Err(Error::other(format!(
16467 "failed to persist durable ILM expected manifest `{manifest_path}` after concurrent updates"
16468 )));
16469 }
16470 Err(err) => {
16471 return Err(Error::other(format!(
16472 "failed to persist durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx}: {err}"
16473 )));
16474 }
16475 }
16476 }
16477 }
16478
16479 async fn load_decommission_durable_ilm_manifest_for_run(
16480 &self,
16481 source_pool_idx: usize,
16482 run_token: &str,
16483 ) -> Result<HashMap<String, DecommissionDurableIlmReceipt>> {
16484 let receipt_paths = self
16485 .list_decommission_durable_ilm_manifest_receipts_for_run(source_pool_idx, run_token)
16486 .await?;
16487 let manifest_path = decommission_durable_ilm_manifest_path(run_token);
16488 let data = read_config_limited_preserve_empty(
16489 self.pools[source_pool_idx].clone(),
16490 &manifest_path,
16491 DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE,
16492 )
16493 .await
16494 .map_err(|err| {
16495 Error::other(format!(
16496 "failed to read durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}"
16497 ))
16498 })?;
16499 DecommissionDurableIlmManifest::decode(&data, run_token, &receipt_paths).map_err(|err| {
16500 Error::other(format!(
16501 "durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx} is invalid: {err}"
16502 ))
16503 })?;
16504
16505 let mut receipts = HashMap::with_capacity(receipt_paths.len());
16506 for receipt_path in receipt_paths {
16507 let receipt = self
16508 .read_decommission_durable_ilm_receipt(source_pool_idx, &receipt_path)
16509 .await?;
16510 if receipts.insert(receipt_path.clone(), receipt).is_some() {
16511 return Err(Error::other(format!(
16512 "durable ILM expected manifest contains duplicate receipt path `{receipt_path}`"
16513 )));
16514 }
16515 }
16516 Ok(receipts)
16517 }
16518
16519 #[cfg(all(test, feature = "test-util"))]
16520 async fn persist_decommission_durable_ilm_receipt(
16521 &self,
16522 source_pool_idx: usize,
16523 target_pool_idx: usize,
16524 receipt: &DecommissionDurableIlmReceipt,
16525 ) -> Result<()> {
16526 let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
16527 self.persist_decommission_durable_ilm_receipt_for_run(target_pool_idx, receipt, &run_token)
16528 .await
16529 }
16530
16531 async fn persist_decommission_durable_ilm_receipt_for_run(
16532 &self,
16533 target_pool_idx: usize,
16534 receipt: &DecommissionDurableIlmReceipt,
16535 run_token: &str,
16536 ) -> Result<()> {
16537 let receipt_path = decommission_durable_ilm_receipt_path(run_token, &receipt.source_path, &receipt.id_kind, &receipt.id);
16538 let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
16539 let mut attempt = 1;
16540 loop {
16541 let (merged, http_preconditions) = match read_config_limited_preserve_empty_with_metadata(
16542 self.pools[target_pool_idx].clone(),
16543 &receipt_path,
16544 DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
16545 )
16546 .await
16547 {
16548 Ok((existing_data, metadata)) => {
16549 let existing = DecommissionDurableIlmReceipt::decode(&existing_data).map_err(|err| {
16550 Error::other(format!(
16551 "durable ILM decommission receipt `{receipt_path}` in pool {target_pool_idx} for {} is invalid: {err}",
16552 locator.context()
16553 ))
16554 })?;
16555 Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &existing)?;
16556 let merged = merge_decommission_durable_ilm_receipts(&existing, receipt)?;
16557 if merged == existing {
16558 return Ok(());
16559 }
16560 let etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| {
16561 Error::other(format!(
16562 "durable ILM decommission receipt `{receipt_path}` in pool {target_pool_idx} is missing an ETag"
16563 ))
16564 })?;
16565 (
16566 merged,
16567 HTTPPreconditions {
16568 if_match: Some(etag),
16569 ..Default::default()
16570 },
16571 )
16572 }
16573 Err(err)
16574 if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
16575 || is_err_object_not_found(&err)
16576 || is_err_version_not_found(&err) =>
16577 {
16578 (
16579 receipt.clone(),
16580 HTTPPreconditions {
16581 if_none_match: Some("*".to_string()),
16582 ..Default::default()
16583 },
16584 )
16585 }
16586 Err(err) => {
16587 return Err(Error::other(format!(
16588 "failed to read durable ILM decommission receipt `{receipt_path}` from pool {target_pool_idx} for {}: {err}",
16589 locator.context()
16590 )));
16591 }
16592 };
16593 let encoded = merged.encode().map_err(|err| {
16594 Error::other(format!(
16595 "failed to encode durable ILM decommission receipt `{receipt_path}` for source path `{}` {}: {err}",
16596 receipt.source_path,
16597 receipt.context()
16598 ))
16599 })?;
16600 match save_config_with_opts(
16601 self.pools[target_pool_idx].clone(),
16602 &receipt_path,
16603 encoded,
16604 &ObjectOptions {
16605 max_parity: true,
16606 write_completion: crate::object_api::WriteCompletion::TailDrained,
16607 http_preconditions: Some(http_preconditions),
16608 ..Default::default()
16609 },
16610 )
16611 .await
16612 {
16613 Ok(()) => return Ok(()),
16614 Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => {
16615 attempt += 1;
16616 }
16617 Err(Error::PreconditionFailed) => {
16618 return Err(Error::other(format!(
16619 "failed to persist durable ILM decommission receipt `{receipt_path}` for {} after concurrent updates",
16620 locator.context()
16621 )));
16622 }
16623 Err(err) => {
16624 return Err(Error::other(format!(
16625 "failed to persist durable ILM decommission receipt `{receipt_path}` for {}: {err}",
16626 locator.context()
16627 )));
16628 }
16629 }
16630 }
16631 }
16632
16633 fn validate_decommission_durable_ilm_receipt_locator(
16634 receipt_path: &str,
16635 locator: &DecommissionDurableIlmReceiptLocator,
16636 receipt: &DecommissionDurableIlmReceipt,
16637 ) -> Result<()> {
16638 if locator.source_path != receipt.source_path || locator.id_kind != receipt.id_kind || locator.id != receipt.id {
16639 return Err(Error::other(format!(
16640 "durable ILM decommission receipt path `{receipt_path}` identity {} does not match receipt {}",
16641 locator.context(),
16642 receipt.context()
16643 )));
16644 }
16645 Ok(())
16646 }
16647
16648 async fn read_decommission_durable_ilm_receipt(
16649 &self,
16650 receipt_pool_idx: usize,
16651 receipt_path: &str,
16652 ) -> Result<DecommissionDurableIlmReceipt> {
16653 let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?;
16654 let data = read_config_limited_preserve_empty(
16655 self.pools[receipt_pool_idx].clone(),
16656 receipt_path,
16657 DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
16658 )
16659 .await
16660 .map_err(|err| {
16661 Error::other(format!(
16662 "failed to read durable ILM decommission receipt `{receipt_path}` from pool {receipt_pool_idx} for {}: {err}",
16663 locator.context()
16664 ))
16665 })?;
16666 let receipt = DecommissionDurableIlmReceipt::decode(&data).map_err(|err| {
16667 Error::other(format!(
16668 "durable ILM decommission receipt `{receipt_path}` in pool {receipt_pool_idx} for {} is invalid: {err}",
16669 locator.context()
16670 ))
16671 })?;
16672 Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?;
16673 Ok(receipt)
16674 }
16675
16676 async fn load_decommission_durable_ilm_terminal_receipt_for_run(
16677 &self,
16678 source_pool_idx: usize,
16679 path: &str,
16680 source_record: &ValidatedDurableIlmRecord,
16681 run_token: &str,
16682 ) -> Result<Option<DecommissionDurableIlmReceipt>> {
16683 let receipt_path = decommission_durable_ilm_receipt_path(run_token, path, source_record.id_kind, &source_record.id);
16684 let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
16685 let mut proof = None::<DecommissionDurableIlmReceipt>;
16686 let mut nonterminal_receipt_found = false;
16687 for pool_idx in 0..self.pools.len() {
16688 if pool_idx == source_pool_idx {
16689 continue;
16690 }
16691 let data = match read_config_limited_preserve_empty(
16692 self.pools[pool_idx].clone(),
16693 &receipt_path,
16694 DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
16695 )
16696 .await
16697 {
16698 Ok(data) => data,
16699 Err(err)
16700 if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
16701 || is_err_object_not_found(&err)
16702 || is_err_version_not_found(&err) =>
16703 {
16704 continue;
16705 }
16706 Err(err) => {
16707 return Err(Error::other(format!(
16708 "failed to read terminal durable ILM decommission receipt `{receipt_path}` from pool {pool_idx} for {}: {err}",
16709 source_record.context()
16710 )));
16711 }
16712 };
16713 let receipt = DecommissionDurableIlmReceipt::decode(&data).map_err(|err| {
16714 Error::other(format!(
16715 "terminal durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} for {} is invalid: {err}",
16716 source_record.context()
16717 ))
16718 })?;
16719 Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &receipt)?;
16720 if receipt.namespace != source_record.namespace
16721 || receipt.id_kind != source_record.id_kind
16722 || receipt.id != source_record.id
16723 {
16724 return Err(Error::other(format!(
16725 "terminal durable ILM decommission receipt identity mismatch at path `{path}` {}; receipt {}",
16726 source_record.context(),
16727 receipt.context()
16728 )));
16729 }
16730 if let Some(terminal_checkpoint) = receipt.terminal_checkpoint.as_ref() {
16731 if !source_record.checkpoint.is_predecessor_of_terminal(terminal_checkpoint) {
16732 return Err(Error::other_with_context(
16733 "terminal durable ILM decommission receipt does not cover source",
16734 format!("path `{path}` {}", source_record.context()),
16735 ));
16736 }
16737 proof = Some(match proof {
16738 Some(existing) => merge_decommission_durable_ilm_receipts(&existing, &receipt)?,
16739 None => receipt,
16740 });
16741 } else {
16742 source_record
16743 .checkpoint
16744 .validate_successor(&receipt.checkpoint)
16745 .map_err(|err| {
16746 Error::other_with_context(
16747 "durable ILM decommission receipt does not cover source",
16748 format!("path `{path}` {}: {err}", source_record.context()),
16749 )
16750 })?;
16751 nonterminal_receipt_found = true;
16752 }
16753 }
16754 if nonterminal_receipt_found { Ok(None) } else { Ok(proof) }
16759 }
16760
16761 async fn verify_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> {
16762 let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
16763 self.verify_decommission_durable_ilm_receipts_for_run(source_pool_idx, &run_token)
16764 .await
16765 }
16766
16767 async fn verify_decommission_durable_ilm_receipts_for_run(&self, source_pool_idx: usize, run_token: &str) -> Result<()> {
16768 let expected_receipts = self
16769 .load_decommission_durable_ilm_manifest_for_run(source_pool_idx, run_token)
16770 .await?;
16771 let receipt_paths = self
16772 .list_decommission_durable_ilm_receipts_for_run(source_pool_idx, run_token)
16773 .await?;
16774 let present_receipt_paths = receipt_paths
16775 .iter()
16776 .map(|(_, receipt_path)| receipt_path.as_str())
16777 .collect::<HashSet<_>>();
16778 for (expected_path, expected) in &expected_receipts {
16779 if !present_receipt_paths.contains(expected_path.as_str()) {
16780 return Err(Error::other(format!(
16781 "durable ILM decommission receipt is missing at `{expected_path}` for source path `{}` {}",
16782 expected.source_path,
16783 expected.context()
16784 )));
16785 }
16786 }
16787
16788 for (receipt_pool_idx, receipt_path) in receipt_paths {
16789 let expected = expected_receipts.get(&receipt_path).ok_or_else(|| {
16790 Error::other(format!(
16791 "durable ILM decommission receipt `{receipt_path}` in pool {receipt_pool_idx} is absent from the expected manifest"
16792 ))
16793 })?;
16794 let receipt = self
16795 .read_decommission_durable_ilm_receipt(receipt_pool_idx, &receipt_path)
16796 .await?;
16797 if receipt.source_path != expected.source_path
16798 || receipt.namespace != expected.namespace
16799 || receipt.id_kind != expected.id_kind
16800 || receipt.id != expected.id
16801 {
16802 return Err(Error::other(format!(
16803 "durable ILM decommission receipt identity mismatch at `{receipt_path}` for source path `{}` {}; decoded {}",
16804 expected.source_path,
16805 expected.context(),
16806 receipt.context()
16807 )));
16808 }
16809 expected.checkpoint.validate_successor(&receipt.checkpoint).map_err(|err| {
16810 Error::other(format!(
16811 "durable ILM decommission receipt generation mismatch at `{receipt_path}` for source path `{}` {}: {err}",
16812 expected.source_path,
16813 expected.context()
16814 ))
16815 })?;
16816 match (&expected.terminal_checkpoint, &receipt.terminal_checkpoint) {
16817 (Some(expected_terminal), Some(receipt_terminal)) => {
16818 expected_terminal.validate_successor(receipt_terminal).map_err(|err| {
16819 Error::other(format!(
16820 "durable ILM decommission terminal receipt generation mismatch at `{receipt_path}` for source path `{}` {}: {err}",
16821 expected.source_path,
16822 expected.context()
16823 ))
16824 })?;
16825 }
16826 (Some(_), None) => {
16827 return Err(Error::other(format!(
16828 "durable ILM decommission terminal receipt is missing at `{receipt_path}` for source path `{}` {}",
16829 expected.source_path,
16830 expected.context()
16831 )));
16832 }
16833 (None, _) => {}
16834 }
16835 let namespace = classify_durable_ilm_record(&receipt.source_path)?
16836 .ok_or_else(|| Error::other(format!("path `{}` is not a durable ILM record", receipt.source_path)))?;
16837 let target = self
16838 .load_decommissioned_durable_ilm_target(
16839 source_pool_idx,
16840 &receipt.source_path,
16841 namespace.max_record_size,
16842 &receipt.context(),
16843 )
16844 .await?;
16845 if let Some((_, target)) = target {
16846 let target_record = validate_durable_ilm_record(&receipt.source_path, &target).map_err(|err| {
16847 Error::other(format!(
16848 "target durable ILM record is invalid at path `{}` {}: {err}",
16849 receipt.source_path,
16850 receipt.context()
16851 ))
16852 })?;
16853 let identity_matches = target_record.namespace == receipt.namespace
16854 && target_record.id_kind == receipt.id_kind
16855 && target_record.id == receipt.id;
16856 let reused_manual_scope = receipt.terminal_checkpoint.is_some()
16857 && matches!(
16858 (&receipt.checkpoint, &target_record.checkpoint),
16859 (
16860 DurableIlmRecordCheckpoint::ManualTransitionScope { .. },
16861 DurableIlmRecordCheckpoint::ManualTransitionScope { .. }
16862 )
16863 );
16864 if !identity_matches && !reused_manual_scope {
16865 return Err(Error::other(format!(
16866 "target durable ILM record identity mismatch at path `{}` {}; decoded {}",
16867 receipt.source_path,
16868 receipt.context(),
16869 target_record.context()
16870 )));
16871 }
16872 if identity_matches {
16873 receipt
16874 .terminal_checkpoint
16875 .as_ref()
16876 .unwrap_or(&receipt.checkpoint)
16877 .validate_successor(&target_record.checkpoint)
16878 .map_err(|err| {
16879 Error::other(format!(
16880 "target durable ILM record generation mismatch at path `{}` {}: {err}",
16881 receipt.source_path,
16882 receipt.context()
16883 ))
16884 })?;
16885 }
16886 } else if receipt.terminal_checkpoint.is_none() {
16887 return Err(Error::other(format!(
16888 "target durable ILM record is missing at path `{}` {} without a recovery terminal checkpoint",
16889 receipt.source_path,
16890 receipt.context()
16891 )));
16892 }
16893 }
16894 Ok(())
16895 }
16896
16897 async fn advance_durable_ilm_decommission_receipt(
16898 &self,
16899 pool_idx: usize,
16900 receipt_path: &str,
16901 record: &ValidatedDurableIlmRecord,
16902 fleet_topology_generation: Option<&str>,
16903 terminal: bool,
16904 ) -> Result<bool> {
16905 let stage = if terminal { "terminal" } else { "progress" };
16906 let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?;
16907 let mut attempt = 1;
16908 loop {
16909 let (receipt_data, metadata) = match read_config_limited_preserve_empty_with_metadata(
16910 self.pools[pool_idx].clone(),
16911 receipt_path,
16912 DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
16913 )
16914 .await
16915 {
16916 Ok(receipt) => receipt,
16917 Err(err)
16918 if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
16919 || is_err_object_not_found(&err)
16920 || is_err_version_not_found(&err) =>
16921 {
16922 return Ok(false);
16923 }
16924 Err(err) => {
16925 return Err(Error::other(format!(
16926 "failed to read durable ILM decommission receipt `{receipt_path}` from pool {pool_idx} for {}: {err}",
16927 locator.context()
16928 )));
16929 }
16930 };
16931 let mut receipt = DecommissionDurableIlmReceipt::decode(&receipt_data).map_err(|err| {
16932 Error::other(format!(
16933 "durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} for {} is invalid: {err}",
16934 locator.context()
16935 ))
16936 })?;
16937 Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?;
16938 if receipt.fleet_topology_generation.as_deref() != fleet_topology_generation {
16939 return Err(Error::other_with_context(
16940 "durable ILM decommission receipt fleet topology mismatch",
16941 format!("path `{}` {}", receipt.source_path, receipt.context()),
16942 ));
16943 }
16944 receipt.checkpoint.validate_successor(&record.checkpoint).map_err(|err| {
16945 Error::other(format!(
16946 "{stage} durable ILM record generation mismatch at path `{}` {}: {err}",
16947 receipt.source_path,
16948 receipt.context()
16949 ))
16950 })?;
16951
16952 if terminal {
16953 if let Some(existing) = &receipt.terminal_checkpoint {
16954 if existing == &record.checkpoint || record.checkpoint.validate_successor(existing).is_ok() {
16955 return Ok(true);
16956 }
16957 existing.validate_successor(&record.checkpoint).map_err(|err| {
16958 Error::other(format!(
16959 "terminal durable ILM record checkpoint conflicts at path `{}` {}: {err}",
16960 receipt.source_path,
16961 receipt.context()
16962 ))
16963 })?;
16964 }
16965 receipt.terminal_checkpoint = Some(record.checkpoint.clone());
16966 } else {
16967 if let Some(existing) = &receipt.terminal_checkpoint {
16968 if existing == &record.checkpoint {
16969 return Ok(true);
16970 }
16971 existing.validate_successor(&record.checkpoint).map_err(|err| {
16972 Error::other(format!(
16973 "progress durable ILM record conflicts with terminal checkpoint at path `{}` {}: {err}",
16974 receipt.source_path,
16975 receipt.context()
16976 ))
16977 })?;
16978 receipt.terminal_checkpoint = None;
16979 }
16980 if receipt.checkpoint == record.checkpoint {
16981 return Ok(true);
16982 }
16983 receipt.checkpoint = record.checkpoint.clone();
16984 }
16985
16986 let etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| {
16987 Error::other(format!(
16988 "durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} is missing an ETag"
16989 ))
16990 })?;
16991 let encoded = receipt.encode()?;
16992 match save_config_with_opts(
16993 self.pools[pool_idx].clone(),
16994 receipt_path,
16995 encoded,
16996 &ObjectOptions {
16997 max_parity: true,
16998 write_completion: crate::object_api::WriteCompletion::TailDrained,
16999 http_preconditions: Some(HTTPPreconditions {
17000 if_match: Some(etag),
17001 ..Default::default()
17002 }),
17003 ..Default::default()
17004 },
17005 )
17006 .await
17007 {
17008 Ok(()) => return Ok(true),
17009 Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => {
17010 attempt += 1;
17011 continue;
17012 }
17013 Err(Error::PreconditionFailed) => {
17014 return Err(Error::other(format!(
17015 "failed to persist {stage} durable ILM decommission receipt `{receipt_path}` for {} after concurrent updates",
17016 locator.context()
17017 )));
17018 }
17019 Err(err) => {
17020 return Err(Error::other(format!(
17021 "failed to persist {stage} durable ILM decommission receipt `{receipt_path}` for {}: {err}",
17022 locator.context()
17023 )));
17024 }
17025 }
17026 }
17027 }
17028
17029 async fn advance_durable_ilm_decommission_receipts(
17030 &self,
17031 path: &str,
17032 data: &[u8],
17033 terminal: bool,
17034 ) -> Result<Option<Vec<usize>>> {
17035 let active_runs = {
17036 let pool_meta = self.pool_meta.read().await;
17037 pool_meta
17038 .pools
17039 .iter()
17040 .enumerate()
17041 .filter_map(|(pool_idx, pool)| {
17042 pool.decommission
17043 .as_ref()
17044 .filter(|info| info.has_decommission_state() && !info.complete)
17045 .and_then(|info| info.start_time)
17046 .map(|start_time| (pool_idx, decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time)))
17047 })
17048 .collect::<Vec<_>>()
17049 };
17050 if active_runs.is_empty() {
17051 return Ok(None);
17052 }
17053
17054 let stage = if terminal { "terminal" } else { "progress" };
17055 let record = validate_durable_ilm_record(path, data)
17056 .map_err(|err| Error::other(format!("{stage} durable ILM record is invalid at path `{path}`: {err}")))?;
17057 let fleet_topology_generation = durable_ilm_v6_topology_generation(path, data)?;
17058 let active_source_pool_indices = active_runs.iter().map(|(pool_idx, _)| *pool_idx).collect::<Vec<_>>();
17059 let mut terminal_target_pool_indices = Vec::new();
17060 for (source_pool_idx, run_token) in active_runs {
17061 let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, record.id_kind, &record.id);
17062 let mut receipt_found = false;
17063 for pool_idx in 0..self.pools.len() {
17064 if pool_idx != source_pool_idx {
17065 let found = self
17066 .advance_durable_ilm_decommission_receipt(
17067 pool_idx,
17068 &receipt_path,
17069 &record,
17070 fleet_topology_generation.as_deref(),
17071 terminal,
17072 )
17073 .await?;
17074 receipt_found |= found;
17075 if terminal
17076 && found
17077 && !active_source_pool_indices.contains(&pool_idx)
17078 && !terminal_target_pool_indices.contains(&pool_idx)
17079 {
17080 terminal_target_pool_indices.push(pool_idx);
17081 }
17082 }
17083 }
17084 if terminal && !receipt_found {
17085 return Err(Error::other(format!(
17086 "terminal durable ILM record at path `{path}` {} is retained until its decommission receipt is committed",
17087 record.context()
17088 )));
17089 }
17090 }
17091 Ok(Some(terminal_target_pool_indices))
17092 }
17093
17094 pub(crate) async fn decommission_durable_ilm_checkpoint_targets(
17102 &self,
17103 path: &str,
17104 next_data: &[u8],
17105 expected_etag: &str,
17106 ) -> Result<Option<Vec<DecommissionDurableIlmCheckpointTarget>>> {
17107 let active_runs = {
17108 let pool_meta = self.pool_meta.read().await;
17109 let mut active_runs = Vec::new();
17110 for (source_pool_index, pool) in pool_meta.pools.iter().enumerate() {
17111 let Some(info) = pool
17112 .decommission
17113 .as_ref()
17114 .filter(|info| info.has_decommission_state() && !info.complete)
17115 else {
17116 continue;
17117 };
17118 let Some(start_time) = info.start_time else {
17119 continue;
17120 };
17121 let reservation = info.capacity_reservation.clone().ok_or_else(|| {
17122 decommission_capacity_blocked_error(format!(
17123 "active decommission source pool {source_pool_index} has no reservation for durable ILM checkpoint"
17124 ))
17125 })?;
17126 if !reservation.active() {
17127 return Err(decommission_capacity_blocked_error(format!(
17128 "active decommission source pool {source_pool_index} has a released reservation for durable ILM checkpoint"
17129 )));
17130 }
17131 active_runs.push((
17132 source_pool_index,
17133 decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time),
17134 reservation,
17135 ));
17136 }
17137 active_runs
17138 };
17139 if active_runs.is_empty() {
17140 return Ok(None);
17141 }
17142
17143 let next_record = validate_durable_ilm_record(path, next_data)?;
17144 if next_record.namespace != TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.name {
17145 return Ok(None);
17146 }
17147 let next_fleet_topology_generation = durable_ilm_v6_topology_generation(path, next_data)?;
17148
17149 let mut targets = Vec::<DecommissionDurableIlmCheckpointTarget>::new();
17150 let mut missing_source_receipts = 0usize;
17151 let mut stale_target_etag_mismatch = false;
17152 for (source_pool_index, run_token, reservation) in active_runs {
17153 let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, next_record.id_kind, &next_record.id);
17154 let mut source_receipt_found = false;
17155 for allocation in &reservation.targets {
17156 let receipt_data = match read_config_limited_preserve_empty(
17157 self.pools[allocation.pool_index].clone(),
17158 &receipt_path,
17159 DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
17160 )
17161 .await
17162 {
17163 Ok(data) => data,
17164 Err(err)
17165 if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17166 || is_err_object_not_found(&err)
17167 || is_err_version_not_found(&err) =>
17168 {
17169 continue;
17170 }
17171 Err(err) => return Err(err),
17172 };
17173 let receipt = DecommissionDurableIlmReceipt::decode(&receipt_data)?;
17174 let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
17175 Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &receipt)?;
17176 if receipt.source_path != path
17177 || receipt.namespace != next_record.namespace
17178 || receipt.id_kind != next_record.id_kind
17179 || receipt.id != next_record.id
17180 {
17181 return Err(Error::other_with_context(
17182 "durable ILM checkpoint receipt identity does not authorize source",
17183 format!("path `{path}` {}", next_record.context()),
17184 ));
17185 }
17186 if receipt.fleet_topology_generation != next_fleet_topology_generation {
17187 return Err(Error::other_with_context(
17188 "durable ILM checkpoint receipt fleet topology does not authorize source",
17189 format!("path `{path}` {}", next_record.context()),
17190 ));
17191 }
17192 receipt
17193 .checkpoint
17194 .validate_successor(&next_record.checkpoint)
17195 .map_err(|err| {
17196 Error::other_with_context(
17197 "durable ILM checkpoint receipt is not a predecessor of the requested generation",
17198 format!(
17199 "target pool {}, path `{path}` {}; receipt checkpoint {:?}, requested checkpoint {:?}: {err}",
17200 allocation.pool_index,
17201 next_record.context(),
17202 receipt.checkpoint,
17203 next_record.checkpoint
17204 ),
17205 )
17206 })?;
17207
17208 let (target_data, metadata) = read_config_limited_preserve_empty_with_metadata(
17209 self.pools[allocation.pool_index].clone(),
17210 path,
17211 TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.max_record_size,
17212 )
17213 .await?;
17214 let target_record = validate_durable_ilm_record(path, &target_data)?;
17215 if target_record.namespace != next_record.namespace
17216 || target_record.id_kind != next_record.id_kind
17217 || target_record.id != next_record.id
17218 {
17219 return Err(Error::other_with_context(
17220 "durable ILM checkpoint target identity does not match source",
17221 format!("path `{path}` {}", next_record.context()),
17222 ));
17223 }
17224 let already_committed = target_data.as_slice() == next_data;
17225 let target_etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| {
17226 Error::other_with_context("durable ILM checkpoint target is missing an ETag", format!("path `{path}`"))
17227 })?;
17228 if already_committed {
17229 receipt
17230 .checkpoint
17231 .validate_successor(&target_record.checkpoint)
17232 .map_err(|err| {
17233 Error::other_with_context(
17234 "durable ILM checkpoint receipt is not a predecessor of the committed target generation",
17235 format!("path `{path}` {}: {err}", next_record.context()),
17236 )
17237 })?;
17238 } else {
17239 if target_record.checkpoint != receipt.checkpoint {
17240 return Err(Error::other_with_context(
17241 "durable ILM checkpoint target is not the receipt generation",
17242 format!("path `{path}` {}", next_record.context()),
17243 ));
17244 }
17245 target_record
17246 .checkpoint
17247 .validate_successor(&next_record.checkpoint)
17248 .map_err(|err| {
17249 Error::other_with_context(
17250 "durable ILM checkpoint target is not a predecessor of the requested generation",
17251 format!("path `{path}` {}: {err}", next_record.context()),
17252 )
17253 })?;
17254 if next_data.len() > target_data.len() {
17255 return Err(decommission_capacity_blocked_error(format!(
17256 "durable ILM checkpoint update at `{path}` grows from {} to {} bytes",
17257 target_data.len(),
17258 next_data.len()
17259 )));
17260 }
17261 stale_target_etag_mismatch |= target_etag != expected_etag;
17262 }
17263
17264 source_receipt_found = true;
17265 if let Some(existing) = targets
17266 .iter_mut()
17267 .find(|target| target.target_pool_index == allocation.pool_index)
17268 {
17269 if existing.already_committed != already_committed {
17270 return Err(Error::other_with_context(
17271 "durable ILM checkpoint target state changed during authorization",
17272 format!("path `{path}`"),
17273 ));
17274 }
17275 continue;
17276 }
17277 let base_owner = DecommissionCapacityOwner {
17278 source_pool_index,
17279 operation_id: reservation.operation_id,
17280 generation: reservation.generation,
17281 owner_nonce: reservation.owner_nonce,
17282 mutation_id: None,
17283 };
17284 let mutation_id = decommission_capacity_mutation_id(
17285 base_owner,
17286 RUSTFS_META_BUCKET,
17287 path,
17288 Some(next_record.checkpoint.content_sha256()),
17289 false,
17290 None,
17291 );
17292 targets.push(DecommissionDurableIlmCheckpointTarget {
17293 source_pool_index,
17294 target_pool_index: allocation.pool_index,
17295 capacity_owner: base_owner.with_mutation_id(mutation_id),
17296 already_committed,
17297 target_etag: Some(target_etag),
17298 });
17299 }
17300 if !source_receipt_found {
17301 missing_source_receipts = missing_source_receipts.saturating_add(1);
17302 }
17303 }
17304
17305 if missing_source_receipts > 0 {
17306 return Err(decommission_capacity_blocked_error(format!(
17307 "durable ILM checkpoint at `{path}` is missing receipt coverage for {missing_source_receipts} active source(s)"
17308 )));
17309 }
17310 if targets.is_empty() {
17311 return Err(decommission_capacity_blocked_error(format!(
17312 "durable ILM checkpoint at `{path}` has no receipt-bearing reservation target"
17313 )));
17314 }
17315 if stale_target_etag_mismatch && !targets.iter().any(|target| target.already_committed) {
17316 return Err(Error::PreconditionFailed);
17317 }
17318 targets.sort_unstable_by_key(|target| target.target_pool_index);
17325 Ok(Some(targets))
17326 }
17327
17328 pub(crate) async fn record_durable_ilm_decommission_progress(&self, path: &str, data: &[u8]) -> Result<()> {
17329 self.advance_durable_ilm_decommission_receipts(path, data, false)
17330 .await
17331 .map(|_| ())
17332 }
17333
17334 pub(crate) async fn record_durable_ilm_decommission_terminal(&self, path: &str, data: &[u8]) -> Result<()> {
17335 self.record_durable_ilm_decommission_terminal_target_pools(path, data)
17336 .await
17337 .map(|_| ())
17338 }
17339
17340 pub(crate) async fn record_durable_ilm_decommission_terminal_target_pools(
17342 &self,
17343 path: &str,
17344 data: &[u8],
17345 ) -> Result<Option<Vec<usize>>> {
17346 self.advance_durable_ilm_decommission_receipts(path, data, true).await
17347 }
17348
17349 pub(crate) async fn durable_ilm_terminal_receipt_covers_active_source(&self, path: &str, data: &[u8]) -> Result<bool> {
17354 let namespace = classify_durable_ilm_record(path)?
17355 .ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{path}`")))?;
17356 let source_record = validate_durable_ilm_record(path, data)?;
17357 let active_runs = {
17358 let pool_meta = self.pool_meta.read().await;
17359 pool_meta
17360 .pools
17361 .iter()
17362 .enumerate()
17363 .filter_map(|(source_pool_index, pool)| {
17364 pool.decommission
17365 .as_ref()
17366 .filter(|info| info.has_decommission_state() && !info.complete)
17367 .and_then(|info| info.start_time)
17368 .map(|start_time| {
17369 (source_pool_index, decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time))
17370 })
17371 })
17372 .collect::<Vec<_>>()
17373 };
17374 let mut covered_active_source = false;
17375 for (source_pool_index, run_token) in active_runs {
17376 let source_data =
17377 match read_config_limited_preserve_empty(self.pools[source_pool_index].clone(), path, namespace.max_record_size)
17378 .await
17379 {
17380 Ok(source_data) => source_data,
17381 Err(err)
17382 if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17383 || is_err_object_not_found(&err)
17384 || is_err_version_not_found(&err) =>
17385 {
17386 continue;
17387 }
17388 Err(err) => return Err(err),
17389 };
17390 if source_data.as_slice() != data {
17391 continue;
17392 }
17393 if self
17394 .load_decommission_durable_ilm_terminal_receipt_for_run(source_pool_index, path, &source_record, &run_token)
17395 .await?
17396 .is_some()
17397 {
17398 covered_active_source = true;
17399 } else {
17400 return Ok(false);
17404 }
17405 }
17406 Ok(covered_active_source)
17407 }
17408
17409 async fn cleanup_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> {
17410 for (pool_idx, receipt_path) in self.list_decommission_durable_ilm_receipts(source_pool_idx).await? {
17411 match delete_config(self.pools[pool_idx].clone(), &receipt_path).await {
17412 Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {}
17413 Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
17414 Err(err) => {
17415 return Err(Error::other(format!(
17416 "failed to clean durable ILM decommission receipt `{receipt_path}` from pool {pool_idx}: {err}"
17417 )));
17418 }
17419 }
17420 }
17421 for receipt_path in self.list_decommission_durable_ilm_manifest_receipts(source_pool_idx).await? {
17422 match delete_config(self.pools[source_pool_idx].clone(), &receipt_path).await {
17423 Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {}
17424 Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
17425 Err(err) => {
17426 return Err(Error::other(format!(
17427 "failed to clean durable ILM expected manifest receipt `{receipt_path}` from source pool {source_pool_idx}: {err}"
17428 )));
17429 }
17430 }
17431 }
17432 let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
17433 let manifest_path = decommission_durable_ilm_manifest_path(&run_token);
17434 match delete_config(self.pools[source_pool_idx].clone(), &manifest_path).await {
17435 Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {}
17436 Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
17437 Err(err) => {
17438 return Err(Error::other(format!(
17439 "failed to clean durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}"
17440 )));
17441 }
17442 }
17443 Ok(())
17444 }
17445
17446 #[cfg(all(test, feature = "test-util"))]
17447 async fn verify_and_cleanup_decommissioned_durable_ilm_record(
17448 &self,
17449 source_pool_idx: usize,
17450 source_set: Arc<SetDisks>,
17451 path: &str,
17452 ) -> Result<()> {
17453 let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
17454 self.verify_and_cleanup_decommissioned_durable_ilm_record_for_run(source_pool_idx, source_set, path, &run_token)
17455 .await
17456 }
17457
17458 async fn verify_and_cleanup_decommissioned_durable_ilm_record_for_run(
17459 &self,
17460 source_pool_idx: usize,
17461 source_set: Arc<SetDisks>,
17462 path: &str,
17463 run_token: &str,
17464 ) -> Result<()> {
17465 let namespace = classify_durable_ilm_record(path)?
17466 .ok_or_else(|| Error::other(format!("path `{path}` is not a durable ILM record")))?;
17467 let source_versions = source_set
17468 .load_file_info_versions_exact(RUSTFS_META_BUCKET, path)
17469 .await
17470 .map_err(|err| Error::other(format!("failed to load source durable ILM versions at path `{path}`: {err}")))?
17471 .ok_or_else(|| Error::other(format!("source durable ILM record is missing at path `{path}`")))?;
17472 let source = read_config_limited_preserve_empty(source_set.clone(), path, namespace.max_record_size)
17473 .await
17474 .map_err(|err| Error::other(format!("failed to read source durable ILM record at path `{path}`: {err}")))?;
17475 let source_record = validate_durable_ilm_record(path, &source)
17476 .map_err(|err| Error::other(format!("source durable ILM record is invalid at path `{path}`: {err}")))?;
17477 let source_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &source)?;
17478 let target = self
17479 .load_decommissioned_durable_ilm_target(source_pool_idx, path, namespace.max_record_size, &source_record.context())
17480 .await?;
17481 let manifest_receipt = if let Some((target_pool_idx, target)) = target {
17482 let target_record = validate_decommission_durable_ilm_copy(path, &source_record, &target)?;
17483 let target_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &target)?;
17484 if target_fleet_topology_generation != source_fleet_topology_generation {
17485 return Err(Error::other_with_context(
17486 "target durable ILM fleet topology generation differs from source",
17487 format!("path `{path}` {}", source_record.context()),
17488 ));
17489 }
17490 let receipt = DecommissionDurableIlmReceipt::new(path, &target_record, target_fleet_topology_generation);
17491 self.persist_decommission_durable_ilm_receipt_for_run(target_pool_idx, &receipt, run_token)
17492 .await?;
17493 receipt
17494 } else {
17495 self.load_decommission_durable_ilm_terminal_receipt_for_run(source_pool_idx, path, &source_record, run_token)
17496 .await?
17497 .ok_or_else(|| {
17498 Error::other(format!(
17499 "target durable ILM record is missing at path `{path}` {} without a matching terminal receipt",
17500 source_record.context()
17501 ))
17502 })?
17503 };
17504 if manifest_receipt.fleet_topology_generation != source_fleet_topology_generation {
17505 return Err(Error::other_with_context(
17506 "terminal durable ILM receipt fleet topology generation does not cover source",
17507 format!("path `{path}` {}", source_record.context()),
17508 ));
17509 }
17510 let fleet_proof = if let Some(expected_generation) = source_fleet_topology_generation.as_deref() {
17511 let proof = acquire_tier_delete_journal_fleet_proof()
17512 .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable for source cleanup"))?;
17513 if tier_delete_journal_topology_generation(&proof) != expected_generation
17514 || !tier_delete_journal_fleet_proof_matches(&proof)
17515 {
17516 return Err(Error::other("tier delete journal v6 fleet generation changed before source cleanup"));
17517 }
17518 Some(proof)
17519 } else {
17520 None
17521 };
17522 self.persist_decommission_durable_ilm_receipt_for_run(source_pool_idx, &manifest_receipt, run_token)
17523 .await?;
17524
17525 if fleet_proof
17526 .as_ref()
17527 .is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof))
17528 {
17529 return Err(Error::other("tier delete journal v6 fleet proof changed before source cleanup"));
17530 }
17531
17532 let cleanup_result = data_movement::cleanup_source_entry_if_unchanged(
17533 source_set,
17534 RUSTFS_META_BUCKET,
17535 path,
17536 &source_versions,
17537 &[],
17538 data_movement::SourceCleanupBucketFence::default(),
17539 "decommission durable ILM final sweep",
17540 )
17541 .await
17542 .map_err(|err| {
17543 Error::other(format!(
17544 "source durable ILM cleanup failed at path `{path}` {}: {err}",
17545 source_record.context()
17546 ))
17547 });
17548 let cleanup_result = resolve_decommission_entry_cleanup_delete_result(cleanup_result, RUSTFS_META_BUCKET, path);
17549 if fleet_proof
17550 .as_ref()
17551 .is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof))
17552 {
17553 return Err(Error::other(
17554 "tier delete journal v6 fleet proof changed during source cleanup; exact source outcome requires verification",
17555 ));
17556 }
17557 cleanup_result
17558 }
17559
17560 #[cfg(all(test, feature = "test-util"))]
17561 pub(crate) async fn verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
17562 &self,
17563 source_pool_idx: usize,
17564 source_set: Arc<SetDisks>,
17565 path: &str,
17566 ) -> Result<()> {
17567 self.verify_and_cleanup_decommissioned_durable_ilm_record(source_pool_idx, source_set, path)
17568 .await
17569 }
17570
17571 #[cfg(all(test, feature = "test-util"))]
17572 pub(crate) async fn decommission_durable_ilm_receipt_count_for_test(&self, source_pool_idx: usize) -> Result<usize> {
17573 Ok(self.list_decommission_durable_ilm_receipts(source_pool_idx).await?.len())
17574 }
17575
17576 #[cfg(all(test, feature = "test-util"))]
17577 pub(crate) async fn decommission_durable_ilm_receipt_paths_for_test(
17578 &self,
17579 source_pool_idx: usize,
17580 ) -> Result<Vec<(usize, String)>> {
17581 self.list_decommission_durable_ilm_receipts(source_pool_idx).await
17582 }
17583
17584 #[cfg(all(test, feature = "test-util"))]
17585 pub(crate) async fn persist_decommission_durable_ilm_receipt_for_test(
17586 &self,
17587 source_pool_idx: usize,
17588 target_pool_idx: usize,
17589 source_path: &str,
17590 record: &ValidatedDurableIlmRecord,
17591 terminal: bool,
17592 ) -> Result<String> {
17593 let fleet_topology_generation = match read_config_limited_preserve_empty(
17594 self.pools[target_pool_idx].clone(),
17595 source_path,
17596 classify_durable_ilm_record(source_path)?
17597 .ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{source_path}`")))?
17598 .max_record_size,
17599 )
17600 .await
17601 {
17602 Ok(target_data) => durable_ilm_v6_topology_generation(source_path, &target_data)?,
17603 Err(err)
17604 if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17605 || is_err_object_not_found(&err)
17606 || is_err_version_not_found(&err) =>
17607 {
17608 None
17609 }
17610 Err(err) => return Err(err),
17611 };
17612 let mut receipt = DecommissionDurableIlmReceipt::new(source_path, record, fleet_topology_generation);
17613 if terminal {
17614 receipt.terminal_checkpoint = Some(record.checkpoint.clone());
17615 }
17616 self.persist_decommission_durable_ilm_receipt(source_pool_idx, target_pool_idx, &receipt)
17617 .await?;
17618 self.decommission_durable_ilm_receipt_path_for_test(source_pool_idx, source_path, record)
17619 .await
17620 }
17621
17622 #[cfg(all(test, feature = "test-util"))]
17623 pub(crate) async fn decommission_durable_ilm_receipt_path_for_test(
17624 &self,
17625 source_pool_idx: usize,
17626 source_path: &str,
17627 record: &ValidatedDurableIlmRecord,
17628 ) -> Result<String> {
17629 let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
17630 Ok(decommission_durable_ilm_receipt_path(&run_token, source_path, record.id_kind, &record.id))
17631 }
17632
17633 #[cfg(all(test, feature = "test-util"))]
17634 pub(crate) async fn persist_decommission_durable_ilm_manifest_for_test(&self, source_pool_idx: usize) -> Result<()> {
17635 self.persist_decommission_durable_ilm_manifest(source_pool_idx).await
17636 }
17637
17638 #[cfg(all(test, feature = "test-util"))]
17639 pub(crate) async fn cleanup_decommission_durable_ilm_receipts_for_test(&self, source_pool_idx: usize) -> Result<()> {
17640 self.cleanup_decommission_durable_ilm_receipts(source_pool_idx).await
17641 }
17642
17643 async fn check_after_decommission(
17644 self: &Arc<Self>,
17645 idx: usize,
17646 rx: &CancellationToken,
17647 generation: OffsetDateTime,
17648 ) -> Result<Vec<DecommissionUnresolvedEntry>> {
17649 let run_token = self.durable_ilm_receipt_run_token_for_generation(idx, generation).await?;
17650 let operation_gate = self.ctx.data_movement_operation_gate();
17651 self.run_guarded_decommission_side_effect(rx, &operation_gate, || {
17652 self.check_after_decommission_unfenced(idx, generation, run_token)
17653 })
17654 .await
17655 }
17656
17657 async fn check_after_decommission_unfenced(
17658 self: &Arc<Self>,
17659 idx: usize,
17660 generation: OffsetDateTime,
17661 run_token: String,
17662 ) -> Result<Vec<DecommissionUnresolvedEntry>> {
17663 let unresolved_entries = {
17664 let pool_meta = self.pool_meta.read().await;
17665 ensure_decommission_generation(&pool_meta, idx, generation)?;
17666 let info = pool_meta.pools[idx]
17667 .decommission
17668 .as_ref()
17669 .ok_or_else(|| decommission_metadata_not_initialized_error("verify unresolved decommission entries"))?;
17670 if info
17671 .unresolved_entries
17672 .iter()
17673 .any(|entry| entry.pool_index != idx || entry.source_generation != generation)
17674 {
17675 return Err(Error::other(format!(
17676 "failed to verify decommission for pool {idx}: unresolved listing ledger contains a different pool or generation"
17677 )));
17678 }
17679 info.unresolved_entries.clone()
17680 };
17681 let unresolved_entries_by_identity = Arc::new(
17682 unresolved_entries
17683 .iter()
17684 .map(|entry| (decommission_unresolved_entry_identity(entry), entry.clone()))
17685 .collect::<HashMap<_, _>>(),
17686 );
17687 let resolved_unresolved_entries = Arc::new(tokio::sync::Mutex::new(Vec::new()));
17688 let buckets = self.get_buckets_to_decommission().await?;
17689 let pool = self.pools[idx].clone();
17690
17691 self.ensure_decommission_multipart_uploads_drained(idx, &pool, &buckets)
17692 .await?;
17693
17694 for (set_index, set) in pool.disk_set.iter().enumerate() {
17695 let require_all_disks = unresolved_entries.iter().any(|entry| entry.set_index == set_index);
17696 for bucket_info in &buckets {
17697 let mut lifecycle_config = None;
17698 let mut object_lock_config = None;
17699 let mut replication_configured = false;
17700 if bucket_info.name != RUSTFS_META_BUCKET {
17701 let expiry_configs = get_expiry_configs(self, &bucket_info.name).await?;
17702 lifecycle_config = expiry_configs.lifecycle.map(|config| (*config).clone());
17703 object_lock_config = expiry_configs.object_lock.map(|config| (*config).clone());
17704 replication_configured = resolve_decommission_optional_bucket_config_result(
17705 &bucket_info.name,
17706 "replication",
17707 metadata_sys::get_replication_config_in(&self.ctx, &bucket_info.name).await,
17708 )?
17709 .is_some();
17710 }
17711
17712 let versions_found = Arc::new(AtomicUsize::new(0));
17713 let entry_error = Arc::new(tokio::sync::Mutex::new(None::<Error>));
17714 let first_remaining_path = Arc::new(tokio::sync::Mutex::new(None::<String>));
17715 let callback_rx = CancellationToken::new();
17716 let versions_found_cb = versions_found.clone();
17717 let entry_error_cb = entry_error.clone();
17718 let first_remaining_path_cb = first_remaining_path.clone();
17719 let bucket_name = bucket_info.name.clone();
17720 let lifecycle_config_cb = lifecycle_config.clone();
17721 let object_lock_config_cb = object_lock_config.clone();
17722 let replication_configured_cb = replication_configured;
17723 let store = Arc::clone(self);
17724 let source_set = set.clone();
17725 let callback_rx_cb = callback_rx.clone();
17726 let unresolved_entries_by_identity_cb = unresolved_entries_by_identity.clone();
17727 let resolved_unresolved_entries_cb = resolved_unresolved_entries.clone();
17728 let run_token_cb = run_token.clone();
17729
17730 let callback: ListCallback = Arc::new(move |entry: MetaCacheEntry| {
17731 let versions_found = versions_found_cb.clone();
17732 let entry_error = entry_error_cb.clone();
17733 let first_remaining_path = first_remaining_path_cb.clone();
17734 let bucket_name = bucket_name.clone();
17735 let lifecycle_config = lifecycle_config_cb.clone();
17736 let object_lock_config = object_lock_config_cb.clone();
17737 let replication_configured = replication_configured_cb;
17738 let store = Arc::clone(&store);
17739 let source_set = source_set.clone();
17740 let callback_rx = callback_rx_cb.clone();
17741 let unresolved_entries_by_identity = unresolved_entries_by_identity_cb.clone();
17742 let resolved_unresolved_entries = resolved_unresolved_entries_cb.clone();
17743 let run_token = run_token_cb.clone();
17744 Box::pin(async move {
17745 if callback_rx.is_cancelled() {
17746 return;
17747 }
17748
17749 if !entry.is_object() {
17750 return;
17751 }
17752
17753 if is_decommission_set_local_usage_cache(&bucket_name, &entry.name) {
17754 return;
17755 }
17756
17757 let durable_ilm_record = if bucket_name == RUSTFS_META_BUCKET {
17758 match classify_durable_ilm_record(&entry.name) {
17759 Ok(record) => record,
17760 Err(err) => {
17761 let mut first_err = entry_error.lock().await;
17762 if first_err.is_none() {
17763 *first_err = Some(with_decommission_entry_context(
17764 "check_after_decommission.durable_ilm_namespace",
17765 &bucket_name,
17766 &entry.name,
17767 err,
17768 ));
17769 callback_rx.cancel();
17770 }
17771 return;
17772 }
17773 }
17774 } else {
17775 None
17776 };
17777
17778 if durable_ilm_record.is_some() {
17779 if let Err(err) = store
17780 .verify_and_cleanup_decommissioned_durable_ilm_record_for_run(
17781 idx,
17782 source_set,
17783 &entry.name,
17784 &run_token,
17785 )
17786 .await
17787 {
17788 let mut first_err = entry_error.lock().await;
17789 if first_err.is_none() {
17790 *first_err = Some(err);
17791 callback_rx.cancel();
17792 }
17793 }
17794 return;
17795 }
17796
17797 let mut fivs = match load_decommission_entry_exact_versions(
17798 &source_set,
17799 &entry,
17800 &bucket_name,
17801 "check_after_decommission.file_info_versions",
17802 )
17803 .await
17804 {
17805 Ok(fivs) => fivs,
17806 Err(err) => {
17807 let mut first_err = entry_error.lock().await;
17808 if first_err.is_none() {
17809 *first_err = Some(err);
17810 callback_rx.cancel();
17811 }
17812 return;
17813 }
17814 };
17815
17816 fivs.versions
17817 .sort_by_key(|version| (version.mod_time.is_none(), std::cmp::Reverse(version.mod_time)));
17818
17819 let mut remaining = 0;
17820 let mut expired = 0;
17821 for version in fivs.versions.iter().chain(fivs.free_versions.iter()) {
17822 if version.tier_free_version() {
17823 remaining += 1;
17824 debug!(
17825 event = EVENT_DECOMMISSION_ENTRY,
17826 component = LOG_COMPONENT_ECSTORE,
17827 subsystem = LOG_SUBSYSTEM_POOLS,
17828 pool_index = idx,
17829 bucket = %bucket_name,
17830 object = %entry.name,
17831 version_id = ?version.version_id,
17832 reason = DECOMMISSION_FREE_VERSION_SWEEP_REASON,
17833 state = "free_version_retained",
17834 "Decommission final sweep retained a free version"
17835 );
17836 continue;
17837 }
17838 let skip_lifecycle = match should_skip_lifecycle_for_data_movement(
17839 Arc::clone(&store),
17840 &bucket_name,
17841 version,
17842 lifecycle_config.as_ref(),
17843 object_lock_config.as_ref(),
17844 false,
17845 &LcEventSrc::Decom,
17846 None,
17847 )
17848 .await
17849 {
17850 Ok(skip_lifecycle) => skip_lifecycle,
17851 Err(err) => {
17852 let mut first_err = entry_error.lock().await;
17853 if first_err.is_none() {
17854 *first_err = Some(err);
17855 callback_rx.cancel();
17856 }
17857 return;
17858 }
17859 };
17860 if skip_lifecycle {
17861 expired += 1;
17862 continue;
17863 }
17864 let remaining_versions = decommission_remaining_version_count(&fivs.versions, expired);
17865 if should_skip_decommission_delete_marker(version, remaining_versions, replication_configured) {
17866 continue;
17867 }
17868 remaining += 1;
17869 }
17870
17871 if remaining > 0 {
17872 let mut first_path = first_remaining_path.lock().await;
17873 if first_path.is_none() {
17874 *first_path = Some(format!("{bucket_name}/{}", entry.name));
17875 }
17876 } else {
17877 let identity = (set_index, bucket_name.clone(), entry.name.clone());
17878 if let Some(unresolved_entry) = unresolved_entries_by_identity.get(&identity) {
17879 let mut resolved = resolved_unresolved_entries.lock().await;
17880 if !resolved.contains(unresolved_entry) {
17881 resolved.push(unresolved_entry.clone());
17882 }
17883 }
17884 }
17885
17886 versions_found.fetch_add(remaining, Ordering::Relaxed);
17887 })
17888 });
17889
17890 let list_result = set
17891 .list_objects_to_decommission(
17892 self.clone(),
17893 callback_rx,
17894 bucket_info.clone(),
17895 callback,
17896 entry_error.clone(),
17897 idx,
17898 set_index,
17899 generation,
17900 require_all_disks,
17901 )
17902 .await;
17903 let entry_error = entry_error.lock().await.clone();
17904 resolve_decommission_check_after_list_result(list_result, entry_error)?;
17905
17906 let versions_found = versions_found.load(Ordering::Relaxed);
17907 if versions_found > 0 {
17908 let first_remaining_path = first_remaining_path
17909 .lock()
17910 .await
17911 .clone()
17912 .unwrap_or_else(|| format!("{}/<unknown>", bucket_info.name));
17913 return Err(Error::other(format!(
17914 "at least {versions_found} object(s)/version(s) were found in bucket `{}` after decommissioning; first remaining path `{first_remaining_path}`",
17915 bucket_info.name,
17916 )));
17917 }
17918 }
17919 }
17920
17921 let mut verified_unresolved_entries = resolved_unresolved_entries.lock().await.clone();
17922 for entry in &unresolved_entries {
17923 let set = pool.disk_set.get(entry.set_index).ok_or_else(|| {
17924 Error::other(format!(
17925 "failed to verify decommission for pool {idx}: unresolved listing entry references missing set {}",
17926 entry.set_index
17927 ))
17928 })?;
17929 if set.decommission_unresolved_entry_absent_on_all_disks(idx, entry).await?
17930 && !verified_unresolved_entries.contains(entry)
17931 {
17932 verified_unresolved_entries.push(entry.clone());
17933 }
17934 if !verified_unresolved_entries.contains(entry) {
17935 return Err(Error::other(format!(
17936 "failed to verify decommission for pool {idx}: unresolved listing entry {}/{} in set {} was neither re-observed as resolved nor absent on every source disk",
17937 entry.bucket, entry.object, entry.set_index
17938 )));
17939 }
17940 }
17941
17942 self.persist_decommission_durable_ilm_manifest_for_run(idx, &run_token)
17943 .await?;
17944 self.verify_decommission_durable_ilm_receipts_for_run(idx, &run_token).await?;
17945
17946 Ok(verified_unresolved_entries)
17947 }
17948
17949 async fn ensure_decommission_multipart_uploads_drained(
17950 &self,
17951 idx: usize,
17952 pool: &Sets,
17953 buckets: &[DecomBucketInfo],
17954 ) -> Result<()> {
17955 let mut bucket_names = buckets
17956 .iter()
17957 .filter(|bucket| bucket.name != RUSTFS_META_BUCKET)
17958 .map(|bucket| bucket.name.as_str())
17959 .collect::<Vec<_>>();
17960 bucket_names.sort_unstable();
17961 bucket_names.dedup();
17962
17963 for bucket in bucket_names {
17966 let lifecycle_guard = self.acquire_bucket_lifecycle_write_lock(bucket).await?;
17967 if lifecycle_guard.is_lock_lost() {
17968 return Err(Error::other(format!(
17969 "decommission multipart drain lost the bucket lifecycle fence for `{bucket}`"
17970 )));
17971 }
17972 for set in &pool.disk_set {
17973 if let Some(upload_path) = set.first_multipart_upload_path_for_decommission(bucket).await? {
17974 return Err(Error::other(format!(
17975 "pool {idx} still contains multipart upload `{upload_path}` for bucket `{bucket}`; resolve it before retrying decommission"
17976 )));
17977 }
17978 }
17979 }
17980
17981 Ok(())
17982 }
17983
17984 #[cfg(test)]
17985 pub(crate) async fn ensure_decommission_multipart_uploads_drained_for_test(self: &Arc<Self>, idx: usize) -> Result<()> {
17986 let buckets = self.get_buckets_to_decommission().await?;
17987 let pool = self.pools[idx].clone();
17988 self.ensure_decommission_multipart_uploads_drained(idx, pool.as_ref(), &buckets)
17989 .await
17990 }
17991 #[cfg(all(test, feature = "test-util"))]
17992 pub(crate) async fn check_after_decommission_for_test(self: &Arc<Self>, idx: usize) -> Result<()> {
17993 let generation = self.active_decommission_generation(idx).await?;
17994 self.check_after_decommission(idx, &CancellationToken::new(), generation)
17995 .await
17996 .map(|_| ())
17997 }
17998
17999 #[tracing::instrument(skip(self, rd))]
18000 async fn decommission_object(
18001 self: Arc<Self>,
18002 pool_idx: usize,
18003 bucket: String,
18004 rd: GetObjectReader,
18005 expected_bucket_incarnation_id: Option<uuid::Uuid>,
18006 capacity_owner: Option<DecommissionCapacityOwner>,
18007 ) -> Result<()> {
18008 warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name);
18009 let object_name = rd.object_info.name.clone();
18010 let mut migration = tokio::task::JoinSet::new();
18011 migration.spawn(data_movement::migrate_decommission_object(
18012 self,
18013 pool_idx,
18014 bucket.clone(),
18015 rd,
18016 expected_bucket_incarnation_id,
18017 "decommission_object",
18018 capacity_owner,
18019 ));
18020 let result = migration
18021 .join_next()
18022 .await
18023 .ok_or_else(|| Error::other("decommission migration task was not started"))?
18024 .map_err(|err| Error::other(format!("decommission migration task join error: {err}")))?;
18025 if result.is_ok() {
18026 warn!("decommission_object: migrated {} {}", &bucket, &object_name);
18027 }
18028 result
18029 }
18030}
18031
18032#[cfg(test)]
18033async fn persist_v3_pool_meta_for_test(store: &Arc<ECStore>) {
18034 let mut meta = PoolMeta::default();
18035 meta.load_no_lock_from_replicas(store.pools.clone())
18036 .await
18037 .expect("the baseline pool metadata should be readable before V3 migration");
18038 meta.version = POOL_META_GENERATION_VERSION;
18039 let mut write_state = store.pool_meta_save_gate.lock().await.clone();
18040 temp_env::async_with_vars(
18041 [
18042 (rustfs_config::ENV_POOL_META_V3_WRITE, Some("true")),
18043 (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("true")),
18044 ],
18045 async {
18046 meta.save_no_lock_observing(store.pools.clone(), &mut write_state)
18047 .await
18048 .expect("the baseline metadata should be upgraded to a durable V3 replica");
18049 },
18050 )
18051 .await;
18052
18053 let mut loaded = PoolMeta::default();
18054 loaded
18055 .load_no_lock_from_replicas(store.pools.clone())
18056 .await
18057 .expect("the upgraded V3 pool metadata should be readable");
18058 assert_eq!(loaded.version, POOL_META_GENERATION_VERSION);
18059 store.pool_meta.write().await.version = POOL_META_GENERATION_VERSION;
18060}
18061
18062#[cfg(test)]
18063#[allow(clippy::items_after_test_module)]
18064mod tests {
18065 use super::*;
18066 use crate::bucket::replication::{ReplicationState, ReplicationStatusType};
18067 use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
18068 use crate::storage_api_contracts::multipart::MultipartOperations as _;
18069 use serde::Serialize;
18070
18071 #[test]
18072 fn decommission_set_local_usage_cache_classification_is_exact() {
18073 for object in [
18074 "buckets/.usage-cache.bin",
18075 "buckets/.usage-cache.bin.bkp",
18076 "buckets/photos/.usage-cache.bin",
18077 "buckets/photos/.usage-cache.bin.bkp",
18078 ] {
18079 assert!(is_decommission_set_local_usage_cache(RUSTFS_META_BUCKET, object), "{object}");
18080 assert!(!is_decommission_set_local_usage_cache("user-bucket", object), "{object}");
18081 }
18082 for object in [
18083 "buckets/.usage.v2.json",
18084 "buckets/.usage.v2.json.bkp",
18085 "buckets/.usage-cache.bin.extra",
18086 "buckets/.usage-cache.bin.bkp.extra",
18087 "buckets/prefix.usage-cache.bin",
18088 "buckets/photos/.usage-cache.bin.bkp.bkp",
18089 "buckets/photos/nested/.usage-cache.bin",
18090 "buckets//.usage-cache.bin",
18091 "buckets/../.usage-cache.bin",
18092 "buckets/./.usage-cache.bin",
18093 "buckets/.usage-cache.bin/child",
18094 "config/.usage-cache.bin",
18095 "buckets-other/.usage-cache.bin",
18096 ".usage-cache.bin",
18097 ] {
18098 assert!(!is_decommission_set_local_usage_cache(RUSTFS_META_BUCKET, object), "{object}");
18099 }
18100 }
18101
18102 #[tokio::test]
18103 #[serial_test::serial]
18104 async fn decommission_keeps_set_local_usage_caches_out_of_target_capacity() {
18105 use crate::object_api::PutObjReader;
18106 use tokio::io::AsyncReadExt as _;
18107
18108 let (_temp_dirs, store, _other_store) =
18111 Box::pin(crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None)).await;
18112 let user_bucket = "decommission-usage-cache-control";
18113 Box::pin(store.make_bucket(user_bucket, &MakeBucketOptions::default()))
18114 .await
18115 .expect("create the ordinary-object control bucket");
18116 let incarnation = Box::pin(store.bucket_incarnation_id(user_bucket))
18117 .await
18118 .expect("control bucket incarnation");
18119 let source_time = OffsetDateTime::now_utc();
18120 let cache_objects = [
18121 "buckets/.usage-cache.bin",
18122 "buckets/.usage-cache.bin.bkp",
18123 "buckets/photos/.usage-cache.bin",
18124 "buckets/photos/.usage-cache.bin.bkp",
18125 ];
18126 let conflict_object = "buckets/.usage-cache.bin.conflict";
18127 let source_read_failure_object = "buckets/.usage-cache.bin.read-error";
18128 let source_body = b"source set cache";
18129 let target_body = b"independent older target set cache";
18130 for object in cache_objects.into_iter().chain([conflict_object]) {
18131 for (pool_index, body, mod_time) in [
18132 (0, source_body.as_slice(), source_time),
18133 (1, target_body.as_slice(), source_time - Duration::seconds(1)),
18134 ] {
18135 store.pools[pool_index]
18137 .get_disks_by_key(object)
18138 .put_object(
18139 RUSTFS_META_BUCKET,
18140 object,
18141 &mut PutObjReader::from_vec(body.to_vec()),
18142 &ObjectOptions {
18143 mod_time: Some(mod_time),
18144 ..Default::default()
18145 },
18146 )
18147 .await
18148 .expect("seed distinct native set-local objects");
18149 }
18150 }
18151 let controls = [
18152 (RUSTFS_META_BUCKET, "buckets/.usage.v2.json"),
18153 (RUSTFS_META_BUCKET, "buckets/photos/.usage-cache.bin.extra"),
18154 (user_bucket, "ordinary-object"),
18155 (user_bucket, "buckets/.usage-cache.bin"),
18156 ];
18157 for (bucket, object) in controls {
18158 store.pools[0]
18159 .put_object(
18160 bucket,
18161 object,
18162 &mut PutObjReader::from_vec(b"ordinary object contents".to_vec()),
18163 &ObjectOptions {
18164 expected_bucket_incarnation_id: (bucket == user_bucket).then_some(incarnation),
18165 ..Default::default()
18166 },
18167 )
18168 .await
18169 .expect("seed a control that must migrate");
18170 }
18171 store.pools[0]
18172 .put_object(
18173 RUSTFS_META_BUCKET,
18174 source_read_failure_object,
18175 &mut PutObjReader::from_vec(source_body.to_vec()),
18176 &ObjectOptions::default(),
18177 )
18178 .await
18179 .expect("seed a similarly named object whose source read will fail");
18180
18181 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
18182 set_decommission_capacity_info_overrides_for_test(
18183 store.id,
18184 vec![vec![
18185 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 16_384, 16_384),
18186 DecommissionPoolCapacityInfo::for_test(1, layout, 131_072, 131_072, 0),
18187 ]],
18188 );
18189 Box::pin(store.save_current_pool_meta_for_decommission_start(&[0], Vec::new()))
18190 .await
18191 .expect("activate the decommission capacity reservation");
18192
18193 for object in cache_objects {
18194 Box::pin(store.decommission_entry_for_test(
18195 0,
18196 MetaCacheEntry {
18197 name: object.to_string(),
18198 ..Default::default()
18199 },
18200 RUSTFS_META_BUCKET.to_string(),
18201 store.pools[0].get_disks_by_key(object),
18202 ))
18203 .await
18204 .expect("set-local cache must not enter cross-pool migration");
18205 for (pool_index, expected) in [(0, source_body.as_slice()), (1, target_body.as_slice())] {
18206 let mut reader = store.pools[pool_index]
18207 .get_disks_by_key(object)
18208 .get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
18209 .await
18210 .expect("each set must retain its own cache");
18211 let mut actual = Vec::new();
18212 reader.stream.read_to_end(&mut actual).await.expect("read retained cache");
18213 assert_eq!(actual, expected, "pool {pool_index}, {object}");
18214 }
18215 let meta = store.pool_meta.read().await;
18216 let info = meta.pools[0].decommission.as_ref().expect("decommission progress");
18217 assert_eq!((info.items_decommissioned, info.items_decommission_failed), (0, 0));
18218 assert_eq!((info.bytes_done, info.bytes_failed), (0, 0));
18219 let reservation = info.capacity_reservation.as_ref().expect("capacity reservation");
18220 assert_eq!(reservation.pending_target_physical_bytes, 0);
18221 assert_eq!(reservation.consumed_target_physical_bytes, 0);
18222 assert!(reservation.targets.iter().all(|target| target.pending_mutation_id.is_none()));
18223 }
18224 let mut persisted = PoolMeta::default();
18225 Box::pin(persisted.load_no_lock_from_replicas(store.pools.clone()))
18226 .await
18227 .expect("reload durable capacity intents after cache entries");
18228 let reservation = persisted.pools[0]
18229 .decommission
18230 .as_ref()
18231 .and_then(|info| info.capacity_reservation.as_ref())
18232 .expect("durable reservation");
18233 assert_eq!(reservation.pending_target_physical_bytes, 0);
18234 assert!(reservation.targets.iter().all(|target| target.pending_mutation_id.is_none()));
18235
18236 let injected_reads = Arc::new(AtomicUsize::new(0));
18237 let observed_reads = Arc::clone(&injected_reads);
18238 let read_fault = DecommissionTestFaultGuard::install(Arc::new(move |stage, bucket, object, _, success| {
18239 if stage == "object_read" && bucket == RUSTFS_META_BUCKET && object == source_read_failure_object && success {
18240 observed_reads.fetch_add(1, Ordering::SeqCst);
18241 return true;
18242 }
18243 false
18244 }));
18245 Box::pin(store.decommission_entry_for_test(
18246 0,
18247 MetaCacheEntry {
18248 name: source_read_failure_object.to_string(),
18249 ..Default::default()
18250 },
18251 RUSTFS_META_BUCKET.to_string(),
18252 store.pools[0].get_disks_by_key(source_read_failure_object),
18253 ))
18254 .await
18255 .expect("entry must record the non-NotFound source read failure");
18256 drop(read_fault);
18257 assert_eq!(injected_reads.load(Ordering::SeqCst), DECOMMISSION_VERSION_COPY_ATTEMPTS);
18258 {
18259 let meta = store.pool_meta.read().await;
18260 let info = meta.pools[0].decommission.as_ref().expect("source read failure progress");
18261 assert_eq!((info.items_decommissioned, info.items_decommission_failed), (0, 1));
18262 assert_eq!(info.bytes_failed, source_body.len());
18263 assert_eq!(
18264 info.capacity_reservation
18265 .as_ref()
18266 .expect("reservation")
18267 .pending_target_physical_bytes,
18268 0
18269 );
18270 }
18271 let mut retained = store.pools[0]
18272 .get_object_reader(
18273 RUSTFS_META_BUCKET,
18274 source_read_failure_object,
18275 None,
18276 HeaderMap::new(),
18277 &ObjectOptions::default(),
18278 )
18279 .await
18280 .expect("source read failure must retain the source");
18281 let mut retained_body = Vec::new();
18282 retained
18283 .stream
18284 .read_to_end(&mut retained_body)
18285 .await
18286 .expect("read retained source");
18287 assert_eq!(retained_body, source_body);
18288 drop(retained);
18289 let target_err = store.pools[1]
18290 .get_object_info(RUSTFS_META_BUCKET, source_read_failure_object, &ObjectOptions::default())
18291 .await
18292 .expect_err("failed source read must not create a target object");
18293 assert!(is_err_object_not_found(&target_err), "unexpected target state: {target_err:?}");
18294
18295 for (bucket, object) in controls {
18296 Box::pin(store.decommission_entry_for_test(
18297 0,
18298 MetaCacheEntry {
18299 name: object.to_string(),
18300 ..Default::default()
18301 },
18302 bucket.to_string(),
18303 store.pools[0].get_disks_by_key(object),
18304 ))
18305 .await
18306 .expect("ordinary and similarly named objects must migrate");
18307 let mut reader = store.pools[1]
18308 .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
18309 .await
18310 .expect("control must exist on the target");
18311 let mut actual = Vec::new();
18312 reader.stream.read_to_end(&mut actual).await.expect("read migrated control");
18313 assert_eq!(actual, b"ordinary object contents", "{bucket}/{object}");
18314 let err = store.pools[0]
18315 .get_object_info(bucket, object, &ObjectOptions::default())
18316 .await
18317 .expect_err("migrated control must be removed from the source");
18318 assert!(is_err_object_not_found(&err), "{bucket}/{object}: {err:?}");
18319 }
18320
18321 Box::pin(store.decommission_entry_for_test(
18322 0,
18323 MetaCacheEntry {
18324 name: conflict_object.to_string(),
18325 ..Default::default()
18326 },
18327 RUSTFS_META_BUCKET.to_string(),
18328 store.pools[0].get_disks_by_key(conflict_object),
18329 ))
18330 .await
18331 .expect("entry must record a real conditional-copy failure");
18332 let meta = store.pool_meta.read().await;
18333 let info = meta.pools[0].decommission.as_ref().expect("final progress");
18334 assert_eq!(info.items_decommissioned, controls.len());
18335 assert_eq!(
18336 info.items_decommission_failed, 2,
18337 "similar names must not hide read or migration failures"
18338 );
18339 assert_eq!(info.bytes_failed, source_body.len() * 2);
18340 drop(meta);
18341 for (pool_index, expected) in [(0, source_body.as_slice()), (1, target_body.as_slice())] {
18342 let mut reader = store.pools[pool_index]
18343 .get_object_reader(RUSTFS_META_BUCKET, conflict_object, None, HeaderMap::new(), &ObjectOptions::default())
18344 .await
18345 .expect("failed migration must preserve both objects");
18346 let mut actual = Vec::new();
18347 reader.stream.read_to_end(&mut actual).await.expect("read conflict object");
18348 assert_eq!(actual, expected);
18349 }
18350 }
18351
18352 #[tokio::test]
18353 #[serial_test::serial]
18354 async fn decommission_final_sweep_excludes_only_set_local_usage_caches() {
18355 use crate::object_api::PutObjReader;
18356
18357 let (_temp_dirs, store, _other_store) =
18358 crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
18359 for object in [
18360 "buckets/.usage-cache.bin",
18361 "buckets/.usage-cache.bin.bkp",
18362 "buckets/photos/.usage-cache.bin",
18363 "buckets/photos/.usage-cache.bin.bkp",
18364 ] {
18365 store.pools[0]
18366 .get_disks_by_key(object)
18367 .put_object(
18368 RUSTFS_META_BUCKET,
18369 object,
18370 &mut PutObjReader::from_vec(b"set-local cache".to_vec()),
18371 &ObjectOptions::default(),
18372 )
18373 .await
18374 .expect("seed each supported set-local cache path");
18375 }
18376 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
18377 set_decommission_capacity_info_overrides_for_test(
18378 store.id,
18379 vec![vec![
18380 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 16_384, 16_384),
18381 DecommissionPoolCapacityInfo::for_test(1, layout, 131_072, 131_072, 0),
18382 ]],
18383 );
18384 store
18385 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
18386 .await
18387 .expect("activate the final-sweep generation");
18388 let generation = store.active_decommission_generation(0).await.expect("active generation");
18389 store
18390 .check_after_decommission(0, &CancellationToken::new(), generation)
18391 .await
18392 .expect("the four set-local cache forms must not block the final sweep");
18393
18394 for object in [
18395 "buckets/.usage-cache.bin.extra",
18396 "buckets/photos/.usage-cache.bin.bkp.extra",
18397 "buckets/.usage.v2.json",
18398 ] {
18399 let source_set = store.pools[0].get_disks_by_key(object);
18400 source_set
18401 .put_object(
18402 RUSTFS_META_BUCKET,
18403 object,
18404 &mut PutObjReader::from_vec(b"unmigrated ordinary metadata".to_vec()),
18405 &ObjectOptions::default(),
18406 )
18407 .await
18408 .expect("seed ordinary metadata that must prevent completion");
18409 let err = store
18410 .check_after_decommission(0, &CancellationToken::new(), generation)
18411 .await
18412 .expect_err("a remaining similar name or global usage snapshot must block completion");
18413 assert!(err.to_string().contains("after decommissioning"), "unexpected final-sweep error: {err:?}");
18414 assert!(err.to_string().contains(object), "the final sweep must identify {object}: {err:?}");
18415 source_set
18416 .delete_object(RUSTFS_META_BUCKET, object, ObjectOptions::default())
18417 .await
18418 .expect("remove only the ordinary-metadata control before the next sweep");
18419 }
18420 store
18421 .check_after_decommission(0, &CancellationToken::new(), generation)
18422 .await
18423 .expect("only the four set-local caches remain after removing the controls");
18424 }
18425
18426 #[test]
18427 fn pool_activation_fleet_proof_error_classifier_matches_only_retryable_proof_failures() {
18428 assert!(is_pool_activation_fleet_proof_error(&Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED)));
18429 assert!(is_pool_activation_fleet_proof_error(&Error::other(POOL_ACTIVATION_FLEET_PROOF_EXPIRED)));
18430 let wrapped = format!("rebalance meta save failed during start_rebalance: {POOL_ACTIVATION_FLEET_PROOF_EXPIRED}");
18431 assert!(is_pool_activation_fleet_proof_error(&Error::other(wrapped)));
18432 assert!(!is_pool_activation_fleet_proof_error(&Error::ConfigNotFound));
18433 }
18434
18435 #[tokio::test]
18436 #[serial_test::serial]
18437 async fn decommission_v1_start_preflights_reject_before_metadata_writes() {
18438 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
18439 let mut v1_meta = store.pool_meta.read().await.clone();
18440 v1_meta.version = POOL_META_V1_VERSION;
18441 let v1_data = pool_meta_v1_replica_test_data(&v1_meta);
18442 for pool in &store.pools {
18443 save_config_with_opts(
18444 pool.clone(),
18445 POOL_META_NAME,
18446 v1_data.clone(),
18447 &ObjectOptions {
18448 max_parity: true,
18449 ..Default::default()
18450 },
18451 )
18452 .await
18453 .expect("the V1 baseline should be persisted to every pool");
18454 }
18455 let baseline = load_pool_meta_replicas(store.pools.clone(), true)
18456 .await
18457 .expect("baseline pool metadata should be readable");
18458 assert_eq!(baseline.meta.version, POOL_META_V1_VERSION);
18459 *store.pool_meta.write().await = baseline.meta.clone();
18460 let start_probe = PoolActivationStartProbe::install(PoolActivationStartKind::Decommission);
18461 let err = store
18462 .start_decommission(vec![0])
18463 .await
18464 .expect_err("the initial V1 start preflight must reject before side effects");
18465 assert!(matches!(err, Error::InvalidArgument(..)));
18466 assert!(
18467 !start_probe.preflight_side_effect_was_attempted(),
18468 "V1 rejection must precede bucket listing, healing, and metadata-bucket creation"
18469 );
18470 assert!(
18471 !start_probe.activation_was_attempted(),
18472 "V1 rejection must not enter the authoritative activation save"
18473 );
18474 let after_early_rejection = load_pool_meta_replicas(store.pools.clone(), true)
18475 .await
18476 .expect("early rejection must leave durable pool metadata readable");
18477 assert_eq!(after_early_rejection.canonical, baseline.canonical);
18478 assert!(
18479 store
18480 .pool_meta
18481 .read()
18482 .await
18483 .pools
18484 .iter()
18485 .all(|pool| pool.decommission.is_none())
18486 );
18487 assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
18488 store
18489 .ensure_pool_meta_side_effects_safe("V1 start preflight")
18490 .await
18491 .expect("a deterministic start rejection must not latch recovery");
18492 drop(start_probe);
18493
18494 let err = store
18495 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
18496 .await
18497 .expect_err("the authoritative V1 start preflight must reject before saving");
18498 assert!(matches!(err, Error::InvalidArgument(..)));
18499
18500 let after_authoritative_rejection = load_pool_meta_replicas(store.pools.clone(), true)
18501 .await
18502 .expect("authoritative rejection must leave durable pool metadata readable");
18503 assert_eq!(after_authoritative_rejection.canonical, baseline.canonical);
18504 assert!(
18505 after_authoritative_rejection
18506 .meta
18507 .pools
18508 .iter()
18509 .all(|pool| pool.decommission.is_none())
18510 );
18511 assert!(
18512 store
18513 .pool_meta
18514 .read()
18515 .await
18516 .pools
18517 .iter()
18518 .all(|pool| pool.decommission.is_none())
18519 );
18520 assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
18521 store
18522 .ensure_pool_meta_side_effects_safe("authoritative V1 start preflight")
18523 .await
18524 .expect("an authoritative capability rejection must not latch recovery");
18525 }
18526
18527 #[tokio::test]
18528 #[serial_test::serial]
18529 async fn decommission_activation_fence_loss_after_durable_save_blocks_publication() {
18530 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
18531 crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
18532 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
18533 set_decommission_capacity_info_overrides_for_test(
18534 store.id,
18535 vec![vec![
18536 DecommissionPoolCapacityInfo::for_test(0, layout, 60, 60, 0),
18537 DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100),
18538 ]],
18539 );
18540 let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
18541 let start_store = Arc::clone(&store);
18542 let mut start_task = tokio::spawn(async move {
18543 start_store
18544 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
18545 .await
18546 });
18547
18548 tokio::select! {
18549 result = &mut start_task => panic!("activation finished before the durable-save barrier: {result:?}"),
18550 () = barrier.wait_until_paused() => {}
18551 }
18552 barrier.release_after_fence_loss();
18553 let err = tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
18554 .await
18555 .expect("decommission activation should stop after losing its fence")
18556 .expect("decommission activation task should not panic")
18557 .expect_err("post-durable-save fence loss must reject in-memory publication");
18558 assert!(err.to_string().contains("activation lock lost"));
18559
18560 let local = store.pool_meta.read().await;
18561 assert!(
18562 !pool_meta_has_active_decommission(&local),
18563 "the activation must not publish after its durable fence is lost"
18564 );
18565 drop(local);
18566 store
18567 .ensure_pool_meta_side_effects_safe("post-durable-save activation fence loss")
18568 .await
18569 .expect_err("the undisarmed transaction must latch the sticky pool metadata gate");
18570
18571 let mut persisted = PoolMeta::default();
18572 persisted
18573 .load_no_lock_from_replicas(vec![store.pools[0].clone()])
18574 .await
18575 .expect("the durable replica should remain readable for recovery");
18576 assert!(
18577 pool_meta_has_active_decommission(&persisted),
18578 "the test must lose the fence only after one durable replica commit"
18579 );
18580 }
18581
18582 #[tokio::test]
18583 #[serial_test::serial]
18584 async fn decommission_activation_adopts_canonical_commit_after_replica_failure() {
18585 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
18586 crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
18587 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
18588 set_decommission_capacity_info_overrides_for_test(
18589 store.id,
18590 vec![vec![
18591 DecommissionPoolCapacityInfo::for_test(0, layout, 60, 60, 0),
18592 DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100),
18593 ]],
18594 );
18595 let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
18596 let start_store = Arc::clone(&store);
18597 let mut start_task = tokio::spawn(async move {
18598 start_store
18599 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
18600 .await
18601 });
18602
18603 tokio::select! {
18604 result = &mut start_task => panic!("activation finished before the durable-save barrier: {result:?}"),
18605 () = barrier.wait_until_paused() => {}
18606 }
18607 let mut replica_disks = Vec::new();
18608 for set in &store.pools[1].disk_set {
18609 let mut disks = set.disks.write().await;
18610 let saved = std::mem::take(&mut *disks);
18611 *disks = vec![None; saved.len()];
18612 replica_disks.push(saved);
18613 }
18614 barrier.release_without_fence_loss();
18615
18616 tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
18617 .await
18618 .expect("decommission activation should finish after its canonical commit")
18619 .expect("decommission activation task should not panic")
18620 .expect("a replica save failure must not report the committed activation as failed");
18621
18622 for (set, disks) in store.pools[1].disk_set.iter().zip(replica_disks) {
18623 *set.disks.write().await = disks;
18624 }
18625
18626 let local = store.pool_meta.read().await;
18627 assert!(pool_meta_has_active_decommission(&local));
18628 drop(local);
18629
18630 let mut canonical = PoolMeta::default();
18631 canonical
18632 .load_no_lock_from_replicas(vec![store.pools[0].clone()])
18633 .await
18634 .expect("the canonical committed decommission metadata should remain readable");
18635 assert!(pool_meta_has_active_decommission(&canonical));
18636
18637 let mut replica = PoolMeta::default();
18638 replica
18639 .load_no_lock_from_replicas(vec![store.pools[1].clone()])
18640 .await
18641 .expect("the stale replica metadata should remain readable after disks recover");
18642 assert!(!pool_meta_has_active_decommission(&replica));
18643
18644 let mut reloaded = PoolMeta::default();
18645 let replica_state = reloaded
18646 .load_no_lock_from_replicas(store.pools.clone())
18647 .await
18648 .expect("the canonical commit should remain reloadable after a replica recovers stale");
18649 assert!(pool_meta_has_active_decommission(&reloaded));
18650 assert!(replica_state.needs_repair);
18651 assert!(replica_state.repair_write_safe);
18652
18653 let worker_cancel = CancellationToken::new();
18654 let index_cancelers = store
18655 .reserve_decommission_routines(&worker_cancel, &[0])
18656 .await
18657 .expect("the committed activation should admit its decommission worker");
18658 drop(spawn_decommission_index_cancelers(
18659 Arc::clone(&store),
18660 worker_cancel.clone(),
18661 index_cancelers,
18662 Arc::new(Semaphore::new(decommission_entry_concurrency_limit())),
18663 ));
18664 let admitted_cancel = store.decommission_cancelers.read().await[0]
18665 .clone()
18666 .expect("the admitted decommission worker should have a cancellation token");
18667 assert!(!admitted_cancel.is_cancelled());
18668 worker_cancel.cancel();
18669 assert!(admitted_cancel.is_cancelled());
18670 }
18671
18672 #[tokio::test]
18673 #[serial_test::serial(pool_meta_version_env)]
18674 async fn decommission_capacity_reservation_serializes_two_nodes_without_overselling() {
18675 let (_temp_dirs, first_node, second_node) =
18676 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
18677 persist_v3_pool_meta_for_test(&first_node).await;
18678 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
18679 let capacity_snapshot = vec![
18680 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
18681 DecommissionPoolCapacityInfo::for_test(1, layout, 0, 30, 30),
18682 DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0),
18683 ];
18684 set_decommission_capacity_info_overrides_for_test(first_node.id, vec![capacity_snapshot.clone()]);
18685 set_decommission_capacity_info_overrides_for_test(second_node.id, vec![capacity_snapshot]);
18686
18687 let barrier =
18688 PutObjectCommitBarrier::install(RUSTFS_META_BUCKET, POOL_META_NAME, PutObjectCommitPause::BeforeQuotaRename);
18689 let first_store = Arc::clone(&first_node);
18690 let first = tokio::spawn(async move {
18691 first_store
18692 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
18693 .await
18694 });
18695 barrier.wait_until_paused().await;
18696
18697 let second_probe = PoolActivationStartProbe::install(PoolActivationStartKind::Decommission);
18698 let second_store = Arc::clone(&second_node);
18699 let second = tokio::spawn(async move {
18700 second_store
18701 .save_current_pool_meta_for_decommission_start(&[1], Vec::new())
18702 .await
18703 });
18704 second_probe.wait_until_attempted().await;
18705 assert!(
18706 !second.is_finished(),
18707 "the competing node must wait behind the distributed activation fence"
18708 );
18709
18710 drop(barrier);
18711 first
18712 .await
18713 .expect("first-node activation should not panic")
18714 .expect("first-node reservation should fit");
18715 let mut started_v3 = PoolMeta::default();
18716 started_v3
18717 .load_no_lock_from_replicas(first_node.pools.clone())
18718 .await
18719 .expect("the started capacity reservation should reload from V3 replicas");
18720 assert_eq!(started_v3.version, POOL_META_GENERATION_VERSION);
18721 let err = second
18722 .await
18723 .expect("second-node activation should not panic")
18724 .expect_err("the second reservation must observe and reject the committed first reservation");
18725 assert!(err.to_string().contains("requires 60 bytes, but 40 bytes are available"));
18726
18727 let mut persisted = PoolMeta::default();
18728 persisted
18729 .load_no_lock_from_replicas(first_node.pools.clone())
18730 .await
18731 .expect("the winning capacity reservation should remain readable");
18732 let reservation = persisted.pools[0]
18733 .decommission
18734 .as_ref()
18735 .and_then(|info| info.capacity_reservation.as_ref())
18736 .expect("the winning operation must retain its durable reservation");
18737 assert_eq!(
18738 reservation.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
18739 "an all-v4 proof must persist the per-target lock model through V3 replicas"
18740 );
18741 assert_eq!(reservation.peak_physical_bytes, 60);
18742 assert!(persisted.pools[1].decommission.is_none());
18743 }
18744
18745 #[tokio::test]
18746 #[serial_test::serial]
18747 async fn decommission_start_without_dedicated_v4_proof_persists_the_legacy_lock_model() {
18748 let (_temp_dirs, store, _other_store) =
18749 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
18750 crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
18751 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
18752 set_decommission_capacity_info_overrides_for_test(
18753 store.id,
18754 vec![vec![
18755 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
18756 DecommissionPoolCapacityInfo::for_test(1, layout, 0, 10, 10),
18757 DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
18758 ]],
18759 );
18760 let _proof_guard = crate::services::notification_sys::without_decommission_target_fence_fleet_proof_for_test();
18761
18762 store
18763 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
18764 .await
18765 .expect("a mixed-version fleet should retain the compatible global lock model");
18766 let reservation = store.pool_meta.read().await.pools[0]
18767 .decommission
18768 .as_ref()
18769 .and_then(|info| info.capacity_reservation.as_ref())
18770 .expect("proofless start should persist a capacity reservation")
18771 .clone();
18772 assert_eq!(reservation.model_version, DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION);
18773 }
18774
18775 #[tokio::test]
18776 #[serial_test::serial]
18777 async fn decommission_cancel_replans_target_fences_after_a_concurrent_v2_start() {
18778 let (_temp_dirs, store, _other_store) =
18779 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
18780 crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
18781 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
18782 set_decommission_capacity_info_overrides_for_test(
18783 store.id,
18784 vec![vec![
18785 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
18786 DecommissionPoolCapacityInfo::for_test(1, layout, 0, 10, 10),
18787 DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
18788 ]],
18789 );
18790 let barrier = super::DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
18791 barrier.pause_cancel_before_start();
18792 let cancel_store = Arc::clone(&store);
18793 let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
18794 barrier.wait_until_cancel_before_start().await;
18795
18796 store
18797 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
18798 .await
18799 .expect("the concurrent start should persist a v2 reservation");
18800 let target_pool_index = store.pool_meta.read().await.pools[0]
18801 .decommission
18802 .as_ref()
18803 .and_then(|info| info.capacity_reservation.as_ref())
18804 .map(|reservation| {
18805 assert_eq!(reservation.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION);
18806 reservation.targets[0].pool_index
18807 })
18808 .expect("the concurrent start should publish its target cohort");
18809 let target_lock = store.pools[0]
18810 .new_ns_lock(
18811 RUSTFS_META_BUCKET,
18812 &format!("{}/{target_pool_index}", super::DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX),
18813 )
18814 .await
18815 .expect("create the cancel/start target-fence probe");
18816 let target_guard = target_lock
18817 .get_write_lock(std::time::Duration::from_secs(30))
18818 .await
18819 .expect("hold the newly started target fence");
18820
18821 barrier.release_cancel_before_start();
18822 tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel)
18823 .await
18824 .expect_err("cancel must wait for the target fence selected by the concurrent start");
18825 assert!(
18826 store.pool_meta.read().await.pools[0]
18827 .decommission
18828 .as_ref()
18829 .and_then(|info| info.capacity_reservation.as_ref())
18830 .is_some_and(DecommissionCapacityReservation::active),
18831 "cancel must not release the new reservation before fencing its target"
18832 );
18833
18834 drop(target_guard);
18835 tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
18836 .await
18837 .expect("cancel should finish after the target fence is released")
18838 .expect("cancel task should not panic")
18839 .expect("cancel should publish the terminal state");
18840 drop(barrier);
18841
18842 let pool_meta = store.pool_meta.read().await;
18843 let info = pool_meta.pools[0]
18844 .decommission
18845 .as_ref()
18846 .expect("canceled decommission metadata should remain present");
18847 assert!(info.canceled);
18848 let reservation = info
18849 .capacity_reservation
18850 .as_ref()
18851 .expect("canceled capacity accounting should remain inspectable");
18852 assert!(!reservation.active());
18853 assert_eq!(reservation.pending_target_physical_bytes, 0);
18854 assert_eq!(reservation.inflight_target_physical_bytes, 0);
18855 }
18856
18857 async fn start_target_fenced_cancel_test(
18858 store: &Arc<ECStore>,
18859 first_target_free: usize,
18860 ) -> (DecommissionCapacityTerminalFencePlan, DecommissionCanceler) {
18861 crate::services::rebalance::promote_test_pool_meta_to_v2(store).await;
18862 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
18863 set_decommission_capacity_info_overrides_for_test(
18864 store.id,
18865 vec![vec![
18866 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
18867 DecommissionPoolCapacityInfo::for_test(1, layout, first_target_free, 10, 10 - first_target_free),
18868 DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
18869 ]],
18870 );
18871 store
18872 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
18873 .await
18874 .expect("persist an active target-fenced decommission");
18875 let plan = decommission_capacity_terminal_fence_plan(&*store.pool_meta.read().await, 0)
18876 .expect("active decommission should have a valid fence plan")
18877 .expect("active decommission should retain its reservation");
18878 assert_eq!(plan.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION);
18879 let canceler = DecommissionCanceler::new(CancellationToken::new());
18880 *store.decommission_cancelers.write().await = vec![Some(canceler.clone()), None, None];
18881 (plan, canceler)
18882 }
18883
18884 #[tokio::test]
18885 #[serial_test::serial]
18886 async fn decommission_cancel_waits_for_target_contention_past_one_lock_timeout() {
18887 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("5"))], async {
18888 let (_temp_dirs, store, _other_store) =
18889 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
18890 let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
18891 assert_eq!(plan.target_pool_indices, vec![2]);
18892 let target_lock = store.pools[0]
18893 .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
18894 .await
18895 .expect("create the active migration target gate");
18896 let target_guard = target_lock
18897 .get_write_lock(std::time::Duration::from_secs(30))
18898 .await
18899 .expect("hold the target gate across the first cancel acquisition timeout");
18900 let movement_gate = store.ctx.data_movement_operation_gate();
18901 let movement_guard = movement_gate.read().await;
18902 let cancel_store = Arc::clone(&store);
18903 let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
18904
18905 tokio::time::timeout(get_lock_acquire_timeout() + std::time::Duration::from_secs(1), &mut cancel)
18906 .await
18907 .expect_err("one target-lock timeout must not end a legitimate cancel while its retry budget remains");
18908 assert!(!canceler.is_cancelled(), "cancel must not signal its worker before durable fencing");
18909 let mut durable = PoolMeta::default();
18910 durable
18911 .load_no_lock_from_replicas(store.pools.clone())
18912 .await
18913 .expect("the active reservation must remain readable while cancel waits");
18914 assert_eq!(
18915 decommission_capacity_terminal_fence_plan(&durable, 0).expect("read the durable active plan"),
18916 Some(plan),
18917 "a failed target acquisition must not release or replace the durable reservation"
18918 );
18919 assert!(!durable.pools[0].decommission.as_ref().expect("active decommission").canceled);
18920
18921 drop(target_guard);
18922 tokio::time::timeout(std::time::Duration::from_secs(30), canceler.token().cancelled())
18923 .await
18924 .expect("cancel should persist and signal its worker after target contention is released");
18925 durable
18926 .load_no_lock_from_replicas(store.pools.clone())
18927 .await
18928 .expect("reload the committed cancellation from native replicas");
18929 let info = durable.pools[0]
18930 .decommission
18931 .as_ref()
18932 .expect("canceled state must remain durable");
18933 assert!(info.canceled);
18934 assert!(!info.complete && !info.failed);
18935 let reservation = info
18936 .capacity_reservation
18937 .as_ref()
18938 .expect("terminal capacity accounting must remain inspectable");
18939 assert!(!reservation.active());
18940 assert_eq!(reservation.pending_target_physical_bytes, 0);
18941 assert_eq!(reservation.inflight_target_physical_bytes, 0);
18942 tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel)
18943 .await
18944 .expect_err("the runtime-fenced cancel must wait for in-flight movement after signaling");
18945 drop(movement_guard);
18946 tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
18947 .await
18948 .expect("cancel should return after in-flight movement quiesces")
18949 .expect("cancel task should not panic")
18950 .expect("the same cancel request should finish its durable terminal transition");
18951 })
18952 .await;
18953 }
18954
18955 #[tokio::test]
18956 #[serial_test::serial]
18957 async fn decommission_cancel_target_contention_exhausts_bounded_attempts_without_committing() {
18958 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
18959 let (_temp_dirs, store, _other_store) =
18960 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
18961 let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
18962 let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
18963 let target_lock = store.pools[0]
18964 .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
18965 .await
18966 .expect("create the persistently contended target gate");
18967 let target_guard = target_lock
18968 .get_write_lock(std::time::Duration::from_secs(30))
18969 .await
18970 .expect("hold the target gate for every cancel attempt");
18971
18972 let err = tokio::time::timeout(std::time::Duration::from_secs(30), store.decommission_cancel(0))
18973 .await
18974 .expect("target contention must not retry indefinitely")
18975 .expect_err("exhausting the acquisition budget must not report cancellation success");
18976 match err {
18977 Error::Lock(rustfs_lock::LockError::Timeout { resource, timeout }) => {
18978 assert_eq!(resource, ".rustfs.sys/decommission/capacity-target/2@latest");
18979 assert_eq!(timeout, std::time::Duration::from_secs(1));
18980 }
18981 other => panic!("expected the final typed target-lock timeout, got {other:?}"),
18982 }
18983 assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 3);
18984 assert!(!canceler.is_cancelled());
18985 assert!(canceler.is_active());
18986 let mut durable = PoolMeta::default();
18987 durable
18988 .load_no_lock_from_replicas(store.pools.clone())
18989 .await
18990 .expect("reload the reservation after the canceled request exhausted its budget");
18991 assert_eq!(
18992 decommission_capacity_terminal_fence_plan(&durable, 0).expect("the active plan should remain valid"),
18993 Some(plan)
18994 );
18995 assert!(!durable.pools[0].decommission.as_ref().expect("active decommission").canceled);
18996 drop(target_guard);
18997 })
18998 .await;
18999 }
19000
19001 #[tokio::test]
19002 #[serial_test::serial]
19003 async fn decommission_cancel_target_retry_releases_partial_cohort_and_rejects_remote_replacement() {
19004 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
19005 let (_temp_dirs, store, other_store) =
19006 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19007 let (plan, _old_canceler) = start_target_fenced_cancel_test(&store, 8).await;
19008 assert_eq!(plan.target_pool_indices, vec![1, 2]);
19009 let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19010 barrier.pause_cancel_target_timeout();
19011 let target_lock = store.pools[0]
19012 .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
19013 .await
19014 .expect("create the second target gate");
19015 let target_guard = target_lock
19016 .get_write_lock(std::time::Duration::from_secs(30))
19017 .await
19018 .expect("block cancellation after it acquires the first target");
19019 let cancel_store = Arc::clone(&store);
19020 let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
19021 barrier.wait_until_cancel_target_timeout().await;
19022
19023 let first_target_lock = store.pools[0]
19024 .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/1"))
19025 .await
19026 .expect("create the partial-cohort release probe");
19027 let first_target_guard = first_target_lock
19028 .get_write_lock(std::time::Duration::from_secs(1))
19029 .await
19030 .expect("cancel must release its first target before retrying the blocked second target");
19031 drop(first_target_guard);
19032 drop(target_guard);
19033
19034 other_store
19035 .reload_pool_meta()
19036 .await
19037 .expect("load the active generation on the remote node");
19038 other_store
19039 .decommission_cancel(0)
19040 .await
19041 .expect("the remote node should cancel the old generation");
19042 other_store
19043 .clear_decommission(0)
19044 .await
19045 .expect("the remote node should clear the old generation");
19046 let (replacement_plan, replacement_canceler) = start_target_fenced_cancel_test(&other_store, 8).await;
19047 assert_ne!(replacement_plan.operation_id, plan.operation_id);
19048
19049 barrier.release_cancel_target_timeout();
19050 let err = tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
19051 .await
19052 .expect("the stale request must finish without retrying a replacement operation")
19053 .expect("the stale cancel task should not panic")
19054 .expect_err("the original request must not cancel a remotely replaced generation");
19055 assert!(err.to_string().contains("changed while retrying terminal fences"));
19056 assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
19057 assert!(!replacement_canceler.is_cancelled());
19058 let mut durable = PoolMeta::default();
19059 durable
19060 .load_no_lock_from_replicas(store.pools.clone())
19061 .await
19062 .expect("reload the replacement after the stale cancel returns");
19063 assert_eq!(
19064 decommission_capacity_terminal_fence_plan(&durable, 0).expect("the replacement must retain its valid plan"),
19065 Some(replacement_plan)
19066 );
19067 assert!(
19068 !durable.pools[0]
19069 .decommission
19070 .as_ref()
19071 .expect("replacement decommission")
19072 .canceled
19073 );
19074 })
19075 .await;
19076 }
19077
19078 #[tokio::test]
19079 #[serial_test::serial]
19080 async fn decommission_cancel_target_retry_survives_caller_abort_and_settles_inflight_mutation() {
19081 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
19082 let (_temp_dirs, store, _other_store) =
19083 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19084 let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
19085 let owner = DecommissionCapacityOwner {
19086 source_pool_index: 0,
19087 operation_id: plan.operation_id,
19088 generation: plan.generation,
19089 owner_nonce: plan.owner_nonce,
19090 mutation_id: Some(uuid::Uuid::new_v4()),
19091 };
19092 let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
19093 let (release_tx, release_rx) = tokio::sync::oneshot::channel();
19094 let mutation_store = Arc::clone(&store);
19095 let mutation = tokio::spawn(async move {
19096 mutation_store
19097 .run_decommission_capacity_admitted_mutation(2, Some(owner), Some(1), || async {
19098 entered_tx
19099 .send(())
19100 .expect("the test should observe the admitted target mutation");
19101 release_rx.await.expect("the test should release the in-flight mutation");
19102 Ok(())
19103 })
19104 .await
19105 });
19106 tokio::time::timeout(std::time::Duration::from_secs(30), entered_rx)
19107 .await
19108 .expect("target mutation should reach its controlled I/O phase")
19109 .expect("target admission should succeed before cancel starts");
19110 let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19111 let cancel_store = Arc::clone(&store);
19112 let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
19113 barrier.wait_until_cancel_target_timeout().await;
19114 assert!(!canceler.is_cancelled(), "in-flight capacity settlement must precede the cancel signal");
19115 cancel.abort();
19116 assert!(cancel.await.expect_err("the RPC waiter should be aborted").is_cancelled());
19117
19118 release_tx
19119 .send(())
19120 .expect("the mutation must still be alive after the caller disconnects");
19121 tokio::time::timeout(std::time::Duration::from_secs(30), mutation)
19122 .await
19123 .expect("the in-flight mutation must be able to settle without a metadata lock cycle")
19124 .expect("the mutation task should not panic")
19125 .expect("the admitted mutation must settle before the target gate is released");
19126 tokio::time::timeout(std::time::Duration::from_secs(30), async {
19127 loop {
19128 if store.pool_meta.read().await.pools[0]
19129 .decommission
19130 .as_ref()
19131 .is_some_and(|info| info.canceled)
19132 {
19133 break;
19134 }
19135 tokio::time::sleep(std::time::Duration::from_millis(20)).await;
19136 }
19137 })
19138 .await
19139 .expect("the detached cancel transaction must finish after target contention clears");
19140 assert!(canceler.is_cancelled());
19141 let mut durable = PoolMeta::default();
19142 durable
19143 .load_no_lock_from_replicas(store.pools.clone())
19144 .await
19145 .expect("reload cancellation after the RPC waiter was dropped");
19146 let info = durable.pools[0].decommission.as_ref().expect("durable canceled decommission");
19147 assert!(info.canceled && !info.failed && !info.complete);
19148 let reservation = info
19149 .capacity_reservation
19150 .as_ref()
19151 .expect("inspect settled capacity accounting");
19152 assert!(!reservation.active());
19153 assert_eq!(reservation.pending_target_physical_bytes, 0);
19154 assert_eq!(reservation.inflight_target_physical_bytes, 0);
19155 })
19156 .await;
19157 }
19158
19159 #[tokio::test]
19160 #[serial_test::serial]
19161 async fn decommission_cancel_target_retry_does_not_replay_a_failed_durable_save() {
19162 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
19163 let (_temp_dirs, store, _other_store) =
19164 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19165 let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
19166 let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19167 barrier.pause_cancel_target_timeout();
19168 let target_lock = store.pools[0]
19169 .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
19170 .await
19171 .expect("create the target gate preceding the failed save");
19172 let target_guard = target_lock
19173 .get_write_lock(std::time::Duration::from_secs(30))
19174 .await
19175 .expect("force one target acquisition retry before persistence");
19176 let save_calls = Arc::new(AtomicUsize::new(0));
19177 let calls = Arc::clone(&save_calls);
19178 let cancel_store = Arc::clone(&store);
19179 let owner = canceler.clone();
19180 let cancel = tokio::spawn(async move {
19181 cancel_store
19182 .decommission_cancel_transaction(0, Some(owner), true, move |_, _| async move {
19183 calls.fetch_add(1, Ordering::AcqRel);
19184 Err(Error::Timeout)
19185 })
19186 .await
19187 });
19188 barrier.wait_until_cancel_target_timeout().await;
19189 assert_eq!(save_calls.load(Ordering::Acquire), 0, "persistence must wait for every target fence");
19190 drop(target_guard);
19191 barrier.release_cancel_target_timeout();
19192 let err = tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
19193 .await
19194 .expect("a persistence failure must end the cancellation transaction")
19195 .expect("the failed-save task should not panic")
19196 .expect_err("the injected durable-save failure must reach the caller");
19197 assert!(matches!(err, Error::Timeout));
19198 assert_eq!(save_calls.load(Ordering::Acquire), 1);
19199 assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
19200 assert!(!canceler.is_cancelled());
19201 assert!(canceler.is_active());
19202 store
19203 .ensure_pool_meta_side_effects_safe("verify ambiguous cancellation save blocks further writes")
19204 .await
19205 .expect_err("a failed durable save must retain the existing recovery gate");
19206 let mut durable = PoolMeta::default();
19207 durable
19208 .load_no_lock_from_replicas(store.pools.clone())
19209 .await
19210 .expect("the original reservation should remain readable after the injected save failure");
19211 assert_eq!(
19212 decommission_capacity_terminal_fence_plan(&durable, 0).expect("the old reservation should remain valid"),
19213 Some(plan)
19214 );
19215 })
19216 .await;
19217 }
19218
19219 #[tokio::test]
19220 #[serial_test::serial]
19221 async fn decommission_cancel_target_retry_converges_after_successive_contenders() {
19222 temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
19223 let (_temp_dirs, store, _other_store) =
19224 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19225 let (_plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
19226 let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19227 barrier.pause_cancel_target_timeout();
19228 let target_lock = store.pools[0]
19229 .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
19230 .await
19231 .expect("create the repeatedly contended target gate");
19232 let first_guard = target_lock
19233 .get_write_lock(std::time::Duration::from_secs(30))
19234 .await
19235 .expect("the first contender should own the target gate");
19236 let cancel_store = Arc::clone(&store);
19237 let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
19238 barrier.wait_until_cancel_target_timeout().await;
19239 assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
19240 drop(first_guard);
19241 let second_guard = target_lock
19242 .get_write_lock(std::time::Duration::from_secs(30))
19243 .await
19244 .expect("a second contender should be able to acquire between cancel attempts");
19245 barrier.release_cancel_target_timeout();
19246 barrier.pause_cancel_target_timeout();
19247 barrier.wait_until_cancel_target_timeout().await;
19248 assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 2);
19249 assert!(!canceler.is_cancelled());
19250 drop(second_guard);
19251 barrier.release_cancel_target_timeout();
19252
19253 tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
19254 .await
19255 .expect("cancel should converge after the repeated contention ends")
19256 .expect("the cancel task should not panic")
19257 .expect("the third target acquisition should permit a durable cancellation");
19258 assert!(canceler.is_cancelled());
19259 let mut durable = PoolMeta::default();
19260 durable
19261 .load_no_lock_from_replicas(store.pools.clone())
19262 .await
19263 .expect("the successful retried cancellation must survive a native metadata reload");
19264 assert!(
19265 durable.pools[0]
19266 .decommission
19267 .as_ref()
19268 .expect("canceled decommission")
19269 .canceled
19270 );
19271 assert!(
19272 decommission_capacity_terminal_fence_plan(&durable, 0)
19273 .expect("valid terminal metadata")
19274 .is_none()
19275 );
19276 })
19277 .await;
19278 }
19279
19280 #[tokio::test]
19281 #[serial_test::serial]
19282 async fn stale_node_cancel_cannot_replace_a_new_durable_v2_operation() {
19283 let (_temp_dirs, first_node, stale_node) =
19284 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19285 crate::services::rebalance::promote_test_pool_meta_to_v2(&first_node).await;
19286 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19287 set_decommission_capacity_info_overrides_for_test(
19288 first_node.id,
19289 vec![vec![
19290 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
19291 DecommissionPoolCapacityInfo::for_test(1, layout, 0, 10, 10),
19292 DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
19293 ]],
19294 );
19295
19296 first_node
19297 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19298 .await
19299 .expect("start the operation retained by the stale node");
19300 let stale_operation = first_node.pool_meta.read().await.clone();
19301 let stale_plan = decommission_capacity_terminal_fence_plan(&stale_operation, 0)
19302 .expect("the stale operation should have a valid terminal fence plan")
19303 .expect("the stale operation should have an active reservation");
19304
19305 first_node
19306 .decommission_cancel(0)
19307 .await
19308 .expect("cancel the first durable operation");
19309 first_node
19310 .clear_decommission(0)
19311 .await
19312 .expect("clear the first terminal operation");
19313 first_node
19314 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19315 .await
19316 .expect("start the replacement durable operation");
19317 let replacement_plan = {
19318 let pool_meta = first_node.pool_meta.read().await;
19319 decommission_capacity_terminal_fence_plan(&pool_meta, 0)
19320 .expect("the replacement operation should have a valid terminal fence plan")
19321 .expect("the replacement operation should have an active reservation")
19322 };
19323 assert_ne!(replacement_plan.operation_id, stale_plan.operation_id);
19324
19325 *stale_node.pool_meta.write().await = stale_operation;
19326 let err = stale_node
19327 .decommission_cancel(0)
19328 .await
19329 .expect_err("a stale local operation must not be merged over the durable replacement");
19330 assert!(err.to_string().contains("differs from the durable terminal fence plan"));
19331
19332 let mut durable = PoolMeta::default();
19333 durable
19334 .load_no_lock_from_replicas(first_node.pools.clone())
19335 .await
19336 .expect("the replacement operation should remain readable from durable replicas");
19337 assert_eq!(
19338 decommission_capacity_terminal_fence_plan(&durable, 0)
19339 .expect("the durable replacement should retain a valid terminal fence plan")
19340 .expect("the durable replacement reservation should remain active"),
19341 replacement_plan
19342 );
19343 let replacement = durable.pools[0]
19344 .decommission
19345 .as_ref()
19346 .expect("the durable replacement metadata should remain present");
19347 assert!(!replacement.canceled);
19348 assert!(
19349 replacement
19350 .capacity_reservation
19351 .as_ref()
19352 .is_some_and(DecommissionCapacityReservation::active)
19353 );
19354 }
19355
19356 #[tokio::test]
19357 #[serial_test::serial]
19358 async fn decommission_capacity_publication_allows_immediate_bucket_completion() {
19359 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
19360 crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
19361 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19362 let capacity = vec![
19363 DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80),
19364 DecommissionPoolCapacityInfo::for_test(1, layout, 200, 200, 0),
19365 ];
19366 set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity.clone(), capacity.clone(), capacity]);
19367 let bucket = DecomBucketInfo {
19368 name: "capacity-publication-bucket".to_string(),
19369 prefix: String::new(),
19370 };
19371 store
19372 .save_current_pool_meta_for_decommission_start(&[0], vec![bucket.clone()])
19373 .await
19374 .expect("activate a reservation and queue the regression bucket");
19375 let (owner, initial_update, generation) = {
19376 let mut meta = store.pool_meta.write().await;
19377 meta.count_item(0, 17, false);
19378 let reservation = meta.pools[0]
19379 .decommission
19380 .as_ref()
19381 .and_then(|info| info.capacity_reservation.as_ref())
19382 .expect("the source must own an active reservation");
19383 (
19384 DecommissionCapacityOwner {
19385 source_pool_index: 0,
19386 operation_id: reservation.operation_id,
19387 generation: reservation.generation,
19388 owner_nonce: reservation.owner_nonce,
19389 mutation_id: Some(uuid::Uuid::new_v4()),
19390 },
19391 meta.pools[0].last_update,
19392 meta.pools[0]
19393 .decommission
19394 .as_ref()
19395 .and_then(|info| info.start_time)
19396 .expect("active worker generation"),
19397 )
19398 };
19399 store
19400 .run_decommission_capacity_admitted_mutation(1, Some(owner), Some(17), || async {
19401 let meta = store.pool_meta.read().await;
19402 assert!(
19403 meta.pools[0].last_update > initial_update,
19404 "durable capacity intent must publish its revision before target I/O"
19405 );
19406 assert_eq!(
19407 meta.pools[0]
19408 .decommission
19409 .as_ref()
19410 .expect("active source")
19411 .items_decommissioned,
19412 1
19413 );
19414 Ok(())
19415 })
19416 .await
19417 .expect("capacity intent and consumption must commit without a progress checkpoint");
19418 let persisted = load_pool_meta_replicas(store.pools.clone(), true)
19419 .await
19420 .expect("reload capacity consumption before bucket completion");
19421 {
19422 let meta = store.pool_meta.read().await;
19423 assert_eq!(meta.pools[0].last_update, persisted.meta.pools[0].last_update);
19424 let info = meta.pools[0].decommission.as_ref().expect("active source");
19425 assert_eq!(info.items_decommissioned, 1, "capacity publication must retain uncheckpointed progress");
19426 assert_eq!(
19427 info.progress_save_item_baseline, 0,
19428 "capacity-only saves do not checkpoint object counters"
19429 );
19430 }
19431 assert!(
19432 store
19433 .mark_decommission_bucket_done_and_save(0, generation, &bucket)
19434 .await
19435 .expect("bucket completion immediately after capacity persistence must not reject its own revision")
19436 );
19437 let persisted = load_pool_meta_replicas(store.pools.clone(), true)
19438 .await
19439 .expect("bucket completion must survive a metadata reload");
19440 let info = persisted.meta.pools[0]
19441 .decommission
19442 .as_ref()
19443 .expect("durable source progress");
19444 assert!(info.queued_buckets.is_empty());
19445 assert_eq!(info.decommissioned_buckets, vec![bucket.to_string()]);
19446 assert_eq!((info.items_decommissioned, info.bytes_done), (1, 17));
19447 let reservation = info.capacity_reservation.as_ref().expect("durable capacity consumption");
19448 assert_eq!(reservation.pending_target_physical_bytes, 0);
19449 assert_eq!(reservation.consumed_target_physical_bytes, 17);
19450 }
19451
19452 #[tokio::test]
19453 #[serial_test::serial]
19454 async fn decommission_bucket_completion_publishes_only_committed_progress() {
19455 for fault_stage in [
19456 "bucket_completion_before_save",
19457 "bucket_completion_after_save",
19458 "count_during_save",
19459 ] {
19460 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
19461 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19462 set_decommission_capacity_info_overrides_for_test(
19463 store.id,
19464 vec![vec![
19465 DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80),
19466 DecommissionPoolCapacityInfo::for_test(1, layout, 200, 200, 0),
19467 ]],
19468 );
19469 let bucket = DecomBucketInfo {
19470 name: format!("bucket-completion-{fault_stage}"),
19471 prefix: String::new(),
19472 };
19473 store
19474 .save_current_pool_meta_for_decommission_start(&[0], vec![bucket.clone()])
19475 .await
19476 .expect("activate the bucket completion fixture");
19477 let generation = {
19478 let mut meta = store.pool_meta.write().await;
19479 meta.count_item(0, 17, false);
19480 meta.pools[0]
19481 .decommission
19482 .as_ref()
19483 .and_then(|info| info.start_time)
19484 .expect("active generation")
19485 };
19486 let hook_store = Arc::clone(&store);
19487 let observed = Arc::new(AtomicBool::new(false));
19488 let hook_observed = Arc::clone(&observed);
19489 let expected_bucket = bucket.name.clone();
19490 let fault = DecommissionTestFaultGuard::install(Arc::new(move |stage, name, _, _, success| {
19491 if name != expected_bucket || !success {
19492 return false;
19493 }
19494 if stage == fault_stage {
19495 hook_observed.store(true, Ordering::SeqCst);
19496 return true;
19497 }
19498 if fault_stage == "count_during_save" && stage == "bucket_completion_after_save" {
19499 hook_observed.store(true, Ordering::SeqCst);
19500 hook_store
19501 .pool_meta
19502 .try_write()
19503 .expect("publication must not hold the local metadata lock over I/O")
19504 .count_item(0, 23, false);
19505 }
19506 false
19507 }));
19508 let result = store.mark_decommission_bucket_done_and_save(0, generation, &bucket).await;
19509 assert!(
19510 observed.load(Ordering::SeqCst),
19511 "{fault_stage}: the production save boundary was not reached"
19512 );
19513 drop(fault);
19514 let persisted = load_pool_meta_replicas(store.pools.clone(), true)
19515 .await
19516 .expect("durable metadata remains readable");
19517 let durable = persisted.meta.pools[0].decommission.as_ref().expect("durable progress");
19518 let local = store.pool_meta.read().await;
19519 let live = local.pools[0].decommission.as_ref().expect("live progress");
19520 if fault_stage == "count_during_save" {
19521 assert!(result.expect("successful bucket completion"));
19522 assert_eq!((live.items_decommissioned, live.bytes_done, live.progress_save_item_baseline), (2, 40, 1));
19523 assert_eq!((durable.items_decommissioned, durable.bytes_done), (1, 17));
19524 assert!(live.queued_buckets.is_empty());
19525 assert_eq!(live.decommissioned_buckets, vec![bucket.to_string()]);
19526 } else {
19527 let err = result.expect_err("injected failure must reach the caller");
19528 assert!(err.to_string().contains(fault_stage));
19529 assert_eq!(live.queued_buckets, vec![bucket.to_string()]);
19530 assert!(live.decommissioned_buckets.is_empty());
19531 assert_eq!(live.progress_save_item_baseline, 0);
19532 if fault_stage == "bucket_completion_before_save" {
19533 assert_eq!(durable.queued_buckets, vec![bucket.to_string()]);
19534 assert!(durable.decommissioned_buckets.is_empty());
19535 } else {
19536 assert!(durable.queued_buckets.is_empty());
19537 assert_eq!(durable.decommissioned_buckets, vec![bucket.to_string()]);
19538 assert_eq!((durable.items_decommissioned, durable.bytes_done), (1, 17));
19539 }
19540 }
19541 drop(local);
19542 if fault_stage == "bucket_completion_after_save" {
19543 store
19544 .ensure_pool_meta_side_effects_safe("ambiguous bucket completion")
19545 .await
19546 .expect_err("commit without publication must require recovery before another write");
19547 } else {
19548 store
19549 .ensure_pool_meta_side_effects_safe("bucket completion")
19550 .await
19551 .expect("no ambiguous commit");
19552 }
19553 }
19554 }
19555
19556 #[tokio::test]
19557 #[serial_test::serial]
19558 async fn legacy_capacity_model_holds_the_global_pool_meta_fence_through_target_io() {
19559 let (_temp_dirs, store, _other_store) =
19560 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19561 crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
19562 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19563 set_decommission_capacity_info_overrides_for_test(
19564 store.id,
19565 vec![vec![
19566 DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80),
19567 DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
19568 DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0),
19569 ]],
19570 );
19571 let _proof_guard = crate::services::notification_sys::without_decommission_target_fence_fleet_proof_for_test();
19572 store
19573 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19574 .await
19575 .expect("activate a legacy reservation spanning two targets");
19576 let base_owner = {
19577 let pool_meta = store.pool_meta.read().await;
19578 let reservation = pool_meta.pools[0]
19579 .decommission
19580 .as_ref()
19581 .and_then(|info| info.capacity_reservation.as_ref())
19582 .expect("legacy capacity reservation should exist");
19583 assert_eq!(reservation.model_version, DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION);
19584 assert_eq!(reservation.targets.iter().map(|target| target.pool_index).collect::<Vec<_>>(), vec![1, 2]);
19585 DecommissionCapacityOwner {
19586 source_pool_index: 0,
19587 operation_id: reservation.operation_id,
19588 generation: reservation.generation,
19589 owner_nonce: reservation.owner_nonce,
19590 mutation_id: None,
19591 }
19592 };
19593
19594 let (first_entered_tx, first_entered_rx) = tokio::sync::oneshot::channel();
19595 let (first_release_tx, first_release_rx) = tokio::sync::oneshot::channel();
19596 let first_store = Arc::clone(&store);
19597 let first_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
19598 let first = tokio::spawn(async move {
19599 first_store
19600 .run_decommission_capacity_admitted_mutation_with_capacity_lease(1, Some(first_owner), Some(1), |_| async {
19601 first_entered_tx.send(()).expect("first legacy mutation should be observed");
19602 first_release_rx.await.expect("first legacy mutation should be released");
19603 Ok(())
19604 })
19605 .await
19606 });
19607 tokio::time::timeout(std::time::Duration::from_secs(30), first_entered_rx)
19608 .await
19609 .expect("first legacy mutation should enter")
19610 .expect("first legacy mutation should report entry");
19611 assert!(
19612 store.pool_meta_save_gate.try_lock().is_err(),
19613 "legacy target I/O must retain the local pool metadata save gate"
19614 );
19615 let pool_meta_lock = store.pools[0]
19616 .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
19617 .await
19618 .expect("create the legacy pool metadata lock probe");
19619 assert!(
19620 pool_meta_lock
19621 .get_write_lock_quiet(std::time::Duration::from_millis(300))
19622 .await
19623 .is_err(),
19624 "legacy target I/O must retain the distributed pool metadata write fence"
19625 );
19626
19627 let (second_entered_tx, mut second_entered_rx) = tokio::sync::oneshot::channel();
19628 let (second_release_tx, second_release_rx) = tokio::sync::oneshot::channel();
19629 let second_store = Arc::clone(&store);
19630 let second_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
19631 let second = tokio::spawn(async move {
19632 second_store
19633 .run_decommission_capacity_admitted_mutation_with_capacity_lease(2, Some(second_owner), Some(1), |_| async {
19634 second_entered_tx.send(()).expect("second legacy mutation should be observed");
19635 second_release_rx.await.expect("second legacy mutation should be released");
19636 Ok(())
19637 })
19638 .await
19639 });
19640 tokio::time::timeout(std::time::Duration::from_millis(500), &mut second_entered_rx)
19641 .await
19642 .expect_err("a different target must still wait behind the legacy global fence");
19643
19644 first_release_tx.send(()).expect("release first legacy mutation");
19645 first
19646 .await
19647 .expect("first legacy mutation task should not panic")
19648 .expect("first legacy mutation should finalize");
19649 tokio::time::timeout(std::time::Duration::from_secs(30), &mut second_entered_rx)
19650 .await
19651 .expect("second legacy mutation should enter after the first finalizes")
19652 .expect("second legacy mutation should report entry");
19653 assert!(
19654 pool_meta_lock
19655 .get_write_lock_quiet(std::time::Duration::from_millis(300))
19656 .await
19657 .is_err(),
19658 "the second legacy target I/O must also retain the global fence"
19659 );
19660 second_release_tx.send(()).expect("release second legacy mutation");
19661 second
19662 .await
19663 .expect("second legacy mutation task should not panic")
19664 .expect("second legacy mutation should finalize");
19665
19666 let pool_meta_guard = pool_meta_lock
19667 .get_write_lock(std::time::Duration::from_secs(30))
19668 .await
19669 .expect("the global pool metadata fence should release after both target tails");
19670 drop(pool_meta_guard);
19671 let pool_meta = store.pool_meta.read().await;
19672 let reservation = pool_meta.pools[0]
19673 .decommission
19674 .as_ref()
19675 .and_then(|info| info.capacity_reservation.as_ref())
19676 .expect("legacy reservation should remain active");
19677 assert_eq!(reservation.pending_target_physical_bytes, 0);
19678 assert_eq!(reservation.consumed_target_physical_bytes, 2);
19679 }
19680
19681 #[tokio::test]
19682 #[serial_test::serial]
19683 async fn decommission_capacity_mutations_overlap_across_targets_but_serialize_per_target() {
19684 let (_temp_dirs, store, _other_store) =
19685 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19686 crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
19687 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19688 let capacity_snapshot = vec![
19689 DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80),
19690 DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
19691 DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0),
19692 ];
19693 set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot]);
19694 store
19695 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19696 .await
19697 .expect("activate a reservation spanning two targets");
19698
19699 let base_owner = {
19700 let pool_meta = store.pool_meta.read().await;
19701 let reservation = pool_meta.pools[0]
19702 .decommission
19703 .as_ref()
19704 .and_then(|info| info.capacity_reservation.as_ref())
19705 .expect("active capacity reservation should exist");
19706 assert_eq!(
19707 reservation.targets.iter().map(|target| target.pool_index).collect::<Vec<_>>(),
19708 vec![1, 2],
19709 "the fixture must reserve both target pools"
19710 );
19711 DecommissionCapacityOwner {
19712 source_pool_index: 0,
19713 operation_id: reservation.operation_id,
19714 generation: reservation.generation,
19715 owner_nonce: reservation.owner_nonce,
19716 mutation_id: None,
19717 }
19718 };
19719
19720 let (first_entered_tx, first_entered_rx) = tokio::sync::oneshot::channel();
19721 let (first_release_tx, first_release_rx) = tokio::sync::oneshot::channel();
19722 let first_store = Arc::clone(&store);
19723 let first_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
19724 let first = tokio::spawn(async move {
19725 first_store
19726 .run_decommission_capacity_admitted_mutation_with_capacity_lease(1, Some(first_owner), Some(1), |_| async {
19727 first_entered_tx.send(()).expect("first target mutation should be observed");
19728 first_release_rx.await.expect("first target mutation should be released");
19729 Ok(())
19730 })
19731 .await
19732 });
19733 tokio::time::timeout(std::time::Duration::from_secs(30), first_entered_rx)
19734 .await
19735 .expect("first target mutation should enter without hanging")
19736 .expect("first target mutation should report entry");
19737
19738 let (second_entered_tx, second_entered_rx) = tokio::sync::oneshot::channel();
19739 let (second_release_tx, second_release_rx) = tokio::sync::oneshot::channel();
19740 let second_store = Arc::clone(&store);
19741 let second_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
19742 let second = tokio::spawn(async move {
19743 second_store
19744 .run_decommission_capacity_admitted_mutation_with_capacity_lease(2, Some(second_owner), Some(1), |_| async {
19745 second_entered_tx.send(()).expect("second target mutation should be observed");
19746 second_release_rx.await.expect("second target mutation should be released");
19747 Ok(())
19748 })
19749 .await
19750 });
19751 tokio::time::timeout(std::time::Duration::from_secs(30), second_entered_rx)
19752 .await
19753 .expect("a different target mutation should overlap instead of waiting for the first")
19754 .expect("second target mutation should report entry");
19755
19756 let save_guard = store
19757 .pool_meta_save_gate
19758 .try_lock()
19759 .expect("target I/O must not retain the local pool metadata save gate");
19760 drop(save_guard);
19761 let pool_meta_lock = store.pools[0]
19762 .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
19763 .await
19764 .expect("create a pool metadata lock probe");
19765 let pool_meta_guard = pool_meta_lock
19766 .get_write_lock_quiet(std::time::Duration::from_secs(1))
19767 .await
19768 .expect("target I/O must not retain the distributed pool metadata write lock");
19769 drop(pool_meta_guard);
19770
19771 let blocked_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
19772 let blocked = store
19773 .run_decommission_capacity_admitted_mutation(1, Some(blocked_owner), Some(1), || async { Ok(()) })
19774 .await
19775 .expect_err("a second mutation on the same target must wait behind its target gate");
19776 assert!(is_decommission_capacity_blocked_error(&blocked));
19777 assert!(
19778 is_decommission_capacity_target_gate_busy(&blocked),
19779 "same-target serialization must remain distinguishable from durable intent conflicts"
19780 );
19781 assert!(
19782 decommission_capacity_retry_kind(&blocked, 0).is_none(),
19783 "target contention must be retried inside the current version instead of replaying the entry"
19784 );
19785
19786 let cancel_store = Arc::clone(&store);
19787 let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
19788 tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel)
19789 .await
19790 .expect_err("terminal reservation release must wait for prepared target mutations");
19791 assert!(
19792 store.pool_meta.read().await.pools[0]
19793 .decommission
19794 .as_ref()
19795 .and_then(|info| info.capacity_reservation.as_ref())
19796 .is_some_and(DecommissionCapacityReservation::active),
19797 "cancellation must not release the reservation while target mutations are active"
19798 );
19799
19800 first_release_tx.send(()).expect("release first target mutation");
19801 second_release_tx.send(()).expect("release second target mutation");
19802 first
19803 .await
19804 .expect("first target mutation task should not panic")
19805 .expect("first target mutation should finalize");
19806 second
19807 .await
19808 .expect("second target mutation task should not panic")
19809 .expect("second target mutation should finalize");
19810 cancel
19811 .await
19812 .expect("decommission cancellation task should not panic")
19813 .expect("decommission cancellation should finish after target mutations");
19814
19815 let pool_meta = store.pool_meta.read().await;
19816 let reservation = pool_meta.pools[0]
19817 .decommission
19818 .as_ref()
19819 .and_then(|info| info.capacity_reservation.as_ref())
19820 .expect("released capacity reservation should remain available for accounting");
19821 assert!(!reservation.active());
19822 assert_eq!(reservation.release_reason.as_deref(), Some(DECOMMISSION_CAPACITY_RELEASE_CANCELED));
19823 assert_eq!(reservation.pending_target_physical_bytes, 0);
19824 assert_eq!(reservation.consumed_target_physical_bytes, 2);
19825 assert_eq!(
19826 reservation
19827 .targets
19828 .iter()
19829 .map(|target| (target.pool_index, target.consumed_physical_bytes))
19830 .collect::<Vec<_>>(),
19831 vec![(1, 1), (2, 1)]
19832 );
19833 }
19834
19835 #[tokio::test]
19836 #[serial_test::serial]
19837 async fn ordinary_write_capacity_fence_serializes_with_decommission_activation() {
19838 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
19839 let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
19840 let (release_tx, release_rx) = tokio::sync::oneshot::channel();
19841 let write_store = Arc::clone(&store);
19842 let ordinary_write = tokio::spawn(async move {
19843 write_store
19844 .run_decommission_capacity_admitted_mutation(1, None, None, || async move {
19845 entered_tx.send(()).expect("ordinary write admission should be observed");
19846 release_rx.await.expect("ordinary write should be released");
19847 Ok(())
19848 })
19849 .await
19850 });
19851 entered_rx
19852 .await
19853 .expect("ordinary write should hold the shared capacity fence");
19854
19855 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19856 set_decommission_capacity_info_overrides_for_test(
19857 store.id,
19858 vec![vec![
19859 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
19860 DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1),
19861 ]],
19862 );
19863 let activation_store = Arc::clone(&store);
19864 let activation = tokio::spawn(async move {
19865 activation_store
19866 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19867 .await
19868 });
19869 tokio::task::yield_now().await;
19870 assert!(
19871 !activation.is_finished(),
19872 "activation must wait until the already-admitted ordinary write leaves the distributed capacity boundary"
19873 );
19874
19875 release_tx.send(()).expect("ordinary write should be released");
19876 ordinary_write
19877 .await
19878 .expect("ordinary write task should join")
19879 .expect("the pre-activation ordinary write should complete");
19880 let err = activation
19881 .await
19882 .expect("activation task should join")
19883 .expect_err("activation must recheck and reject capacity consumed by the ordinary write");
19884 assert!(err.to_string().contains("requires 60 bytes, but 59 bytes are available"));
19885 }
19886
19887 async fn single_pool_capacity_admission_test_store() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
19888 let (temp_dirs, store) =
19889 crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta::default()).await;
19890 crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
19891 (temp_dirs, store)
19892 }
19893
19894 #[tokio::test]
19895 #[serial_test::serial]
19896 async fn single_pool_public_writes_skip_decommission_capacity_admission() {
19897 let (_temp_dirs, store) = single_pool_capacity_admission_test_store().await;
19898 let bucket = format!("single-pool-capacity-skip-{}", uuid::Uuid::new_v4());
19899 store
19900 .make_bucket(&bucket, &MakeBucketOptions::default())
19901 .await
19902 .expect("create single-pool bucket before blocking pool metadata writes");
19903 let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
19904 store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
19905 let write_gate = store.pool_meta_write_gate_status().await;
19906 assert!(!write_gate.writes_ready);
19907 assert!(write_gate.write_blocked);
19908 assert!(!write_gate.transaction_aborted);
19909
19910 let object = "ordinary-put.bin";
19911 let mut put_data = crate::object_api::PutObjReader::from_vec(b"ordinary single-pool body".to_vec());
19912 store
19913 .put_object(&bucket, object, &mut put_data, &ObjectOptions::default())
19914 .await
19915 .expect("single-pool ordinary PUT must not enter decommission capacity admission");
19916 store
19917 .get_object_info(&bucket, object, &ObjectOptions::default())
19918 .await
19919 .expect("single-pool ordinary PUT must remain readable");
19920
19921 let multipart_object = "ordinary-multipart.bin";
19922 let upload = store
19923 .new_multipart_upload(
19924 &bucket,
19925 multipart_object,
19926 &ObjectOptions {
19927 expected_bucket_incarnation_id: Some(incarnation),
19928 ..Default::default()
19929 },
19930 )
19931 .await
19932 .expect("single-pool MPU creation must not enter decommission capacity admission");
19933 let mut part_data = crate::object_api::PutObjReader::from_vec(b"single-pool multipart body".to_vec());
19934 let part = store
19935 .put_object_part(
19936 &bucket,
19937 multipart_object,
19938 &upload.upload_id,
19939 1,
19940 &mut part_data,
19941 &ObjectOptions {
19942 expected_bucket_incarnation_id: Some(incarnation),
19943 ..Default::default()
19944 },
19945 )
19946 .await
19947 .expect("single-pool UploadPart must not enter decommission capacity admission");
19948 store
19949 .clone()
19950 .complete_multipart_upload(
19951 &bucket,
19952 multipart_object,
19953 &upload.upload_id,
19954 vec![crate::storage_api_contracts::multipart::CompletePart {
19955 part_num: part.part_num,
19956 etag: part.etag,
19957 ..Default::default()
19958 }],
19959 &ObjectOptions {
19960 expected_bucket_incarnation_id: Some(incarnation),
19961 ..Default::default()
19962 },
19963 )
19964 .await
19965 .expect("single-pool CompleteMultipartUpload must not enter decommission capacity admission");
19966 store
19967 .get_object_info(&bucket, multipart_object, &ObjectOptions::default())
19968 .await
19969 .expect("single-pool completed MPU must remain readable");
19970 }
19971
19972 #[tokio::test]
19973 #[serial_test::serial]
19974 async fn multipart_mutations_locate_later_upload_before_reserved_pool_admission() {
19975 let (_temp_dirs, store, _other_store) =
19976 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19977 let bucket = format!("multipart-capacity-routing-{}", uuid::Uuid::new_v4());
19978 let object = "later-pool.bin";
19979 store
19980 .make_bucket(&bucket, &MakeBucketOptions::default())
19981 .await
19982 .expect("create multipart routing bucket");
19983 let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
19984 let bucket_guard = store
19985 .acquire_bucket_lifecycle_read_lock(&bucket)
19986 .await
19987 .expect("acquire multipart routing bucket lifecycle guard");
19988 let mut upload_opts = ObjectOptions {
19989 expected_bucket_incarnation_id: Some(incarnation),
19990 ..Default::default()
19991 };
19992 upload_opts.add_bucket_lifecycle_lock_guard(&bucket_guard);
19993 let upload = store.pools[1]
19994 .new_multipart_upload(&bucket, object, &upload_opts)
19995 .await
19996 .expect("seed an upload in the later pool");
19997
19998 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19999 set_decommission_capacity_info_overrides_for_test(
20000 store.id,
20001 vec![vec![
20002 DecommissionPoolCapacityInfo::for_test(0, layout, 20, 20, 0),
20003 DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
20004 DecommissionPoolCapacityInfo::for_test(2, layout, 0, 10, 10),
20005 ]],
20006 );
20007 store
20008 .save_current_pool_meta_for_decommission_start(&[2], Vec::new())
20009 .await
20010 .expect("reserve the first target pool");
20011 {
20012 let pool_meta = store.pool_meta.read().await;
20013 let targets = &pool_meta.pools[2]
20014 .decommission
20015 .as_ref()
20016 .and_then(|info| info.capacity_reservation.as_ref())
20017 .expect("source reservation should exist")
20018 .targets;
20019 assert_eq!(targets.len(), 1);
20020 assert_eq!(targets[0].pool_index, 0, "the first probed pool must be reserved");
20021 }
20022
20023 let mut data = crate::object_api::PutObjReader::from_vec(b"multipart body".to_vec());
20024 let part = store
20025 .put_object_part(&bucket, object, &upload.upload_id, 1, &mut data, &ObjectOptions::default())
20026 .await
20027 .expect("put-part must locate the later upload before capacity admission");
20028 store
20029 .clone()
20030 .complete_multipart_upload(
20031 &bucket,
20032 object,
20033 &upload.upload_id,
20034 vec![crate::storage_api_contracts::multipart::CompletePart {
20035 part_num: part.part_num,
20036 etag: part.etag,
20037 ..Default::default()
20038 }],
20039 &ObjectOptions::default(),
20040 )
20041 .await
20042 .expect("complete must locate the later upload before capacity admission");
20043
20044 store.pools[1]
20045 .get_object_info(&bucket, object, &ObjectOptions::default())
20046 .await
20047 .expect("the later-pool multipart upload should commit in place");
20048 let first_pool_result = store.pools[0]
20049 .get_object_info(&bucket, object, &ObjectOptions::default())
20050 .await;
20051 assert!(
20052 first_pool_result
20053 .as_ref()
20054 .err()
20055 .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)),
20056 "the reserved first pool must not receive the multipart mutation"
20057 );
20058 }
20059
20060 #[tokio::test]
20061 #[serial_test::serial]
20062 async fn expired_capacity_owner_nonce_rejects_stale_put_and_multipart_without_consuming() {
20063 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
20064 let bucket = format!("stale-capacity-owner-{}", uuid::Uuid::new_v4());
20065 let multipart_object = "stale-multipart.bin";
20066 store
20067 .make_bucket(&bucket, &MakeBucketOptions::default())
20068 .await
20069 .expect("create stale owner bucket");
20070 let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
20071 let bucket_guard = store
20072 .acquire_bucket_lifecycle_read_lock(&bucket)
20073 .await
20074 .expect("acquire stale owner bucket lifecycle guard");
20075 let mut upload_opts = ObjectOptions {
20076 expected_bucket_incarnation_id: Some(incarnation),
20077 ..Default::default()
20078 };
20079 upload_opts.add_bucket_lifecycle_lock_guard(&bucket_guard);
20080 let upload = store.pools[1]
20081 .new_multipart_upload(&bucket, multipart_object, &upload_opts)
20082 .await
20083 .expect("seed multipart upload before reserving the target");
20084
20085 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20086 set_decommission_capacity_info_overrides_for_test(
20087 store.id,
20088 vec![vec![
20089 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
20090 DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
20091 ]],
20092 );
20093 store
20094 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20095 .await
20096 .expect("activate target reservation");
20097
20098 let stale_owner = {
20099 let pool_meta = store.pool_meta.read().await;
20100 let reservation = pool_meta.pools[0]
20101 .decommission
20102 .as_ref()
20103 .and_then(|info| info.capacity_reservation.as_ref())
20104 .expect("active reservation should exist");
20105 DecommissionCapacityOwner {
20106 source_pool_index: 0,
20107 operation_id: reservation.operation_id,
20108 generation: reservation.generation,
20109 owner_nonce: reservation.owner_nonce,
20110 mutation_id: None,
20111 }
20112 };
20113 {
20114 let mut pool_meta = store.pool_meta.write().await;
20115 let reservation = pool_meta.pools[0]
20116 .decommission
20117 .as_mut()
20118 .and_then(|info| info.capacity_reservation.as_mut())
20119 .expect("active reservation should remain mutable");
20120 reservation.expires_at = OffsetDateTime::now_utc() - Duration::seconds(1);
20121 }
20122 store
20123 .save_current_pool_meta(&[0])
20124 .await
20125 .expect("persist the expired lease before recovery");
20126 store
20127 .pause_decommission_for_capacity(0, &decommission_capacity_blocked_error("test lease recovery"))
20128 .await
20129 .expect("pause recovery should renew the lease with a new owner nonce");
20130
20131 let (current_owner, before_progress) = {
20132 let pool_meta = store.pool_meta.read().await;
20133 let reservation = pool_meta.pools[0]
20134 .decommission
20135 .as_ref()
20136 .and_then(|info| info.capacity_reservation.as_ref())
20137 .expect("renewed reservation should exist");
20138 (
20139 DecommissionCapacityOwner {
20140 source_pool_index: 0,
20141 operation_id: reservation.operation_id,
20142 generation: reservation.generation,
20143 owner_nonce: reservation.owner_nonce,
20144 mutation_id: None,
20145 },
20146 (
20147 reservation.consumed_target_physical_bytes,
20148 reservation.inflight_target_physical_bytes,
20149 reservation.pending_target_physical_bytes,
20150 reservation.observed_target_physical_bytes,
20151 ),
20152 )
20153 };
20154 assert_ne!(stale_owner.owner_nonce, current_owner.owner_nonce);
20155 {
20156 let pool_meta = store.pool_meta.read().await;
20157 ensure_decommission_target_owner_admission(&pool_meta, current_owner, 1, 1, OffsetDateTime::now_utc())
20158 .expect("the renewed owner should retain its reservation");
20159 assert!(
20160 ensure_decommission_target_owner_admission(&pool_meta, stale_owner, 1, 1, OffsetDateTime::now_utc(),).is_err(),
20161 "the expired owner token must be rejected after nonce rotation"
20162 );
20163 }
20164
20165 let stale_put_object = "stale-put.bin";
20166 let mut stale_put_opts = ObjectOptions {
20167 data_movement: true,
20168 version_id: Some(uuid::Uuid::new_v4().to_string()),
20169 ..ObjectOptions::with_capacity_expected_data_bytes(Some(9))
20170 };
20171 stale_owner.apply_to(&mut stale_put_opts);
20172 let mut stale_put_data = crate::object_api::PutObjReader::from_vec(b"stale put".to_vec());
20173 let put_err = store
20174 .put_object_for_data_movement(&bucket, stale_put_object, &mut stale_put_data, &stale_put_opts, None)
20175 .await
20176 .expect_err("stale owner PUT must fail before selecting a target");
20177 assert!(is_decommission_capacity_blocked_error(&put_err));
20178
20179 let mut stale_part_opts = ObjectOptions {
20180 data_movement: true,
20181 part_number: Some(1),
20182 expected_bucket_incarnation_id: Some(incarnation),
20183 ..Default::default()
20184 };
20185 stale_owner.apply_to(&mut stale_part_opts);
20186 let mut stale_part_data = crate::object_api::PutObjReader::from_vec(b"stale part".to_vec());
20187 let part_err = store
20188 .put_object_part_for_data_movement(
20189 1,
20190 &bucket,
20191 multipart_object,
20192 &upload.upload_id,
20193 &mut stale_part_data,
20194 &stale_part_opts,
20195 )
20196 .await
20197 .expect_err("stale owner multipart PUT must fail before mutation");
20198 assert!(is_decommission_capacity_blocked_error(&part_err));
20199
20200 let after_progress = {
20201 let pool_meta = store.pool_meta.read().await;
20202 let reservation = pool_meta.pools[0]
20203 .decommission
20204 .as_ref()
20205 .and_then(|info| info.capacity_reservation.as_ref())
20206 .expect("renewed reservation should remain active");
20207 (
20208 reservation.consumed_target_physical_bytes,
20209 reservation.inflight_target_physical_bytes,
20210 reservation.pending_target_physical_bytes,
20211 reservation.observed_target_physical_bytes,
20212 )
20213 };
20214 assert_eq!(after_progress, before_progress, "stale mutations must not consume the capacity ledger");
20215 let parts = store.pools[1]
20216 .list_object_parts(
20217 &bucket,
20218 multipart_object,
20219 &upload.upload_id,
20220 None,
20221 1_000,
20222 &ObjectOptions {
20223 expected_bucket_incarnation_id: Some(incarnation),
20224 ..Default::default()
20225 },
20226 )
20227 .await
20228 .expect("the seeded upload should remain readable");
20229 assert!(parts.parts.is_empty(), "the stale multipart write must not stage a part");
20230 assert!(
20231 store.pools[1]
20232 .get_object_info(&bucket, stale_put_object, &ObjectOptions::default())
20233 .await
20234 .is_err(),
20235 "the stale PUT must not create a target object"
20236 );
20237 }
20238
20239 #[tokio::test]
20240 #[serial_test::serial]
20241 async fn decommission_activation_lock_recheck_rejects_sudden_space_drop_without_persisting() {
20242 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
20243 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20244 let source = DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30);
20245 let preflight = vec![source, DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0)];
20246 {
20247 let pool_meta = store.pool_meta.read().await;
20248 ensure_decommission_start_target_capacity(&pool_meta, &[0], &preflight, true)
20249 .expect("the pre-lock capacity snapshot should fit exactly");
20250 }
20251
20252 set_decommission_capacity_info_overrides_for_test(
20253 store.id,
20254 vec![vec![source, DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)]],
20255 );
20256 let err = store
20257 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20258 .await
20259 .expect_err("the capacity snapshot inside the activation lock must be authoritative");
20260 assert!(err.to_string().contains("requires 60 bytes, but 59 bytes are available"));
20261
20262 let local = store.pool_meta.read().await;
20263 assert!(local.pools[0].decommission.is_none());
20264 drop(local);
20265 let mut persisted = PoolMeta::default();
20266 persisted
20267 .load_no_lock_from_replicas(store.pools.clone())
20268 .await
20269 .expect("the rejected activation must leave baseline metadata readable");
20270 assert!(persisted.pools[0].decommission.is_none());
20271 }
20272
20273 #[tokio::test]
20274 #[serial_test::serial]
20275 async fn decommission_worker_pauses_on_runtime_capacity_shortage_without_source_side_effect() {
20276 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
20277 let bucket = format!("capacity-blocked-{}", uuid::Uuid::new_v4());
20278 let object = "object.bin";
20279 store
20280 .make_bucket(&bucket, &MakeBucketOptions::default())
20281 .await
20282 .expect("the production-path bucket should be created");
20283 let mut source = crate::object_api::PutObjReader::from_vec(b"source remains authoritative".to_vec());
20284 store.pools[0]
20285 .put_object(&bucket, object, &mut source, &ObjectOptions::default())
20286 .await
20287 .expect("the source object should be written before decommission starts");
20288
20289 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20290 let enough = vec![
20291 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
20292 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
20293 ];
20294 set_decommission_capacity_info_overrides_for_test(store.id, vec![enough.clone()]);
20295 store
20296 .save_current_pool_meta_for_decommission_start(
20297 &[0],
20298 vec![DecomBucketInfo {
20299 name: bucket.clone(),
20300 prefix: String::new(),
20301 }],
20302 )
20303 .await
20304 .expect("the initial reservation should be activated");
20305
20306 let shortage = vec![enough[0], DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)];
20307 set_decommission_capacity_info_overrides_for_test(store.id, vec![shortage]);
20308 let canceler = DecommissionCanceler::new(CancellationToken::new());
20309 store.decommission_cancelers.write().await[0] = Some(canceler.clone());
20310 store
20311 .do_decommission_in_routine(canceler, 0, Arc::new(Semaphore::new(1)))
20312 .await
20313 .expect("runtime capacity shortage should pause the worker, not fail it");
20314
20315 store.pools[0]
20316 .get_object_info(&bucket, object, &ObjectOptions::default())
20317 .await
20318 .expect("capacity pause must not delete the source object");
20319 let target_result = store.pools[1]
20320 .get_object_info(&bucket, object, &ObjectOptions::default())
20321 .await;
20322 assert!(
20323 target_result
20324 .as_ref()
20325 .err()
20326 .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)),
20327 "capacity pause must happen before a target mutation"
20328 );
20329
20330 let local = store.pool_meta.read().await;
20331 let info = local.pools[0]
20332 .decommission
20333 .as_ref()
20334 .expect("the blocked decommission state should remain present");
20335 assert!(
20336 !info.complete && !info.failed && !info.canceled,
20337 "capacity-blocked state unexpectedly became terminal: complete={}, failed={}, canceled={}, blocked_reason={:?}",
20338 info.complete,
20339 info.failed,
20340 info.canceled,
20341 info.capacity_blocked_reason
20342 );
20343 assert_eq!(info.items_decommission_failed, 0);
20344 assert_eq!(info.bytes_failed, 0);
20345 assert!(info.capacity_blocked_reason.is_some());
20346 assert!(
20347 info.capacity_reservation
20348 .as_ref()
20349 .is_some_and(DecommissionCapacityReservation::active),
20350 "blocked work must retain and renew its reservation"
20351 );
20352 drop(local);
20353
20354 let mut persisted = PoolMeta::default();
20355 persisted
20356 .load_no_lock_from_replicas(store.pools.clone())
20357 .await
20358 .expect("the blocked state should be durable");
20359 let persisted_info = persisted.pools[0]
20360 .decommission
20361 .as_ref()
20362 .expect("the durable blocked state should remain nonterminal");
20363 assert!(!persisted_info.complete && !persisted_info.failed && !persisted_info.canceled);
20364 assert!(persisted_info.capacity_blocked_reason.is_some());
20365 assert!(
20366 persisted_info
20367 .capacity_reservation
20368 .as_ref()
20369 .is_some_and(DecommissionCapacityReservation::active)
20370 );
20371 }
20372
20373 #[tokio::test]
20374 #[serial_test::serial]
20375 async fn decommission_worker_rechecks_runtime_capacity_before_empty_background_completion() {
20376 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
20377 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20378 let enough = vec![
20379 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
20380 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
20381 ];
20382 set_decommission_capacity_info_overrides_for_test(store.id, vec![enough.clone()]);
20383 store
20384 .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20385 .await
20386 .expect("the initial reservation should be activated");
20387
20388 let shortage = vec![enough[0], DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)];
20389 set_decommission_capacity_info_overrides_for_test(store.id, vec![shortage]);
20390 let canceler = DecommissionCanceler::new(CancellationToken::new());
20391 store.decommission_cancelers.write().await[0] = Some(canceler.clone());
20392 store
20393 .do_decommission_in_routine(canceler, 0, Arc::new(Semaphore::new(1)))
20394 .await
20395 .expect("runtime capacity shortage should pause the worker before background completion");
20396
20397 let local = store.pool_meta.read().await;
20398 let info = local.pools[0]
20399 .decommission
20400 .as_ref()
20401 .expect("the blocked decommission state should remain present");
20402 assert!(!info.complete && !info.failed && !info.canceled);
20403 assert!(info.capacity_blocked_reason.is_some());
20404 assert!(
20405 info.capacity_reservation
20406 .as_ref()
20407 .is_some_and(DecommissionCapacityReservation::active)
20408 );
20409 }
20410
20411 fn pool_meta_replica_test_meta(cmd_line: &str) -> PoolMeta {
20412 PoolMeta {
20413 version: POOL_META_VERSION,
20414 pools: vec![PoolStatus {
20415 id: 0,
20416 cmd_line: cmd_line.to_string(),
20417 last_update: OffsetDateTime::UNIX_EPOCH,
20418 decommission: None,
20419 }],
20420 dont_save: false,
20421 }
20422 }
20423
20424 fn pool_meta_replica_test_data(cmd_line: &str) -> Vec<u8> {
20425 pool_meta_replica_test_meta(cmd_line)
20426 .encode_config_data()
20427 .expect("pool metadata should encode")
20428 }
20429
20430 fn pool_meta_v1_replica_test_data(meta: &PoolMeta) -> Vec<u8> {
20431 let mut data = Vec::new();
20432 data.write_u16::<LittleEndian>(POOL_META_FORMAT)
20433 .expect("pool metadata format should encode");
20434 data.write_u16::<LittleEndian>(POOL_META_V1_VERSION)
20435 .expect("pool metadata version should encode");
20436 PersistedPoolMetaV1::from(meta)
20437 .serialize(&mut Serializer::new(&mut data))
20438 .expect("legacy pool metadata should encode");
20439 data
20440 }
20441
20442 fn pool_meta_persisted_v1_replica_test_data(cmd_line: &str) -> Vec<u8> {
20443 pool_meta_v1_replica_test_data(&pool_meta_replica_test_meta(cmd_line))
20444 }
20445
20446 fn pool_meta_legacy_v1_replica_test_data(cmd_line: &str) -> Vec<u8> {
20447 let mut data = Vec::new();
20448 data.write_u16::<LittleEndian>(POOL_META_FORMAT)
20449 .expect("pool metadata format should encode");
20450 data.write_u16::<LittleEndian>(POOL_META_V1_VERSION)
20451 .expect("pool metadata version should encode");
20452 LegacyPoolMeta {
20453 version: POOL_META_V1_VERSION,
20454 pools: vec![LegacyPoolStatus {
20455 id: 0,
20456 cmd_line: cmd_line.to_string(),
20457 last_update: OffsetDateTime::UNIX_EPOCH,
20458 decommission: None,
20459 }],
20460 dont_save: false,
20461 }
20462 .serialize(&mut Serializer::new(&mut data))
20463 .expect("legacy v1 pool metadata should encode");
20464 data
20465 }
20466
20467 #[test]
20468 fn pool_meta_replica_selection_falls_back_from_corrupt_first_copy() {
20469 let selection = select_pool_meta_replica(vec![
20470 PoolMetaReplica::Corrupt("truncated".to_string()),
20471 decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
20472 ])
20473 .expect("a validated backup replica should be selected");
20474
20475 assert!(selection.replica_state.needs_repair);
20476 assert!(selection.replica_state.repair_write_safe);
20477 assert_eq!(selection.meta.pools.len(), 1);
20478 assert_eq!(selection.meta.pools[0].cmd_line, "pool-0");
20479 }
20480
20481 #[test]
20482 fn pool_meta_replica_selection_rejects_incompatible_copy() {
20483 let valid = pool_meta_replica_test_data("pool-0");
20484 let mut incompatible = valid.clone();
20485 LittleEndian::write_u16(&mut incompatible[2..4], POOL_META_VERSION + 1);
20486
20487 let err = select_pool_meta_replica(vec![decode_pool_meta_replica(valid), decode_pool_meta_replica(incompatible)])
20488 .expect_err("an incompatible replica must block fallback and repair writes");
20489
20490 assert!(err.to_string().contains("pool 1 is incompatible"));
20491 assert!(err.to_string().contains("without overwriting it"));
20492 }
20493
20494 #[test]
20495 fn pool_meta_replica_selection_prefers_valid_canonical_first_copy() {
20496 let selection = select_pool_meta_replica(vec![
20497 decode_pool_meta_replica(pool_meta_replica_test_data("pool-canonical")),
20498 decode_pool_meta_replica(pool_meta_replica_test_data("pool-stale")),
20499 ])
20500 .expect("pool zero is the durable commit record when a later replica is stale");
20501
20502 assert_eq!(selection.meta.pools[0].cmd_line, "pool-canonical");
20503 assert!(selection.replica_state.needs_repair);
20504 assert!(selection.replica_state.repair_write_safe);
20505 }
20506
20507 #[test]
20508 fn pool_meta_replica_selection_rejects_divergent_backups_without_canonical() {
20509 let err = select_pool_meta_replica(vec![
20510 PoolMetaReplica::Corrupt("canonical unavailable".to_string()),
20511 decode_pool_meta_replica(pool_meta_replica_test_data("pool-old")),
20512 decode_pool_meta_replica(pool_meta_replica_test_data("pool-new")),
20513 ])
20514 .expect_err("divergent backups have no safe ordering when the canonical copy is unavailable");
20515
20516 assert!(err.to_string().contains("valid replicas in pools 1 and 2 diverge"));
20517 }
20518
20519 #[test]
20520 fn pool_meta_replica_selection_normalizes_equivalent_legacy_copy() {
20521 let selection = select_pool_meta_replica(vec![
20522 decode_pool_meta_replica(pool_meta_legacy_v1_replica_test_data("pool-0")),
20523 decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
20524 ])
20525 .expect("equivalent legacy and current encodings should be compatible");
20526
20527 assert!(selection.replica_state.needs_repair);
20528 assert!(selection.replica_state.repair_write_safe);
20529 assert_eq!(selection.meta.pools[0].cmd_line, "pool-0");
20530 }
20531
20532 #[test]
20533 fn pool_meta_v1_replica_migrates_to_v2_canonical_form() {
20534 let mut source = pool_meta_replica_test_meta("pool-0");
20535 source.pools[0].decommission = Some(PoolDecommissionInfo {
20536 start_time: Some(OffsetDateTime::UNIX_EPOCH),
20537 queued_buckets: vec!["bucket-a".to_string()],
20538 ..Default::default()
20539 });
20540 let data = pool_meta_v1_replica_test_data(&source);
20541 let PoolMetaReplica::Valid {
20542 raw, canonical, meta, ..
20543 } = decode_pool_meta_replica(data)
20544 else {
20545 panic!("v1 pool metadata should remain readable");
20546 };
20547
20548 assert_eq!(LittleEndian::read_u16(&raw[2..4]), POOL_META_V1_VERSION);
20549 assert_eq!(LittleEndian::read_u16(&canonical[2..4]), POOL_META_VERSION);
20550 assert_eq!(meta.version, POOL_META_V1_VERSION);
20551 let info = meta.pools[0]
20552 .decommission
20553 .as_ref()
20554 .expect("v1 decommission state should migrate");
20555 assert_eq!(info.queued_buckets, vec!["bucket-a".to_string()]);
20556 assert!(info.unresolved_entries.is_empty());
20557 }
20558
20559 #[test]
20560 fn pool_meta_legacy_v1_replica_migrates_to_v2_canonical_form() {
20561 let data = pool_meta_legacy_v1_replica_test_data("pool-0");
20562 let PoolMetaReplica::Valid {
20563 raw, canonical, meta, ..
20564 } = decode_pool_meta_replica(data)
20565 else {
20566 panic!("legacy v1 pool metadata should remain readable");
20567 };
20568
20569 assert_eq!(LittleEndian::read_u16(&raw[2..4]), POOL_META_V1_VERSION);
20570 assert_eq!(LittleEndian::read_u16(&canonical[2..4]), POOL_META_VERSION);
20571 assert_eq!(meta.version, POOL_META_V1_VERSION);
20572 assert_eq!(meta.pools[0].cmd_line, "pool-0");
20573 }
20574
20575 #[test]
20576 fn pool_meta_replica_selection_rejects_v1_v2_divergence() {
20577 let err = select_pool_meta_replica(vec![
20578 decode_pool_meta_replica(pool_meta_persisted_v1_replica_test_data("pool-old")),
20579 decode_pool_meta_replica(pool_meta_replica_test_data("pool-new")),
20580 ])
20581 .expect_err("different v1 and v2 snapshots must remain fail-closed");
20582
20583 assert!(err.to_string().contains("valid replicas in pools 0 and 1 diverge"));
20584 }
20585
20586 #[test]
20587 fn pool_meta_replica_rejects_header_payload_version_mismatch() {
20588 let mut v1_with_v2_header = pool_meta_persisted_v1_replica_test_data("pool-0");
20589 LittleEndian::write_u16(&mut v1_with_v2_header[2..4], POOL_META_VERSION);
20590 assert!(matches!(decode_pool_meta_replica(v1_with_v2_header), PoolMetaReplica::Corrupt(_)));
20591
20592 let mut v2_with_v1_header = pool_meta_replica_test_data("pool-0");
20593 LittleEndian::write_u16(&mut v2_with_v1_header[2..4], POOL_META_V1_VERSION);
20594 assert!(matches!(
20595 decode_pool_meta_replica(v2_with_v1_header),
20596 PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
20597 ));
20598 }
20599
20600 #[test]
20601 fn pool_meta_v2_header_guards_v1_readers_from_tuple_extension() {
20602 let mut meta = pool_meta_replica_test_meta("pool-0");
20603 meta.pools[0].decommission = Some(PoolDecommissionInfo::default());
20604 let data = meta.encode_config_data().expect("v2 pool metadata should encode");
20605
20606 assert_eq!(LittleEndian::read_u16(&data[2..4]), POOL_META_VERSION);
20607 assert_ne!(POOL_META_VERSION, POOL_META_V1_VERSION);
20608 assert!(rmp_serde::from_slice::<PersistedPoolMetaV1>(&data[4..]).is_err());
20609 }
20610
20611 #[test]
20612 fn pool_meta_writer_stays_v1_until_v2_is_fleet_confirmed() {
20613 let mut meta = pool_meta_replica_test_meta("pool-0");
20614 meta.version = POOL_META_V1_VERSION;
20615
20616 let v1 = meta
20617 .encode_config_data_for_v2_gate(false)
20618 .expect("the default writer should preserve v1");
20619 let v2 = meta
20620 .encode_config_data_for_v2_gate(true)
20621 .expect("the confirmed writer should emit v2");
20622
20623 assert_eq!(LittleEndian::read_u16(&v1[2..4]), POOL_META_V1_VERSION);
20624 assert_eq!(LittleEndian::read_u16(&v2[2..4]), POOL_META_VERSION);
20625 assert!(rmp_serde::from_slice::<PersistedPoolMetaV1>(&v1[4..]).is_ok());
20626 }
20627
20628 #[test]
20629 fn pool_meta_v2_writer_requires_both_gates() {
20630 assert!(!pool_meta_v2_writer_enabled_for(false, false));
20631 assert!(!pool_meta_v2_writer_enabled_for(true, false));
20632 assert!(!pool_meta_v2_writer_enabled_for(false, true));
20633 assert!(pool_meta_v2_writer_enabled_for(true, true));
20634 }
20635
20636 #[test]
20637 fn pool_meta_v3_writer_requires_both_gates() {
20638 assert!(!pool_meta_v3_writer_enabled_for(false, false));
20639 assert!(!pool_meta_v3_writer_enabled_for(true, false));
20640 assert!(!pool_meta_v3_writer_enabled_for(false, true));
20641 assert!(pool_meta_v3_writer_enabled_for(true, true));
20642 }
20643
20644 #[test]
20645 fn decommission_ledger_persistence_requires_an_observed_or_confirmed_format() {
20646 for (version, v2_enabled, v3_enabled, expected) in [
20647 (POOL_META_V1_VERSION, false, false, false),
20648 (POOL_META_V1_VERSION, true, false, true),
20649 (POOL_META_V1_VERSION, false, true, true),
20650 (POOL_META_VERSION, false, false, true),
20651 (super::POOL_META_GENERATION_VERSION, false, false, true),
20652 ] {
20653 let result = super::ensure_decommission_ledger_persistence_supported_for(version, v2_enabled, v3_enabled);
20654 assert_eq!(
20655 result.is_ok(),
20656 expected,
20657 "unexpected capability result for pool metadata version {version}"
20658 );
20659 }
20660
20661 let half_confirmed_v2 = pool_meta_v2_writer_enabled_for(true, false);
20662 let half_confirmed_v3 = pool_meta_v3_writer_enabled_for(false, true);
20663 let err = super::ensure_decommission_ledger_persistence_supported_for(
20664 POOL_META_V1_VERSION,
20665 half_confirmed_v2,
20666 half_confirmed_v3,
20667 )
20668 .expect_err("half-enabled rollout gates must not admit decommission");
20669 assert!(matches!(err, Error::InvalidArgument(..)));
20670 assert!(err.to_string().contains("durable unresolved-entry recovery"));
20671 assert!(err.to_string().contains(rustfs_config::ENV_POOL_META_V3_WRITE));
20672 assert!(err.to_string().contains(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED));
20673 }
20674
20675 #[test]
20676 fn decommission_capacity_accepts_v3_only_writer_rollout() {
20677 assert!(decommission_capacity_writer_supported_for(
20678 POOL_META_V1_VERSION,
20679 false,
20680 pool_meta_v3_writer_enabled_for(true, true),
20681 ));
20682 assert!(!decommission_capacity_writer_supported_for(
20683 POOL_META_V1_VERSION,
20684 false,
20685 pool_meta_v3_writer_enabled_for(true, false),
20686 ));
20687 }
20688
20689 #[test]
20690 fn pool_meta_stale_write_rejection_metric_is_countable() {
20691 let recorder = metrics_util::debugging::DebuggingRecorder::new();
20692 let snapshotter = recorder.snapshotter();
20693 metrics::with_local_recorder(&recorder, || {
20694 record_pool_meta_stale_write_rejection("prepare_cas");
20695 record_pool_meta_stale_write_rejection("prepare_cas");
20696 });
20697
20698 let total = snapshotter
20699 .snapshot()
20700 .into_vec()
20701 .into_iter()
20702 .filter(|(composite, _, _, _)| composite.key().name() == METRIC_POOL_META_STALE_WRITE_REJECTIONS_TOTAL)
20703 .filter_map(|(_, _, _, value)| match value {
20704 metrics_util::debugging::DebugValue::Counter(value) => Some(value),
20705 _ => None,
20706 })
20707 .sum::<u64>();
20708 assert_eq!(total, 2);
20709 }
20710
20711 #[test]
20712 fn pool_meta_block_transition_is_counted_once_across_both_latches() {
20713 let recorder = metrics_util::debugging::DebuggingRecorder::new();
20714 let snapshotter = recorder.snapshotter();
20715 metrics::with_local_recorder(&recorder, || {
20716 let mut state = PoolMetaWriteState::default();
20717 let mut arm = state.arm_transaction();
20718 arm.phase = Some("commit_cas");
20719 drop(arm);
20720 let first = state
20721 .ensure_write_safe("first block")
20722 .unwrap_err()
20723 .pool_metadata_failure()
20724 .unwrap()
20725 .since;
20726 state.block_writes_after_fence_loss();
20727 state.block_writes();
20728 assert_eq!(
20729 state
20730 .ensure_write_safe("later block")
20731 .unwrap_err()
20732 .pool_metadata_failure()
20733 .unwrap()
20734 .since,
20735 first
20736 );
20737 });
20738 let total = snapshotter
20739 .snapshot()
20740 .into_vec()
20741 .into_iter()
20742 .filter(|(key, _, _, _)| key.key().name() == "rustfs_pool_metadata_blocks_total")
20743 .filter_map(|(_, _, _, value)| match value {
20744 metrics_util::debugging::DebugValue::Counter(count) => Some(count),
20745 _ => None,
20746 })
20747 .sum::<u64>();
20748 assert_eq!(total, 1);
20749 }
20750
20751 fn pool_meta_v3_test_revision(cluster_id: uuid::Uuid, generation: u64, transaction_id: uuid::Uuid) -> PoolMetaRevision {
20752 PoolMetaRevision {
20753 version: POOL_META_GENERATION_VERSION,
20754 cluster_id: Some(cluster_id),
20755 epoch: POOL_META_INITIAL_EPOCH,
20756 generation,
20757 transaction_id: Some(transaction_id),
20758 }
20759 }
20760
20761 fn pool_meta_legacy_candidate(meta: PoolMeta) -> PoolMetaCommittedCandidate {
20762 PoolMetaCommittedCandidate {
20763 canonical: meta
20764 .encode_config_data_for_v2_gate(true)
20765 .expect("legacy pool metadata should encode"),
20766 revision: PoolMetaRevision::legacy(meta.version),
20767 meta,
20768 }
20769 }
20770
20771 #[test]
20772 fn pool_meta_v3_pending_prepare_recovers_previous_snapshot_after_restart() {
20773 let previous = pool_meta_replica_test_meta("pool-before");
20774 let mut next = pool_meta_replica_test_meta("pool-after");
20775 next.version = POOL_META_GENERATION_VERSION;
20776 let revision = pool_meta_v3_test_revision(uuid::Uuid::new_v4(), 1, uuid::Uuid::new_v4());
20777 let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&pool_meta_legacy_candidate(previous.clone())))
20778 .expect("pending generation should encode");
20779
20780 let selected = select_pool_meta_replica(vec![
20781 decode_pool_meta_replica(pending),
20782 decode_pool_meta_replica(
20783 previous
20784 .encode_config_data_for_v2_gate(true)
20785 .expect("previous snapshot should encode"),
20786 ),
20787 ])
20788 .expect("a prepare-only transaction should expose its committed predecessor");
20789
20790 assert_eq!(selected.meta.pools[0].cmd_line, "pool-before");
20791 assert_eq!(selected.revision.version, POOL_META_VERSION);
20792 assert!(selected.replica_state.needs_repair);
20793 }
20794
20795 #[test]
20796 fn pool_meta_v3_partial_commit_selects_new_generation_after_restart() {
20797 let previous = pool_meta_replica_test_meta("pool-before");
20798 let mut next = pool_meta_replica_test_meta("pool-after");
20799 next.version = POOL_META_GENERATION_VERSION;
20800 let revision = pool_meta_v3_test_revision(uuid::Uuid::new_v4(), 1, uuid::Uuid::new_v4());
20801 let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&pool_meta_legacy_candidate(previous)))
20802 .expect("pending generation should encode");
20803 let committed = encode_pool_meta_v3_envelope(&next, revision, true, None).expect("committed generation should encode");
20804
20805 let selected = select_pool_meta_replica(vec![decode_pool_meta_replica(committed), decode_pool_meta_replica(pending)])
20806 .expect("one committed replica should make the cross-pool transaction durable");
20807
20808 assert_eq!(selected.meta.pools[0].cmd_line, "pool-after");
20809 assert_eq!(selected.revision, revision);
20810 assert!(selected.replica_state.needs_repair);
20811 }
20812
20813 #[test]
20814 fn pool_meta_v3_uses_valid_committed_backup_but_rejects_same_generation_fork() {
20815 let cluster_id = uuid::Uuid::new_v4();
20816 let mut next = pool_meta_replica_test_meta("pool-after");
20817 next.version = POOL_META_GENERATION_VERSION;
20818 let revision = pool_meta_v3_test_revision(cluster_id, 7, uuid::Uuid::new_v4());
20819 let committed = encode_pool_meta_v3_envelope(&next, revision, true, None).expect("committed generation should encode");
20820 let selected = select_pool_meta_replica(vec![
20821 PoolMetaReplica::Corrupt("truncated".to_string()),
20822 decode_pool_meta_replica(committed.clone()),
20823 ])
20824 .expect("a corrupt first copy should fall back to a valid committed generation");
20825 assert_eq!(selected.revision, revision);
20826 assert!(selected.replica_state.needs_repair);
20827
20828 let fork_revision = pool_meta_v3_test_revision(cluster_id, 7, uuid::Uuid::new_v4());
20829 let fork = encode_pool_meta_v3_envelope(&next, fork_revision, true, None).expect("fork generation should encode");
20830 let err = select_pool_meta_replica(vec![decode_pool_meta_replica(committed), decode_pool_meta_replica(fork)])
20831 .expect_err("same-generation transactions must never be selected by pool order");
20832 assert!(err.to_string().contains("committed generation 7 diverges"));
20833 }
20834
20835 #[test]
20836 fn pool_meta_v3_migration_keeps_legacy_committed_until_commit_record_exists() {
20837 let previous = pool_meta_replica_test_meta("pool-before");
20838 let mut next = pool_meta_replica_test_meta("pool-after");
20839 next.version = POOL_META_GENERATION_VERSION;
20840 let revision = pool_meta_v3_test_revision(uuid::Uuid::new_v4(), 1, uuid::Uuid::new_v4());
20841 let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&pool_meta_legacy_candidate(previous.clone())))
20842 .expect("pending migration should encode");
20843 let legacy = previous
20844 .encode_config_data_for_v2_gate(true)
20845 .expect("legacy snapshot should encode");
20846
20847 let prepared = select_pool_meta_replica(vec![decode_pool_meta_replica(pending), decode_pool_meta_replica(legacy)])
20848 .expect("prepared migration should retain the legacy commit");
20849 assert_eq!(prepared.meta.version, POOL_META_VERSION);
20850 assert_eq!(prepared.meta.pools[0].cmd_line, "pool-before");
20851
20852 let committed = encode_pool_meta_v3_envelope(&next, revision, true, None).expect("committed migration should encode");
20853 let migrated = select_pool_meta_replica(vec![decode_pool_meta_replica(committed)])
20854 .expect("committed migration should establish the V3 floor");
20855 assert_eq!(migrated.meta.version, POOL_META_GENERATION_VERSION);
20856 assert_eq!(migrated.meta.pools[0].cmd_line, "pool-after");
20857 }
20858
20859 #[test]
20860 fn pool_meta_v3_unknown_fields_remain_incompatible_not_silently_ignored() {
20861 #[derive(Serialize)]
20862 struct FuturePoolMetaV3 {
20863 version: u16,
20864 cluster_id: String,
20865 epoch: u64,
20866 generation: u64,
20867 transaction_id: String,
20868 committed: bool,
20869 pools: Vec<PersistedPoolStatus>,
20870 previous: Option<PersistedPoolMetaV3Previous>,
20871 future_guard: u64,
20872 }
20873
20874 let mut data = Vec::new();
20875 data.write_u16::<LittleEndian>(POOL_META_FORMAT)
20876 .expect("pool metadata format should encode");
20877 data.write_u16::<LittleEndian>(POOL_META_GENERATION_VERSION)
20878 .expect("pool metadata version should encode");
20879 FuturePoolMetaV3 {
20880 version: POOL_META_GENERATION_VERSION,
20881 cluster_id: uuid::Uuid::new_v4().to_string(),
20882 epoch: POOL_META_INITIAL_EPOCH,
20883 generation: 1,
20884 transaction_id: uuid::Uuid::new_v4().to_string(),
20885 committed: true,
20886 pools: Vec::new(),
20887 previous: None,
20888 future_guard: 1,
20889 }
20890 .serialize(&mut Serializer::new(&mut data))
20891 .expect("future V3 payload should encode");
20892
20893 assert!(matches!(decode_pool_meta_replica(data), PoolMetaReplica::Incompatible(_)));
20894 }
20895
20896 #[test]
20897 fn pool_meta_identity_classifies_corrupt_incompatible_and_divergent_replicas() {
20898 let cluster_id = uuid::Uuid::new_v4();
20899 let identity = PersistedPoolMetaIdentity {
20900 version: POOL_META_IDENTITY_VERSION,
20901 cluster_id,
20902 epoch: POOL_META_INITIAL_EPOCH,
20903 initialized: true,
20904 fresh_bootstrap_nonce: None,
20905 };
20906 let selected = select_pool_meta_identity(
20907 vec![
20908 PoolMetaIdentityRead {
20909 replica: PoolMetaIdentityReplica::Corrupt("truncated".to_string()),
20910 cas: PoolMetaCasToken::Existing("corrupt".to_string()),
20911 },
20912 PoolMetaIdentityRead {
20913 replica: PoolMetaIdentityReplica::Valid(identity),
20914 cas: PoolMetaCasToken::Existing("valid".to_string()),
20915 },
20916 ],
20917 cluster_id,
20918 )
20919 .expect("a verified identity backup should survive a corrupt first replica");
20920 assert_eq!(selected.identity, Some(identity));
20921 assert!(selected.needs_repair);
20922
20923 let incompatible = select_pool_meta_identity(
20924 vec![PoolMetaIdentityRead {
20925 replica: PoolMetaIdentityReplica::Incompatible("future version".to_string()),
20926 cas: PoolMetaCasToken::Existing("future".to_string()),
20927 }],
20928 cluster_id,
20929 )
20930 .expect_err("an incompatible identity must fail closed");
20931 assert!(incompatible.to_string().contains("incompatible"));
20932
20933 let divergent = select_pool_meta_identity(
20934 vec![
20935 PoolMetaIdentityRead {
20936 replica: PoolMetaIdentityReplica::Valid(identity),
20937 cas: PoolMetaCasToken::Existing("epoch-1".to_string()),
20938 },
20939 PoolMetaIdentityRead {
20940 replica: PoolMetaIdentityReplica::Valid(PersistedPoolMetaIdentity {
20941 epoch: POOL_META_INITIAL_EPOCH + 1,
20942 ..identity
20943 }),
20944 cas: PoolMetaCasToken::Existing("epoch-2".to_string()),
20945 },
20946 ],
20947 cluster_id,
20948 )
20949 .expect_err("identity epoch divergence must require recovery");
20950 assert!(divergent.to_string().contains("recovery required"));
20951 }
20952
20953 #[test]
20954 fn pool_meta_v2_floor_is_sticky_after_observation() {
20955 let meta = pool_meta_replica_test_meta("pool-0");
20956 let data = meta
20957 .encode_config_data_for_v2_gate(false)
20958 .expect("an observed v2 snapshot must not be downgraded");
20959
20960 assert_eq!(LittleEndian::read_u16(&data[2..4]), POOL_META_VERSION);
20961 }
20962
20963 #[test]
20964 fn pool_meta_v1_writer_rejects_unresolved_ledger() {
20965 let mut meta = pool_meta_replica_test_meta("pool-0");
20966 meta.version = POOL_META_V1_VERSION;
20967 meta.pools[0].decommission = Some(PoolDecommissionInfo {
20968 start_time: Some(OffsetDateTime::UNIX_EPOCH),
20969 unresolved_entries: vec![DecommissionUnresolvedEntry {
20970 bucket: "bucket-a".to_string(),
20971 object: "object-a".to_string(),
20972 pool_index: 0,
20973 set_index: 0,
20974 source_generation: OffsetDateTime::UNIX_EPOCH,
20975 candidate_count: 1,
20976 disk_error_count: 1,
20977 observed_at: OffsetDateTime::UNIX_EPOCH,
20978 reason: "metadata_resolution_failed".to_string(),
20979 }],
20980 ..Default::default()
20981 });
20982
20983 let err = meta
20984 .encode_config_data_for_v2_gate(false)
20985 .expect_err("v1 must not drop the unresolved ledger");
20986 assert!(err.to_string().contains("pool metadata V2 is required"));
20987 }
20988
20989 #[test]
20990 fn pool_meta_v1_writer_rejects_capacity_reservations() {
20991 let mut meta = pool_meta_replica_test_meta("pool-0");
20992 meta.version = POOL_META_V1_VERSION;
20993 let reservation = build_decommission_capacity_reservation(
20994 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 1, parity: 0 }, 0, 10, 10),
20995 DecommissionErasureLayout { data: 1, parity: 0 },
20996 uuid::Uuid::new_v4(),
20997 1,
20998 OffsetDateTime::UNIX_EPOCH,
20999 )
21000 .expect("test reservation should be valid");
21001 meta.pools[0].decommission = Some(PoolDecommissionInfo {
21002 start_time: Some(OffsetDateTime::UNIX_EPOCH),
21003 capacity_reservation: Some(reservation),
21004 ..Default::default()
21005 });
21006
21007 let err = meta
21008 .encode_config_data_for_v2_gate(false)
21009 .expect_err("v1 must not drop a distributed capacity reservation");
21010 assert!(
21011 err.to_string()
21012 .contains("pool metadata V2 is required to persist decommission capacity reservations")
21013 );
21014 }
21015
21016 #[test]
21017 fn pool_meta_replica_selection_preserves_observed_v2_floor() {
21018 let selection = select_pool_meta_replica(vec![
21019 decode_pool_meta_replica(pool_meta_persisted_v1_replica_test_data("pool-0")),
21020 decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
21021 ])
21022 .expect("equivalent v1 and v2 replicas should converge");
21023
21024 assert_eq!(selection.meta.version, POOL_META_VERSION);
21025 let data = selection
21026 .meta
21027 .encode_config_data_for_v2_gate(false)
21028 .expect("the selected v2 floor must remain writable");
21029 assert_eq!(LittleEndian::read_u16(&data[2..4]), POOL_META_VERSION);
21030 }
21031
21032 #[test]
21033 fn pool_meta_replica_selection_distinguishes_absent_from_unrecoverable() {
21034 assert!(matches!(decode_pool_meta_replica(Vec::new()), PoolMetaReplica::Corrupt(_)));
21035
21036 let empty = select_pool_meta_replica(vec![PoolMetaReplica::Missing, PoolMetaReplica::Missing])
21037 .expect("all missing replicas should preserve new-deployment behavior");
21038 assert!(empty.meta.pools.is_empty());
21039 assert!(!empty.replica_state.needs_repair);
21040 assert!(empty.replica_state.repair_write_safe);
21041
21042 let err = select_pool_meta_replica(vec![
21043 PoolMetaReplica::Missing,
21044 PoolMetaReplica::Unreadable(Arc::new(Error::other("read quorum unavailable"))),
21045 ])
21046 .expect_err("an unreadable replica must not be treated as a new deployment");
21047 assert!(err.to_string().contains("no valid committed replica is available"));
21048 assert!(err.to_string().contains("pool 1 is unreadable"));
21049 }
21050
21051 #[test]
21052 fn pool_meta_replica_selection_blocks_repair_for_unreadable_copy() {
21053 let selection = select_pool_meta_replica(vec![
21054 decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
21055 PoolMetaReplica::Unreadable(Arc::new(Error::other("read quorum unavailable"))),
21056 ])
21057 .expect("a validated replica should remain usable while another copy is unreadable");
21058
21059 assert!(selection.replica_state.needs_repair);
21060 assert!(!selection.replica_state.repair_write_safe);
21061 }
21062
21063 #[test]
21064 fn pool_meta_write_state_remains_blocked_after_unreadable_replica() {
21065 let mut write_state = PoolMetaWriteState::default();
21066 write_state.observe_replicas(PoolMetaReplicaState {
21067 needs_repair: true,
21068 repair_write_safe: false,
21069 });
21070 write_state.observe_replicas(PoolMetaReplicaState {
21071 needs_repair: false,
21072 repair_write_safe: true,
21073 });
21074
21075 let err = write_state
21076 .ensure_write_safe("pool metadata save failed")
21077 .expect_err("a later clean read must not clear the startup write block");
21078 assert!(
21079 err.to_string()
21080 .contains("restart after all replicas are readable and consistent")
21081 );
21082 }
21083
21084 #[test]
21085 fn pool_meta_read_probe_does_not_latch_writer_state() {
21086 let write_state = PoolMetaWriteState::default();
21087 select_pool_meta_replicas_for_read_probe(
21088 &write_state,
21089 vec![PoolMetaReplica::Unreadable(Arc::new(Error::Timeout))],
21090 "capacity probe",
21091 )
21092 .expect_err("an unreadable probe replica must fail the current admission");
21093
21094 assert!(
21095 write_state.ensure_write_safe("ordinary object write").is_ok(),
21096 "a read-only capacity probe must not permanently latch the pool metadata writer"
21097 );
21098 }
21099
21100 #[test]
21101 fn pool_meta_preflight_drop_does_not_arm_but_started_drop_retains_first_failure() {
21102 let state = PoolMetaWriteState::default();
21103 drop(state.arm_transaction());
21104 state.ensure_write_safe("preflight retry").unwrap();
21105 let mut first = state.arm_transaction();
21106 first.phase = Some("identity_cas");
21107 let err = first.failure(Error::Timeout);
21108 drop(first);
21109 let context = state.ensure_write_safe("retry").unwrap_err();
21110 let first_failure = context.pool_metadata_failure().unwrap();
21111 assert_eq!(first_failure.kind, crate::error::PoolMetadataFailure::TransactionUnknown);
21112 assert_eq!(first_failure.phase, "identity_cas");
21113 assert!(matches!(first_failure.source.as_deref(), Some(Error::Timeout)));
21114 assert!(matches!(err.pool_metadata_failure().unwrap().source.as_deref(), Some(Error::Timeout)));
21115 let mut late = state.arm_transaction();
21116 late.phase = Some("commit_cas");
21117 drop(late);
21118 let later = state.ensure_write_safe("retry").unwrap_err();
21119 assert_eq!(later.pool_metadata_failure().unwrap().since, first_failure.since);
21120 assert_eq!(later.pool_metadata_failure().unwrap().phase, "identity_cas");
21121 assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
21122 }
21123
21124 #[tokio::test]
21125 #[serial_test::serial]
21126 async fn pool_meta_writer_retries_after_identity_read_failure_without_restart() {
21127 let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
21128 let mut disks = Vec::new();
21129 for set in &store.pools[1].disk_set {
21130 let mut guard = set.disks.write().await;
21131 let disk_count = guard.len();
21132 let original = std::mem::replace(&mut *guard, vec![None; disk_count]);
21133 disks.push((set.clone(), original));
21134 }
21135 let mut state = store.pool_meta_save_gate.lock().await;
21136 let err = store
21137 .acquire_pool_meta_write_guard(&mut state, "writer preflight")
21138 .await
21139 .unwrap_err();
21140 assert_eq!(
21141 err.pool_metadata_failure().unwrap().kind,
21142 crate::error::PoolMetadataFailure::ReadUnavailable
21143 );
21144 state.ensure_write_safe("retry").unwrap();
21145 for (set, original) in disks {
21146 *set.disks.write().await = original;
21147 }
21148 let (_guard, _) = store.acquire_pool_meta_write_guard(&mut state, "writer retry").await.unwrap();
21149 }
21150
21151 #[tokio::test]
21152 #[serial_test::serial]
21153 async fn pool_meta_runtime_recovery_publishes_durable_state_and_drains_old_workers() {
21154 let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
21155 let before = store.pool_meta.read().await.clone();
21156 let mut requested = before.clone();
21157 requested.pools[0].last_update += Duration::seconds(1);
21158 requested.version = POOL_META_GENERATION_VERSION;
21159 {
21160 let mut state = store.pool_meta_save_gate.lock().await;
21161 let (fence, _) = store.acquire_pool_meta_write_guard(&mut state, "test save").await.unwrap();
21162 let outcome = requested
21163 .save_no_lock_armed(store.pools.clone(), &mut state, fence.lock_lost_signal(), &[0])
21164 .await
21165 .unwrap();
21166 assert_eq!(outcome.committed.pools[0].last_update, requested.pools[0].last_update);
21167 drop(outcome);
21168 }
21169 assert!(!store.pool_meta_writes_ready().await);
21170 let worker = DecommissionCanceler::new(CancellationToken::new());
21171 store.decommission_cancelers.write().await[0] = Some(worker.clone());
21172 assert!(store.recover_pool_meta_transaction().await.is_err());
21173 assert!(worker.is_cancelled());
21174 assert!(!store.pool_meta_writes_ready().await);
21175 assert_eq!(store.pool_meta.read().await.pools[0].last_update, before.pools[0].last_update);
21176 worker.release();
21177 store.pool_meta.write().await.pools[0].last_update = requested.pools[0].last_update + Duration::seconds(10);
21178 let outstanding = store.pool_meta_save_gate.lock().await.arm_transaction();
21180 assert!(!store.recover_pool_meta_transaction().await.unwrap());
21181 drop(outstanding);
21182 assert!(store.recover_pool_meta_transaction().await.unwrap());
21183 assert!(store.pool_meta_writes_ready().await);
21184 let recovered = store.pool_meta_write_gate_status().await;
21185 assert!(recovered.writes_ready);
21186 assert!(!recovered.check_timed_out);
21187 assert!(!recovered.write_blocked);
21188 assert!(!recovered.transaction_aborted);
21189 assert_eq!((recovered.reason, recovered.phase, recovered.since_unix_secs), (None, None, None));
21190 assert_eq!(store.pool_meta.read().await.pools[0].last_update, requested.pools[0].last_update);
21191 assert!(!store.recover_pool_meta_transaction().await.unwrap());
21192 store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
21193 assert!(!store.recover_pool_meta_transaction().await.unwrap());
21194 assert!(!store.pool_meta_writes_ready().await);
21195 }
21196
21197 #[tokio::test]
21198 #[serial_test::serial]
21199 async fn pool_meta_readiness_is_bounded_while_save_gate_is_held() {
21200 let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
21201 let _state = store.pool_meta_save_gate.lock().await;
21202 let result = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_status())
21203 .await
21204 .unwrap();
21205 assert!(matches!(result, Err(Error::Timeout)));
21206 let snapshot = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_gate_status())
21207 .await
21208 .unwrap();
21209 assert!(!snapshot.writes_ready);
21210 assert!(snapshot.check_timed_out);
21211 assert!(!snapshot.write_blocked);
21212 assert!(!snapshot.transaction_aborted);
21213 assert_eq!((snapshot.reason, snapshot.phase, snapshot.since_unix_secs), (None, None, None));
21214 }
21215
21216 #[tokio::test]
21217 #[serial_test::serial]
21218 async fn pool_meta_cancelled_recovery_and_new_integrity_block_never_clear_original_gate() {
21219 let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
21220 let since = OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap();
21221 {
21222 let state = store.pool_meta_save_gate.lock().await;
21223 let mut arm = state.arm_transaction();
21224 arm.phase = Some("publication");
21225 drop(arm);
21226 state.transaction_failure.lock().unwrap().as_mut().unwrap().since = since;
21228 *state.block_started_at.lock().unwrap() = Some(since);
21229 }
21230 let original = store.pool_meta_write_gate_status().await;
21231 assert!(!original.writes_ready);
21232 assert!(!original.check_timed_out);
21233 assert!(!original.write_blocked);
21234 assert!(original.transaction_aborted);
21235 assert_eq!(original.reason, Some("transaction_unknown"));
21236 assert_eq!(original.phase, Some("publication"));
21237 assert_eq!(original.since_unix_secs, Some(since.unix_timestamp()));
21238 assert_eq!(store.pool_meta_write_gate_status().await, original);
21239 let publication_guard = store.pool_meta.write().await;
21240 let recovery = tokio::spawn({
21241 let store = store.clone();
21242 async move { store.recover_pool_meta_transaction().await }
21243 });
21244 tokio::time::timeout(std::time::Duration::from_secs(10), async {
21245 loop {
21246 if store.pool_meta_save_gate.try_lock().is_err() {
21247 break;
21248 }
21249 tokio::task::yield_now().await;
21250 }
21251 })
21252 .await
21253 .unwrap();
21254 recovery.abort();
21255 assert!(recovery.await.unwrap_err().is_cancelled());
21256 drop(publication_guard);
21257 assert!(!store.pool_meta_writes_ready().await);
21258 assert_eq!(store.pool_meta_write_gate_status().await, original);
21259 let start_guard = store.start_gate.lock().await;
21260 let recovery = tokio::spawn({
21261 let store = store.clone();
21262 async move { store.recover_pool_meta_transaction().await }
21263 });
21264 store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
21265 drop(start_guard);
21266 assert!(!recovery.await.unwrap().unwrap());
21267 assert_eq!(
21268 store
21269 .pool_meta_write_status()
21270 .await
21271 .unwrap_err()
21272 .pool_metadata_failure()
21273 .unwrap()
21274 .kind,
21275 crate::error::PoolMetadataFailure::FenceLost
21276 );
21277 let fenced = store.pool_meta_write_gate_status().await;
21278 assert!(!fenced.writes_ready);
21279 assert!(fenced.write_blocked);
21280 assert_eq!(fenced.reason, Some("fence_lost"));
21281 assert_eq!(fenced.phase, Some("format_heal"));
21282 assert_eq!(fenced.since_unix_secs, original.since_unix_secs);
21283 }
21284
21285 #[test]
21286 fn pool_meta_read_probe_rejects_missing_runtime_metadata_without_latching() {
21287 let write_state = PoolMetaWriteState {
21288 expected_cluster_id: Some(uuid::Uuid::new_v4()),
21289 identity_initialized: Some(true),
21290 ..Default::default()
21291 };
21292 select_pool_meta_replicas_for_read_probe(&write_state, vec![PoolMetaReplica::Missing], "capacity probe")
21293 .expect_err("runtime metadata disappearance must reject the current probe");
21294 write_state
21295 .ensure_write_safe("ordinary object write")
21296 .expect("a missing-metadata probe must not permanently latch the writer");
21297 }
21298
21299 #[tokio::test]
21300 #[serial_test::serial]
21301 async fn pool_meta_read_guard_does_not_latch_after_unreadable_replica() {
21302 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
21303 let mut saved_disks = Vec::new();
21304 for set in &store.pools[1].disk_set {
21305 let mut disks = set.disks.write().await;
21306 let original = std::mem::take(&mut *disks);
21307 let disk_count = original.len();
21308 saved_disks.push((set.clone(), original));
21309 *disks = vec![None; disk_count];
21310 }
21311
21312 let write_state = store.pool_meta_save_gate.lock().await;
21313 store
21314 .acquire_pool_meta_read_guard(&write_state, "capacity probe")
21315 .await
21316 .expect_err("an unreadable metadata replica must reject this probe");
21317 write_state
21318 .ensure_write_safe("ordinary object write")
21319 .expect("a failed read-only probe must remain retryable");
21320
21321 for (set, disks) in saved_disks {
21322 *set.disks.write().await = disks;
21323 }
21324 store
21325 .acquire_pool_meta_read_guard(&write_state, "capacity probe retry")
21326 .await
21327 .expect("a read-only probe must succeed after the replica recovers");
21328 }
21329
21330 #[test]
21331 fn pool_meta_writer_read_failure_is_retryable_when_all_replicas_are_unreadable() {
21332 let replicas = vec![
21333 PoolMetaReplica::Unreadable(Arc::new(Error::other("pool 0 read quorum unavailable"))),
21334 PoolMetaReplica::Unreadable(Arc::new(Error::other("pool 1 read quorum unavailable"))),
21335 ];
21336 let mut write_state = PoolMetaWriteState::default();
21337
21338 let err = select_pool_meta_replicas_observing(&mut write_state, replicas)
21339 .expect_err("all unreadable replicas must fail selection");
21340 assert_eq!(
21341 err.pool_metadata_failure().unwrap().kind,
21342 crate::error::PoolMetadataFailure::ReadUnavailable
21343 );
21344 write_state
21345 .ensure_write_safe("pool metadata save failed")
21346 .expect("a read-only preflight has no unknown writes and must remain retryable");
21347 }
21348
21349 #[test]
21350 fn pool_meta_write_state_blocks_on_any_recovery_required_selection() {
21351 fn assert_selection_blocks(replicas: Vec<PoolMetaReplica>) {
21352 let mut write_state = PoolMetaWriteState::default();
21353 select_pool_meta_replicas_observing(&mut write_state, replicas)
21354 .expect_err("recovery-required replicas must fail selection");
21355 write_state
21356 .ensure_write_safe("pool metadata save failed")
21357 .expect_err("a recovery-required selection must latch the write block");
21358 }
21359
21360 assert_selection_blocks(vec![PoolMetaReplica::Corrupt("truncated".to_string())]);
21361 assert_selection_blocks(vec![PoolMetaReplica::Incompatible("future format".to_string())]);
21362 assert_selection_blocks(vec![
21363 PoolMetaReplica::Corrupt("canonical unavailable".to_string()),
21364 decode_pool_meta_replica(pool_meta_replica_test_data("pool-old")),
21365 decode_pool_meta_replica(pool_meta_replica_test_data("pool-new")),
21366 ]);
21367 }
21368
21369 #[test]
21370 fn pool_meta_replica_selection_rejects_same_version_tuple_extension() {
21371 #[derive(Serialize)]
21372 struct FuturePersistedPoolMeta {
21373 version: u16,
21374 pools: Vec<PersistedPoolStatus>,
21375 generation: u64,
21376 }
21377
21378 let mut data = Vec::new();
21379 data.write_u16::<LittleEndian>(POOL_META_FORMAT)
21380 .expect("pool metadata format should encode");
21381 data.write_u16::<LittleEndian>(POOL_META_VERSION)
21382 .expect("pool metadata version should encode");
21383 FuturePersistedPoolMeta {
21384 version: POOL_META_VERSION,
21385 pools: Vec::new(),
21386 generation: 2,
21387 }
21388 .serialize(&mut Serializer::new(&mut data))
21389 .expect("extended tuple pool metadata should encode");
21390
21391 let err = select_pool_meta_replica(vec![
21392 decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
21393 decode_pool_meta_replica(data),
21394 ])
21395 .expect_err("same-version tuple extensions must block fallback repair writes");
21396
21397 assert!(err.to_string().contains("pool 1 is incompatible"));
21398 assert!(err.to_string().contains("version 2 tuple has unsupported field count"));
21399 }
21400
21401 #[test]
21402 fn pool_meta_replica_selection_falls_back_from_truncated_current_tuple() {
21403 let mut truncated = pool_meta_replica_test_data("pool-truncated");
21404 truncated.pop();
21405
21406 let selection = select_pool_meta_replica(vec![
21407 decode_pool_meta_replica(truncated),
21408 decode_pool_meta_replica(pool_meta_replica_test_data("pool-valid")),
21409 ])
21410 .expect("a truncated tuple should not block a validated backup replica");
21411
21412 assert!(selection.replica_state.needs_repair);
21413 assert!(selection.replica_state.repair_write_safe);
21414 assert_eq!(selection.meta.pools[0].cmd_line, "pool-valid");
21415 }
21416
21417 #[test]
21418 fn ensure_pool_not_left_in_cmdline_after_decommission_allows_active_pool() {
21419 assert!(ensure_pool_not_left_in_cmdline_after_decommission(0, "http://node{1...4}/disk{1...4}", false).is_ok());
21420 }
21421
21422 #[test]
21423 fn ensure_pool_not_left_in_cmdline_after_decommission_rejects_completed_pool() {
21424 let err = ensure_pool_not_left_in_cmdline_after_decommission(1, "http://node{1...4}/disk{1...4}", true)
21425 .expect_err("completed decommissioned pool should fail validation");
21426
21427 assert!(
21428 err.to_string()
21429 .contains("pool(2) = http://node{1...4}/disk{1...4} is decommissioned, please remove from server command line")
21430 );
21431 }
21432
21433 #[test]
21434 fn determine_decommission_final_state_marks_failures_and_cancellations() {
21435 assert_eq!(determine_decommission_final_state(0, false), DecommissionFinalState::Complete);
21436 assert_eq!(determine_decommission_final_state(1, false), DecommissionFinalState::Failed);
21437 assert_eq!(determine_decommission_final_state(0, true), DecommissionFinalState::Failed);
21438 }
21439
21440 #[test]
21441 fn lifecycle_action_removes_data_movement_version_rejects_delete_marker_action() {
21442 assert!(!lifecycle_action_removes_data_movement_version(IlmAction::DeleteAction));
21443 }
21444
21445 #[test]
21446 fn lifecycle_action_removes_data_movement_version_accepts_version_delete_actions() {
21447 assert!(lifecycle_action_removes_data_movement_version(IlmAction::DeleteVersionAction));
21448 assert!(lifecycle_action_removes_data_movement_version(IlmAction::DeleteAllVersionsAction));
21449 assert!(lifecycle_action_removes_data_movement_version(
21450 IlmAction::DelMarkerDeleteAllVersionsAction
21451 ));
21452 }
21453
21454 #[test]
21455 fn lifecycle_action_skips_heal_version_for_every_delete_action() {
21456 assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAction));
21457 assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteVersionAction));
21458 assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredAction));
21459 assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredVersionAction));
21460 assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAllVersionsAction));
21461 assert!(lifecycle_action_skips_heal_version(IlmAction::DelMarkerDeleteAllVersionsAction));
21462 assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionAction));
21463 assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionVersionAction));
21464 assert!(!lifecycle_action_skips_heal_version(IlmAction::NoneAction));
21465 }
21466
21467 #[test]
21468 fn resolve_data_movement_lifecycle_expiry_result_allows_dry_run_skip() {
21469 let skip = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, false, false)
21470 .expect("dry-run lifecycle evaluation should not require expiry enqueue");
21471
21472 assert!(skip);
21473 }
21474
21475 #[test]
21476 fn resolve_data_movement_lifecycle_expiry_result_rejects_apply_failure() {
21477 let err = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, true, false)
21478 .expect_err("failed lifecycle expiry enqueue should not be treated as skipped");
21479
21480 assert!(err.to_string().contains("failed to apply lifecycle expiry action"));
21481 }
21482
21483 #[test]
21484 fn decommission_copy_cleanup_safe_error_accepts_missing_source_errors() {
21485 assert!(is_decommission_copy_cleanup_safe_error(&Error::ObjectNotFound(
21486 "bucket".to_string(),
21487 "object".to_string()
21488 )));
21489 assert!(is_decommission_copy_cleanup_safe_error(&Error::VersionNotFound(
21490 "bucket".to_string(),
21491 "object".to_string(),
21492 "version".to_string()
21493 )));
21494 }
21495
21496 #[test]
21497 fn decommission_free_version_attempt_treats_missing_source_as_consumed() {
21498 let attempt =
21499 classify_decommission_free_version_attempt(Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())));
21500
21501 assert!(matches!(attempt, DecommissionFreeVersionAttempt::Consumed));
21502 }
21503
21504 #[test]
21505 fn decommission_free_version_attempt_preserves_capacity_failure() {
21506 let attempt = classify_decommission_free_version_attempt(Err(Error::DiskFull));
21507
21508 assert!(matches!(attempt, DecommissionFreeVersionAttempt::CapacityFailure(Error::DiskFull)));
21509 }
21510
21511 #[test]
21512 fn decommission_delete_marker_copy_error_rejects_data_movement_overwrite() {
21513 let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
21514
21515 assert!(!is_decommission_copy_cleanup_safe_error(&err));
21516 }
21517
21518 #[test]
21519 fn decommission_remote_tiered_copy_error_rejects_data_movement_overwrite() {
21520 let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
21521
21522 assert!(!is_decommission_copy_cleanup_safe_error(&err));
21523 }
21524
21525 #[test]
21526 fn decommission_target_capacity_error_accepts_direct_capacity_errors() {
21527 assert!(is_decommission_target_capacity_error(&Error::DiskFull));
21528 assert!(is_decommission_target_capacity_error(&Error::StorageFull));
21529 }
21530
21531 #[test]
21536 fn decommission_classifiers_see_through_a_stage_wrapper() {
21537 let wrap = |inner: Error| {
21538 data_movement::data_movement_stage_error_for_test("decommission_object", "put_object", "bucket-a", "object-a", inner)
21539 };
21540
21541 assert!(is_decommission_target_capacity_error(&wrap(Error::DiskFull)));
21543 assert!(is_decommission_target_capacity_error(&wrap(Error::StorageFull)));
21544 assert!(!is_decommission_target_capacity_error(&wrap(Error::SlowDown)));
21545
21546 let gate_busy = decommission_capacity_blocked_error(format!(
21547 "{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}7{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}"
21548 ));
21549 assert_eq!(decommission_capacity_target_gate_busy_index(&wrap(gate_busy)), Some(7));
21550
21551 let not_found = Error::ObjectNotFound("bucket-a".to_string(), "object-a".to_string());
21555 assert!(is_decommission_copy_cleanup_safe_error(¬_found));
21556 assert!(is_decommission_copy_cleanup_safe_error(&wrap(not_found)));
21557 assert!(!is_decommission_copy_cleanup_safe_error(&wrap(Error::SlowDown)));
21558 }
21559
21560 #[test]
21561 fn decommission_target_gate_retry_recognizes_multipart_part_errors() {
21562 let wrap = |inner: Error| {
21563 data_movement::data_movement_part_stage_error_for_test(
21564 "decommission_object",
21565 "put_object_part",
21566 "bucket-a",
21567 "object-a",
21568 1,
21569 inner,
21570 )
21571 };
21572 let gate_busy_message =
21573 format!("{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}7{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}");
21574 let wrapped = wrap(decommission_capacity_blocked_error(&gate_busy_message));
21575 assert!(is_decommission_capacity_target_gate_busy(&wrapped));
21576 assert_eq!(decommission_capacity_target_gate_busy_index(&wrapped), Some(7));
21577 assert_eq!(
21578 wrapped.to_string(),
21579 format!(
21580 "Io error: decommission_object: put_object_part failed for bucket-a/object-a part 1: {}",
21581 decommission_capacity_blocked_error(&gate_busy_message)
21582 )
21583 );
21584
21585 for unrelated in [
21586 Error::SlowDown,
21587 Error::DiskFull,
21588 decommission_capacity_blocked_error("target capacity is exhausted"),
21589 Error::other(gate_busy_message),
21590 ] {
21591 let wrapped = wrap(unrelated);
21592 assert!(!is_decommission_capacity_target_gate_busy(&wrapped));
21593 assert_eq!(decommission_capacity_target_gate_busy_index(&wrapped), None);
21594 }
21595 for missing_target in [
21596 Error::FileNotFound,
21597 Error::ObjectNotFound("bucket-a".to_string(), "object-a".to_string()),
21598 Error::VersionNotFound("bucket-a".to_string(), "object-a".to_string(), "version-a".to_string()),
21599 ] {
21600 assert!(is_decommission_copy_cleanup_safe_error(&missing_target));
21601 assert!(
21602 !is_decommission_copy_cleanup_safe_error(&wrap(missing_target)),
21603 "a missing target part must never authorize source cleanup"
21604 );
21605 }
21606 assert!(is_decommission_target_capacity_error(&wrap(Error::DiskFull)));
21607 }
21608
21609 #[test]
21610 fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
21611 let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
21612 let storage_full = Error::other(format!(
21613 "decommission_object: put_object failed for bucket/object: {}",
21614 Error::StorageFull
21615 ));
21616
21617 assert!(is_decommission_target_capacity_error(&disk_full));
21618 assert!(is_decommission_target_capacity_error(&storage_full));
21619 }
21620
21621 #[test]
21622 fn decommission_capacity_intent_conflict_accepts_context_wrapped_error() {
21623 let err = with_decommission_entry_context(
21624 "migrate_object",
21625 "bucket",
21626 "object",
21627 decommission_capacity_blocked_error("target has an unresolved target capacity intent"),
21628 );
21629
21630 assert!(is_decommission_capacity_intent_conflict(&err));
21631 assert_eq!(
21632 decommission_capacity_retry_kind(&err, DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS - 1),
21633 Some(DecommissionCapacityRetryKind::IntentConflict)
21634 );
21635 assert_eq!(
21636 decommission_capacity_retry_kind(&err, DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS),
21637 None,
21638 "a durable intent conflict must retain its bounded recovery policy"
21639 );
21640 }
21641
21642 #[test]
21643 fn decommission_target_capacity_error_rejects_unrelated_errors() {
21644 assert!(!is_decommission_target_capacity_error(&Error::SlowDown));
21645 }
21646
21647 #[test]
21648 fn should_skip_decommission_delete_marker_characterizes_empty_marker_without_replication() {
21649 let version = rustfs_filemeta::FileInfo {
21650 deleted: true,
21651 ..Default::default()
21652 };
21653
21654 assert!(should_skip_decommission_delete_marker(&version, 1, false));
21655 }
21656
21657 #[test]
21658 fn should_skip_decommission_delete_marker_characterizes_replication_configured() {
21659 let version = rustfs_filemeta::FileInfo {
21660 deleted: true,
21661 ..Default::default()
21662 };
21663
21664 assert!(!should_skip_decommission_delete_marker(&version, 1, true));
21665 }
21666
21667 #[test]
21668 fn should_skip_decommission_delete_marker_rejects_non_deleted_versions() {
21669 let version = rustfs_filemeta::FileInfo::default();
21670
21671 assert!(!should_skip_decommission_delete_marker(&version, 1, false));
21672 }
21673
21674 #[test]
21675 fn should_skip_decommission_delete_marker_rejects_multiple_remaining_versions() {
21676 let version = rustfs_filemeta::FileInfo {
21677 deleted: true,
21678 ..Default::default()
21679 };
21680
21681 assert!(!should_skip_decommission_delete_marker(&version, 2, false));
21682 }
21683
21684 #[test]
21685 fn decommission_delete_marker_opts_preserves_replication_state() {
21686 let mod_time = OffsetDateTime::now_utc();
21687 let version = rustfs_filemeta::FileInfo {
21688 mod_time: Some(mod_time),
21689 replication_state_internal: Some(crate::bucket::replication::replication_state_to_filemeta(&ReplicationState {
21690 replica_status: ReplicationStatusType::Replica,
21691 delete_marker: true,
21692 replicate_decision_str: "existing".to_string(),
21693 ..Default::default()
21694 })),
21695 ..Default::default()
21696 };
21697
21698 let incarnation = uuid::Uuid::new_v4();
21699 let opts = decommission_delete_marker_opts(&version, Some("version-id".to_string()), 7, Some(incarnation));
21700 let replication = opts.delete_replication.expect("replication state should be preserved");
21701
21702 assert!(opts.versioned);
21703 assert!(opts.data_movement);
21704 assert!(opts.delete_marker);
21705 assert!(opts.skip_decommissioned);
21706 assert_eq!(opts.src_pool_idx, 7);
21707 assert_eq!(opts.version_id.as_deref(), Some("version-id"));
21708 assert_eq!(opts.mod_time, Some(mod_time));
21709 assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
21710 assert_eq!(replication.replica_status, ReplicationStatusType::Replica);
21711 assert!(replication.delete_marker);
21712 assert_eq!(replication.replicate_decision_str, "existing");
21713 }
21714
21715 #[test]
21716 fn decommission_delete_marker_opts_preserves_suspended_null_version() {
21717 let version = rustfs_filemeta::FileInfo {
21718 name: "object".to_string(),
21719 deleted: true,
21720 ..Default::default()
21721 };
21722 let opts = decommission_delete_marker_opts(&version, None, 7, None);
21723
21724 assert!(!opts.versioned);
21725 assert!(opts.version_suspended);
21726 assert_eq!(opts.version_id.as_deref(), Some(uuid::Uuid::nil().to_string().as_str()));
21727
21728 let owner = DecommissionCapacityOwner {
21729 source_pool_index: 7,
21730 operation_id: uuid::Uuid::new_v4(),
21731 generation: 1,
21732 owner_nonce: uuid::Uuid::new_v4(),
21733 mutation_id: None,
21734 };
21735 assert_eq!(
21736 decommission_capacity_version_mutation_id(owner, "bucket", &version),
21737 decommission_capacity_mutation_id(
21738 owner,
21739 "bucket",
21740 &version.name,
21741 opts.version_id.as_deref(),
21742 opts.delete_marker,
21743 opts.mod_time,
21744 )
21745 );
21746 }
21747
21748 #[test]
21749 fn test_decommission_object_migration_read_opts_are_raw_data_movement() {
21750 let opts = decommission_object_migration_read_opts(Some("vid-1".to_string()));
21751
21752 assert_eq!(opts.version_id.as_deref(), Some("vid-1"));
21753 assert!(opts.no_lock);
21754 assert!(opts.data_movement);
21755 assert!(opts.raw_data_movement_read);
21756 assert!(opts.skip_rebalancing);
21757 assert!(opts.skip_decommissioned);
21758 }
21759
21760 #[test]
21761 fn decommission_remote_tiered_opts_preserves_versioning_context() {
21762 let mod_time = OffsetDateTime::now_utc();
21763 let version = rustfs_filemeta::FileInfo {
21764 mod_time: Some(mod_time),
21765 metadata: HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())]),
21766 ..Default::default()
21767 };
21768
21769 let incarnation = uuid::Uuid::new_v4();
21770 let opts = decommission_remote_tiered_opts(&version, Some("version-id".to_string()), 9, Some(incarnation));
21771
21772 assert!(opts.versioned);
21773 assert!(opts.data_movement);
21774 assert_eq!(opts.src_pool_idx, 9);
21775 assert_eq!(opts.version_id.as_deref(), Some("version-id"));
21776 assert_eq!(opts.mod_time, Some(mod_time));
21777 assert_eq!(opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value"));
21778 assert!(opts.include_part_checksums);
21779 assert!(opts.http_preconditions.is_some());
21780 assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
21781 assert!(!opts.incl_free_versions);
21782
21783 let mut free_version = version;
21784 free_version.set_tier_free_version();
21785 let free_opts = decommission_remote_tiered_opts(&free_version, Some("free-version-id".to_string()), 9, Some(incarnation));
21786 assert!(free_opts.incl_free_versions);
21787 }
21788
21789 #[test]
21790 fn decommission_terminal_state_transitions_update_start_time() {
21791 let start_time = OffsetDateTime::now_utc();
21792 let build_pool_meta = || PoolMeta {
21793 version: POOL_META_VERSION,
21794 pools: vec![PoolStatus {
21795 id: 0,
21796 cmd_line: "/tmp/pool".to_string(),
21797 last_update: start_time,
21798 decommission: Some(PoolDecommissionInfo {
21799 start_time: Some(start_time),
21800 ..Default::default()
21801 }),
21802 }],
21803 dont_save: true,
21804 };
21805
21806 let mut pool_meta = build_pool_meta();
21807 assert!(pool_meta.decommission_failed(0));
21808 assert_eq!(pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), None);
21809
21810 let mut pool_meta = build_pool_meta();
21811 assert!(pool_meta.decommission_complete(0));
21812 assert_eq!(
21813 pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time),
21814 Some(start_time)
21815 );
21816
21817 let mut pool_meta = build_pool_meta();
21818 assert!(pool_meta.decommission_cancel(0));
21819 assert_eq!(pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), None);
21820
21821 let mut pool_meta = build_pool_meta();
21822 assert!(pool_meta.decommission_cancel(0));
21823 assert!(!pool_meta.decommission_complete(0));
21824
21825 let mut pool_meta = build_pool_meta();
21826 assert!(pool_meta.decommission_failed(0));
21827 assert!(!pool_meta.decommission_complete(0));
21828 }
21829
21830 #[test]
21831 fn pool_meta_persists_decommission_resume_queues() {
21832 let start_time = OffsetDateTime::now_utc();
21833 let unresolved_entry = DecommissionUnresolvedEntry {
21834 bucket: "bucket-b".to_string(),
21835 object: "prefix/unresolved.txt".to_string(),
21836 pool_index: 0,
21837 set_index: 1,
21838 source_generation: start_time,
21839 candidate_count: 2,
21840 disk_error_count: 1,
21841 observed_at: start_time,
21842 reason: "metadata_resolution_failed".to_string(),
21843 };
21844 let pool_meta = PoolMeta {
21845 version: POOL_META_VERSION,
21846 pools: vec![PoolStatus {
21847 id: 1,
21848 cmd_line: "/data/pool1/disk{1...4}".to_string(),
21849 last_update: start_time,
21850 decommission: Some(PoolDecommissionInfo {
21851 start_time: Some(start_time),
21852 queued: true,
21853 queued_buckets: vec!["bucket-a".to_string(), "bucket-b/prefix".to_string()],
21854 decommissioned_buckets: vec!["bucket-done".to_string()],
21855 bucket: "bucket-b".to_string(),
21856 prefix: "prefix".to_string(),
21857 object: "object.txt".to_string(),
21858 items_decommissioned: 7,
21859 items_decommission_failed: 1,
21860 bytes_done: 1024,
21861 bytes_failed: 128,
21862 terminal_reload_attempt_at: Some(start_time),
21863 terminal_reload_failures: vec!["complete_decommission: peer node-a failed".to_string()],
21864 unresolved_entries: vec![unresolved_entry.clone()],
21865 ..Default::default()
21866 }),
21867 }],
21868 dont_save: false,
21869 };
21870
21871 let mut buf = Vec::new();
21872 PersistedPoolMeta::from(&pool_meta)
21873 .serialize(&mut Serializer::new(&mut buf))
21874 .expect("pool meta should serialize");
21875
21876 let mut deserializer = Deserializer::new(Cursor::new(&buf));
21877 let restored: PoolMeta = PersistedPoolMeta::deserialize(&mut deserializer)
21878 .expect("pool meta should deserialize")
21879 .try_into()
21880 .expect("pool meta should validate");
21881
21882 let restored_decommission = restored.pools[0]
21883 .decommission
21884 .as_ref()
21885 .expect("decommission info should survive round-trip");
21886 assert_eq!(
21887 restored_decommission.queued_buckets,
21888 vec!["bucket-a".to_string(), "bucket-b/prefix".to_string()]
21889 );
21890 assert_eq!(restored_decommission.decommissioned_buckets, vec!["bucket-done".to_string()]);
21891 assert_eq!(restored_decommission.bucket, "bucket-b");
21892 assert_eq!(restored_decommission.prefix, "prefix");
21893 assert_eq!(restored_decommission.object, "object.txt");
21894 assert!(restored_decommission.stage.is_empty());
21895 assert_eq!(restored_decommission.items_decommissioned, 7);
21896 assert_eq!(restored_decommission.items_decommission_failed, 1);
21897 assert_eq!(restored_decommission.bytes_done, 1024);
21898 assert_eq!(restored_decommission.bytes_failed, 128);
21899 assert_eq!(restored_decommission.terminal_reload_attempt_at, Some(start_time));
21900 assert_eq!(
21901 restored_decommission.terminal_reload_failures,
21902 vec!["complete_decommission: peer node-a failed".to_string()]
21903 );
21904 assert_eq!(restored_decommission.unresolved_entries, vec![unresolved_entry]);
21905 assert!(restored_decommission.queued);
21906 assert_eq!(restored_decommission.items_since_last_progress_save(), 0);
21907 }
21908
21909 #[test]
21910 fn pool_meta_records_decommission_terminal_reload_failure_once() {
21911 let start_time = OffsetDateTime::now_utc();
21912 let mut pool_meta = PoolMeta {
21913 version: POOL_META_VERSION,
21914 pools: vec![PoolStatus {
21915 id: 1,
21916 cmd_line: "/data/pool1/disk{1...4}".to_string(),
21917 last_update: start_time,
21918 decommission: Some(PoolDecommissionInfo {
21919 complete: true,
21920 ..Default::default()
21921 }),
21922 }],
21923 dont_save: false,
21924 };
21925
21926 assert!(
21927 pool_meta
21928 .record_decommission_terminal_reload_failure(0, "complete_decommission", "peer node-a failed".to_string())
21929 .expect("terminal reload failure should be recorded")
21930 );
21931 assert!(
21932 !pool_meta
21933 .record_decommission_terminal_reload_failure(0, "complete_decommission", "peer node-a failed".to_string())
21934 .expect("duplicate terminal reload failure should be ignored")
21935 );
21936
21937 let decommission = pool_meta.pools[0].decommission.as_ref().expect("decommission should exist");
21938 assert!(decommission.terminal_reload_attempt_at.is_some());
21939 assert_eq!(
21940 decommission.terminal_reload_failures,
21941 vec!["complete_decommission: peer node-a failed".to_string()]
21942 );
21943 }
21944
21945 #[test]
21946 fn pool_meta_decode_supports_legacy_payload() {
21947 let start_time = OffsetDateTime::now_utc();
21948 let legacy_meta = LegacyPoolMeta {
21949 version: POOL_META_V1_VERSION,
21950 pools: vec![LegacyPoolStatus {
21951 id: 3,
21952 cmd_line: "/legacy/pool".to_string(),
21953 last_update: start_time,
21954 decommission: Some(LegacyPoolDecommissionInfo {
21955 start_time: Some(start_time),
21956 items_decommissioned: 9,
21957 items_decommission_failed: 2,
21958 bytes_done: 2048,
21959 bytes_failed: 256,
21960 ..Default::default()
21961 }),
21962 }],
21963 dont_save: true,
21964 };
21965
21966 let mut legacy_payload = Vec::new();
21967 legacy_meta
21968 .serialize(&mut Serializer::new(&mut legacy_payload))
21969 .expect("legacy payload should serialize");
21970
21971 let persisted_decode: std::result::Result<PersistedPoolMeta, _> = rmp_serde::from_slice(&legacy_payload);
21973 assert!(persisted_decode.is_err());
21974
21975 let decoded =
21976 PoolMeta::decode_pool_meta_payload(POOL_META_V1_VERSION, &legacy_payload).expect("legacy payload should decode");
21977 assert_eq!(decoded.version, POOL_META_V1_VERSION);
21978 assert!(!decoded.dont_save, "runtime-only flag should reset on load");
21979 assert_eq!(decoded.pools.len(), 1);
21980 assert_eq!(decoded.pools[0].id, 3);
21981 assert_eq!(decoded.pools[0].cmd_line, "/legacy/pool");
21982 assert_eq!(decoded.pools[0].last_update, start_time);
21983
21984 let decommission = decoded.pools[0].decommission.as_ref().expect("decommission should decode");
21985 assert_eq!(decommission.start_time, Some(start_time));
21986 assert_eq!(decommission.items_decommissioned, 9);
21987 assert_eq!(decommission.items_decommission_failed, 2);
21988 assert_eq!(decommission.bytes_done, 2048);
21989 assert_eq!(decommission.bytes_failed, 256);
21990 assert_eq!(decommission.items_since_last_progress_save(), 0);
21991 assert!(decommission.queued_buckets.is_empty());
21993 assert!(decommission.decommissioned_buckets.is_empty());
21994 assert!(decommission.bucket.is_empty());
21995 assert!(decommission.prefix.is_empty());
21996 assert!(decommission.object.is_empty());
21997 assert!(decommission.unresolved_entries.is_empty());
21998 }
21999
22000 #[test]
22001 fn pool_meta_decode_rejects_unknown_legacy_fields() {
22002 #[derive(Serialize)]
22003 struct LegacyPoolMetaWithUnknownField {
22004 version: u16,
22005 pools: Vec<LegacyPoolStatus>,
22006 dont_save: bool,
22007 unexpected: bool,
22008 }
22009
22010 let payload = rmp_serde::to_vec_named(&LegacyPoolMetaWithUnknownField {
22011 version: POOL_META_V1_VERSION,
22012 pools: Vec::new(),
22013 dont_save: true,
22014 unexpected: true,
22015 })
22016 .expect("legacy pool metadata with unknown field should serialize");
22017
22018 let err = PoolMeta::decode_pool_meta_payload(POOL_META_V1_VERSION, payload.as_slice())
22019 .expect_err("unknown legacy pool metadata field should fail decode");
22020 let rendered = err.to_string();
22021 assert!(rendered.contains("PoolMeta v1 decode failed for both persisted and legacy formats"));
22022 assert!(rendered.contains("unknown field") || rendered.contains("missing field"));
22023 }
22024
22025 #[test]
22026 fn pool_meta_decode_rejects_unknown_persisted_fields() {
22027 #[derive(Serialize)]
22028 struct PersistedPoolMetaWithUnknownField {
22029 version: u16,
22030 pools: Vec<PersistedPoolStatus>,
22031 unexpected: bool,
22032 }
22033
22034 let payload = rmp_serde::to_vec_named(&PersistedPoolMetaWithUnknownField {
22035 version: POOL_META_VERSION,
22036 pools: Vec::new(),
22037 unexpected: true,
22038 })
22039 .expect("pool metadata with unknown field should serialize");
22040
22041 let err = PoolMeta::decode_pool_meta_payload(POOL_META_VERSION, payload.as_slice())
22042 .expect_err("unknown persisted pool metadata field should fail decode");
22043 let rendered = err.to_string();
22044 assert!(rendered.contains("PoolMeta v2 decode failed"));
22045 assert!(rendered.contains("unknown field") || rendered.contains("missing field"));
22046 }
22047
22048 #[test]
22049 fn pool_meta_decode_rejects_missing_critical_persisted_fields() {
22050 #[derive(Serialize)]
22051 struct PersistedPoolMetaWithoutPools {
22052 version: u16,
22053 }
22054
22055 let payload = rmp_serde::to_vec_named(&PersistedPoolMetaWithoutPools {
22056 version: POOL_META_VERSION,
22057 })
22058 .expect("pool metadata without pools should serialize");
22059
22060 let err = PoolMeta::decode_pool_meta_payload(POOL_META_VERSION, payload.as_slice())
22061 .expect_err("missing persisted pool metadata pools should fail decode");
22062 assert!(err.to_string().contains("PoolMeta v2 decode failed"));
22063 }
22064
22065 #[test]
22066 fn pool_meta_decode_rejects_unknown_decommission_fields() {
22067 #[derive(Serialize)]
22068 struct PersistedPoolStatusWithUnknownDecommission {
22069 #[serde(rename = "id")]
22070 id: usize,
22071 #[serde(rename = "cmdline")]
22072 cmd_line: String,
22073 #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
22074 last_update: OffsetDateTime,
22075 #[serde(rename = "decommissionInfo")]
22076 decommission: Option<PersistedPoolDecommissionInfoWithUnknownField>,
22077 }
22078
22079 #[derive(Serialize)]
22080 struct PersistedPoolDecommissionInfoWithUnknownField {
22081 #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
22082 start_time: Option<OffsetDateTime>,
22083 #[serde(rename = "startSize")]
22084 start_size: usize,
22085 #[serde(rename = "totalSize")]
22086 total_size: usize,
22087 #[serde(rename = "currentSize")]
22088 current_size: usize,
22089 #[serde(rename = "complete")]
22090 complete: bool,
22091 #[serde(rename = "failed")]
22092 failed: bool,
22093 #[serde(rename = "canceled")]
22094 canceled: bool,
22095 #[serde(rename = "queuedBuckets")]
22096 queued_buckets: Vec<String>,
22097 #[serde(rename = "decommissionedBuckets")]
22098 decommissioned_buckets: Vec<String>,
22099 #[serde(rename = "bucket")]
22100 bucket: String,
22101 #[serde(rename = "prefix")]
22102 prefix: String,
22103 #[serde(rename = "object")]
22104 object: String,
22105 #[serde(rename = "objectsDecommissioned")]
22106 items_decommissioned: usize,
22107 #[serde(rename = "objectsDecommissionedFailed")]
22108 items_decommission_failed: usize,
22109 #[serde(rename = "bytesDecommissioned")]
22110 bytes_done: usize,
22111 #[serde(rename = "bytesDecommissionedFailed")]
22112 bytes_failed: usize,
22113 #[serde(rename = "unexpected")]
22114 unexpected: bool,
22115 }
22116
22117 #[derive(Serialize)]
22118 struct PersistedPoolMetaWithUnknownDecommission {
22119 version: u16,
22120 pools: Vec<PersistedPoolStatusWithUnknownDecommission>,
22121 }
22122
22123 let start_time = OffsetDateTime::now_utc();
22124 let payload = rmp_serde::to_vec_named(&PersistedPoolMetaWithUnknownDecommission {
22125 version: POOL_META_VERSION,
22126 pools: vec![PersistedPoolStatusWithUnknownDecommission {
22127 id: 0,
22128 cmd_line: "/data/pool".to_string(),
22129 last_update: start_time,
22130 decommission: Some(PersistedPoolDecommissionInfoWithUnknownField {
22131 start_time: Some(start_time),
22132 start_size: 0,
22133 total_size: 0,
22134 current_size: 0,
22135 complete: false,
22136 failed: false,
22137 canceled: false,
22138 queued_buckets: Vec::new(),
22139 decommissioned_buckets: Vec::new(),
22140 bucket: String::new(),
22141 prefix: String::new(),
22142 object: String::new(),
22143 items_decommissioned: 0,
22144 items_decommission_failed: 0,
22145 bytes_done: 0,
22146 bytes_failed: 0,
22147 unexpected: true,
22148 }),
22149 }],
22150 })
22151 .expect("pool metadata with unknown decommission field should serialize");
22152
22153 let err = PoolMeta::decode_pool_meta_payload(POOL_META_VERSION, payload.as_slice())
22154 .expect_err("unknown persisted decommission metadata field should fail decode");
22155 assert!(err.to_string().contains("PoolMeta v2 decode failed"));
22156 }
22157
22158 #[test]
22159 fn pool_meta_decode_rejects_invalid_decommission_terminal_state() {
22160 let start_time = OffsetDateTime::now_utc();
22161 let persisted_meta = PersistedPoolMeta {
22162 version: POOL_META_VERSION,
22163 pools: vec![PersistedPoolStatus {
22164 id: 1,
22165 cmd_line: "/data/pool1/disk{1...4}".to_string(),
22166 last_update: start_time,
22167 decommission: Some(PersistedPoolDecommissionInfo {
22168 start_time: Some(start_time),
22169 complete: true,
22170 failed: true,
22171 canceled: false,
22172 ..Default::default()
22173 }),
22174 }],
22175 };
22176
22177 let mut payload = Vec::new();
22178 persisted_meta
22179 .serialize(&mut Serializer::new(&mut payload))
22180 .expect("persisted payload should serialize");
22181
22182 let err = PoolMeta::decode_pool_meta_payload(POOL_META_VERSION, &payload)
22183 .expect_err("invalid terminal state should fail decode");
22184 assert!(err.to_string().contains("invalid decommission terminal state"));
22185 }
22186
22187 #[test]
22188 fn pool_meta_decode_rejects_invalid_legacy_decommission_terminal_state() {
22189 let start_time = OffsetDateTime::now_utc();
22190 let legacy_meta = LegacyPoolMeta {
22191 version: POOL_META_V1_VERSION,
22192 pools: vec![LegacyPoolStatus {
22193 id: 1,
22194 cmd_line: "/legacy/pool".to_string(),
22195 last_update: start_time,
22196 decommission: Some(LegacyPoolDecommissionInfo {
22197 start_time: Some(start_time),
22198 complete: true,
22199 failed: false,
22200 canceled: true,
22201 ..Default::default()
22202 }),
22203 }],
22204 dont_save: false,
22205 };
22206
22207 let mut payload = Vec::new();
22208 legacy_meta
22209 .serialize(&mut Serializer::new(&mut payload))
22210 .expect("legacy payload should serialize");
22211
22212 let err = PoolMeta::decode_pool_meta_payload(POOL_META_V1_VERSION, &payload)
22213 .expect_err("invalid legacy terminal state should fail decode");
22214 assert!(err.to_string().contains("invalid decommission terminal state"));
22215 }
22216}
22217
22218pub type ListCallback = Arc<dyn Fn(MetaCacheEntry) -> BoxFuture<'static, ()> + Send + Sync + 'static>;
22221
22222const DECOMMISSION_ENTRY_QUEUE_HARD_CAP: usize = 256;
22223
22224struct QueuedDecommissionEntry {
22225 entry: MetaCacheEntry,
22226 queue_permit: OwnedSemaphorePermit,
22227}
22228
22229enum DecommissionEntryEnqueueResult {
22230 Enqueued,
22231 Canceled,
22232 Closed,
22233}
22234
22235fn decommission_entry_queue_capacity(worker_limit: usize) -> usize {
22236 worker_limit.saturating_mul(2).clamp(1, DECOMMISSION_ENTRY_QUEUE_HARD_CAP)
22237}
22238
22239async fn enqueue_decommission_entry(
22240 rx: &CancellationToken,
22241 outstanding: &Arc<Semaphore>,
22242 tx: &mpsc::Sender<QueuedDecommissionEntry>,
22243 entry: MetaCacheEntry,
22244) -> DecommissionEntryEnqueueResult {
22245 let queue_permit = match tokio::select! {
22246 biased;
22247 _ = rx.cancelled() => return DecommissionEntryEnqueueResult::Canceled,
22248 permit = outstanding.clone().acquire_owned() => permit,
22249 } {
22250 Ok(permit) => permit,
22251 Err(_) => return DecommissionEntryEnqueueResult::Closed,
22252 };
22253
22254 let queued = QueuedDecommissionEntry { entry, queue_permit };
22255 tokio::select! {
22256 biased;
22257 _ = rx.cancelled() => DecommissionEntryEnqueueResult::Canceled,
22258 result = tx.send(queued) => {
22259 if result.is_ok() {
22260 DecommissionEntryEnqueueResult::Enqueued
22261 } else {
22262 DecommissionEntryEnqueueResult::Closed
22263 }
22264 }
22265 }
22266}
22267
22268async fn drain_decommission_entry_queue(rx: &CancellationToken, outstanding: &Arc<Semaphore>, capacity: usize) -> bool {
22269 let Ok(permits) = u32::try_from(capacity) else {
22270 return true;
22271 };
22272
22273 tokio::select! {
22274 _ = rx.cancelled() => true,
22275 result = outstanding.acquire_many(permits) => result.is_err(),
22276 }
22277}
22278
22279async fn record_decommission_entry_error(
22280 entry_error: &Arc<tokio::sync::Mutex<Option<Error>>>,
22281 rx: &CancellationToken,
22282 err: Error,
22283) -> bool {
22284 if rx.is_cancelled() {
22285 return false;
22286 }
22287
22288 let mut first_err = entry_error.lock().await;
22289 if first_err.is_none() && !rx.is_cancelled() {
22290 *first_err = Some(err);
22291 rx.cancel();
22292 return true;
22293 }
22294 false
22295}
22296
22297fn ensure_decommission_unresolved_verification_disk_count(
22298 expected: usize,
22299 actual: usize,
22300 pool_index: usize,
22301 set_index: usize,
22302) -> Result<()> {
22303 if actual == expected {
22304 return Ok(());
22305 }
22306 Err(Error::other(format!(
22307 "decommission unresolved-entry verification for pool {pool_index} set {set_index} requires all {expected} source disks, but only {actual} are online"
22308 )))
22309}
22310
22311impl SetDisks {
22312 async fn decommission_unresolved_entry_absent_on_all_disks(
22313 &self,
22314 pool_index: usize,
22315 entry: &DecommissionUnresolvedEntry,
22316 ) -> Result<bool> {
22317 let (disks, _) = self.get_online_disks_with_healing(false).await;
22318 ensure_decommission_unresolved_verification_disk_count(self.set_drive_count, disks.len(), pool_index, entry.set_index)?;
22319
22320 let object = encode_dir_object(&entry.object);
22321 let reads = join_all(disks.iter().map(|disk| disk.read_xl(&entry.bucket, &object, false))).await;
22322 let mut found = false;
22323 for read in reads {
22324 match read {
22325 Ok(_) => found = true,
22326 Err(DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound) => {}
22327 Err(err) => {
22328 return Err(Error::other(format!(
22329 "decommission unresolved-entry verification failed for pool {pool_index} set {} path {}/{}: {err}",
22330 entry.set_index, entry.bucket, entry.object
22331 )));
22332 }
22333 }
22334 }
22335 Ok(!found)
22336 }
22337
22338 #[tracing::instrument(skip(self, store, rx, cb_func, entry_error))]
22339 #[allow(clippy::too_many_arguments)]
22340 async fn list_objects_to_decommission(
22341 self: &Arc<Self>,
22342 store: Arc<ECStore>,
22343 rx: CancellationToken,
22344 bucket_info: DecomBucketInfo,
22345 cb_func: ListCallback,
22346 entry_error: Arc<tokio::sync::Mutex<Option<Error>>>,
22347 pool_index: usize,
22348 set_index: usize,
22349 source_generation: OffsetDateTime,
22350 require_all_disks: bool,
22351 ) -> Result<()> {
22352 let (disks, _) = self.get_online_disks_with_healing(false).await;
22353 ensure_decommission_listing_disks_available(!disks.is_empty(), &bucket_info.name)?;
22354 if require_all_disks {
22355 ensure_decommission_unresolved_verification_disk_count(self.set_drive_count, disks.len(), pool_index, set_index)?;
22356 }
22357
22358 let listing_quorum = self.set_drive_count.div_ceil(2);
22359
22360 let resolver = MetadataResolutionParams {
22361 dir_quorum: listing_quorum,
22362 obj_quorum: listing_quorum,
22363 bucket: bucket_info.name.clone(),
22364 ..Default::default()
22365 };
22366
22367 let cb1 = cb_func.clone();
22368 let unresolved_error = entry_error.clone();
22369 let unresolved_rx = rx.clone();
22370 let unresolved_bucket = bucket_info.name.clone();
22371 let unresolved_prefix = bucket_info.prefix.clone();
22372 let unresolved_pool_index = pool_index;
22373 let unresolved_set_index = set_index;
22374 let unresolved_generation = source_generation;
22375 let unresolved_store = store;
22376
22377 list_path_raw(
22378 rx,
22379 ListPathRawOptions {
22380 disks: disks.iter().cloned().map(Some).collect(),
22381 bucket: bucket_info.name.clone(),
22382 path: bucket_info.prefix.clone(),
22383 recursive: true,
22384 min_disks: listing_quorum,
22385 skip_walkdir_total_timeout: true,
22386 walkdir_stall_timeout: Some(DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT),
22387 agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
22388 partial: Some(Box::new(move |entries: MetaCacheEntries, errs: &[Option<DiskError>]| {
22389 let resolver = resolver.clone();
22390 let cb_func = cb_func.clone();
22391 let bucket = unresolved_bucket.clone();
22392 let prefix = unresolved_prefix.clone();
22393 let unresolved_error = unresolved_error.clone();
22394 let unresolved_rx = unresolved_rx.clone();
22395 let unresolved_store = unresolved_store.clone();
22396 let pool_index = unresolved_pool_index;
22397 let set_index = unresolved_set_index;
22398 let source_generation = unresolved_generation;
22399 let disk_error_count = errs.iter().flatten().count();
22400 if unresolved_rx.is_cancelled() {
22401 return Box::pin(async {});
22402 }
22403
22404 match resolve_decommission_partial_listing_entry(
22405 entries,
22406 resolver,
22407 &bucket,
22408 &prefix,
22409 disk_error_count,
22410 pool_index,
22411 set_index,
22412 source_generation,
22413 ) {
22414 Ok(entry) => {
22415 warn!("decommission_pool: list_objects_to_decommission get {}", &entry.name);
22416 Box::pin(async move {
22417 cb_func(entry).await;
22418 })
22419 }
22420 Err(unresolved_entry) => Box::pin(async move {
22421 if unresolved_rx.is_cancelled() {
22422 return;
22423 }
22424 let err = decommission_unresolved_listing_error(&unresolved_entry);
22425 warn!(
22426 event = EVENT_DECOMMISSION_BUCKET,
22427 component = LOG_COMPONENT_ECSTORE,
22428 subsystem = LOG_SUBSYSTEM_POOLS,
22429 bucket = %bucket,
22430 prefix = %prefix,
22431 state = "unresolved_entry",
22432 error = %err,
22433 "Decommission listing failed closed on unresolved metadata"
22434 );
22435 let err = match unresolved_store
22436 .persist_decommission_unresolved_entry(pool_index, source_generation, unresolved_entry)
22437 .await
22438 {
22439 Ok(()) => err,
22440 Err(ledger_err) => Error::other(format!("{err}; {ledger_err}")),
22441 };
22442 record_decommission_entry_error(&unresolved_error, &unresolved_rx, err).await;
22443 }),
22444 }
22445 })),
22446 ..Default::default()
22447 },
22448 )
22449 .await?;
22450
22451 if let Some(err) = entry_error.lock().await.clone() {
22452 return Err(err);
22453 }
22454
22455 Ok(())
22456 }
22457}
22458
22459fn is_disk_online_state(state: &str) -> bool {
22460 let s = state.trim().to_lowercase();
22465 if s.is_empty() {
22466 return true;
22467 }
22468 if s.contains("offline") {
22469 return false;
22470 }
22471 if s.contains("not found") || s.contains("disk not found") {
22472 return false;
22473 }
22474 true
22475}
22476
22477fn decommission_physical_pool_capacity(
22478 disks: &[rustfs_madmin::Disk],
22479 pool_index: usize,
22480 layout: DecommissionErasureLayout,
22481 logical: PoolSpaceInfo,
22482) -> (usize, usize, usize) {
22483 let width = layout.width();
22484 let mut sets: HashMap<i32, Vec<&rustfs_madmin::Disk>> = HashMap::new();
22485 let mut seen = HashSet::new();
22486 for disk in disks {
22487 let state = disk.state.trim().to_ascii_lowercase();
22488 if disk.pool_index != pool_index as i32
22489 || disk.set_index < 0
22490 || disk.disk_index < 0
22491 || disk.disk_index as usize >= width
22492 || !matches!(state.as_str(), "ok" | "online")
22493 {
22494 continue;
22495 }
22496 let identity = (disk.set_index, disk.disk_index);
22497 if seen.insert(identity) {
22498 sets.entry(disk.set_index).or_default().push(disk);
22499 }
22500 }
22501
22502 let mut physical_total = 0usize;
22503 let mut physical_free = 0usize;
22504 let mut physical_used = 0usize;
22505 let mut observed_set = false;
22506 for set_disks in sets.values() {
22507 if set_disks.is_empty() {
22508 continue;
22509 }
22510 observed_set = true;
22511 let min_total = set_disks
22512 .iter()
22513 .map(|disk| disk.total_space as usize)
22514 .min()
22515 .unwrap_or_default();
22516 let min_free = set_disks
22517 .iter()
22518 .map(|disk| disk.available_space as usize)
22519 .min()
22520 .unwrap_or_default();
22521 let max_used = set_disks
22522 .iter()
22523 .map(|disk| (disk.used_space as usize).max((disk.total_space as usize).saturating_sub(disk.available_space as usize)))
22524 .max()
22525 .unwrap_or_default();
22526 physical_total = physical_total.saturating_add(min_total.saturating_mul(width));
22527 physical_used = physical_used.saturating_add(max_used.saturating_mul(width));
22528 if set_disks.len() >= width {
22529 physical_free = physical_free.saturating_add(min_free.saturating_mul(width));
22530 }
22531 }
22532
22533 if observed_set {
22534 return (physical_total, physical_free, physical_used);
22535 }
22536
22537 let fallback_total = capacity_mul_div_ceil(logical.total, width, layout.data);
22538 let fallback_used = capacity_mul_div_ceil(logical.used, width, layout.data)
22539 .max(fallback_total.saturating_sub(capacity_mul_div_ceil(logical.free, width, layout.data)));
22540 (fallback_total, 0, fallback_used)
22541}
22542
22543#[deprecated(since = "0.1.0", note = "Use fallback_total_capacity_dedup instead")]
22544#[allow(
22545 dead_code,
22546 reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
22547)]
22548fn fallback_total_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
22549 fallback_total_capacity_dedup(disks)
22550}
22551
22552#[deprecated(since = "0.1.0", note = "Use fallback_free_capacity_dedup instead")]
22553#[allow(
22554 dead_code,
22555 reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
22556)]
22557fn fallback_free_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
22558 fallback_free_capacity_dedup(disks)
22559}
22560
22561pub fn get_total_usable_capacity(disks: &[rustfs_madmin::Disk], info: &rustfs_madmin::StorageInfo) -> usize {
22562 if info.backend.standard_sc_data.is_empty() {
22564 return fallback_total_capacity_dedup(disks);
22565 }
22566 let mut capacity = 0usize;
22567 let mut matched_any = false;
22568 let mut counted_disks: HashSet<String> = HashSet::new();
22569
22570 for disk in disks.iter() {
22571 if disk.pool_index < 0 {
22572 continue;
22573 }
22574 let pool_idx = disk.pool_index as usize;
22575 if info.backend.standard_sc_data.len() <= pool_idx {
22576 continue;
22577 }
22578
22579 let usable_disks_per_set = info.backend.standard_sc_data[pool_idx];
22580 if usable_disks_per_set == 0 {
22581 continue;
22582 }
22583
22584 if (disk.disk_index as usize) < usable_disks_per_set {
22585 let disk_key = format!(
22587 "{}|{}|p{}s{}d{}",
22588 disk.endpoint, disk.drive_path, disk.pool_index, disk.set_index, disk.disk_index );
22594 debug!("get_total_usable_capacity disk_key: {}", disk_key);
22595 if counted_disks.insert(disk_key) {
22597 matched_any = true;
22598 capacity += disk.total_space as usize;
22599 } else {
22600 warn!(
22602 "Duplicate disk detected in capacity calculation: {} at {}",
22603 disk.endpoint, disk.drive_path
22604 );
22605 }
22606 }
22607 }
22608
22609 if matched_any {
22610 capacity
22611 } else {
22612 fallback_total_capacity_dedup(disks)
22615 }
22616}
22617
22618pub fn get_total_usable_capacity_free(disks: &[rustfs_madmin::Disk], info: &rustfs_madmin::StorageInfo) -> usize {
22619 if info.backend.standard_sc_data.is_empty() {
22620 return fallback_free_capacity_dedup(disks);
22621 }
22622
22623 let mut capacity = 0usize;
22624 let mut matched_any = false;
22625 let mut counted_disks: HashSet<String> = HashSet::new();
22626
22627 for disk in disks.iter() {
22628 if disk.pool_index < 0 {
22629 continue;
22630 }
22631 let pool_idx = disk.pool_index as usize;
22632 if info.backend.standard_sc_data.len() <= pool_idx {
22633 continue;
22634 }
22635
22636 let usable_disks_per_set = info.backend.standard_sc_data[pool_idx];
22637 if usable_disks_per_set == 0 {
22638 continue;
22639 }
22640
22641 if (disk.disk_index as usize) < usable_disks_per_set {
22642 let disk_key = format!(
22643 "{}|{}|p{}s{}d{}",
22644 disk.endpoint, disk.drive_path, disk.pool_index, disk.set_index, disk.disk_index
22645 );
22646
22647 if counted_disks.insert(disk_key) {
22648 matched_any = true;
22649 capacity += disk.available_space as usize;
22650 }
22651 }
22652 }
22653
22654 if matched_any {
22655 capacity
22656 } else {
22657 fallback_free_capacity_dedup(disks)
22658 }
22659}
22660
22661pub(crate) fn fallback_total_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usize {
22665 let mut counted_disks: HashSet<String> = HashSet::new();
22666 let mut total = 0usize;
22667
22668 for disk in disks.iter() {
22669 if !is_disk_online_state(&disk.state) {
22671 continue;
22672 }
22673
22674 let disk_key = format!("{}|{}", disk.endpoint, disk.drive_path);
22676
22677 if counted_disks.insert(disk_key) {
22679 total += disk.total_space as usize;
22680 }
22681 }
22682
22683 total
22684}
22685
22686pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usize {
22690 let mut counted_disks: HashSet<String> = HashSet::new();
22691 let mut total = 0usize;
22692
22693 for disk in disks.iter() {
22694 if !is_disk_online_state(&disk.state) {
22695 continue;
22696 }
22697
22698 let disk_key = format!("{}|{}", disk.endpoint, disk.drive_path);
22699
22700 if counted_disks.insert(disk_key) {
22701 total += disk.available_space as usize;
22702 }
22703 }
22704
22705 total
22706}
22707
22708#[cfg(test)]
22709mod pools_tests {
22710 use super::DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
22711 use super::persist_v3_pool_meta_for_test;
22712 use super::record_decommission_entry_error;
22713 use super::resolve_decommission_listing_error;
22714 use super::resolve_decommission_partial_listing_entry;
22715 use super::{
22716 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, DECOMMISSION_CAPACITY_MODEL_VERSION, DECOMMISSION_CAPACITY_RELEASE_CANCELED,
22717 DECOMMISSION_CAPACITY_RELEASE_COMPLETED, DECOMMISSION_CAPACITY_RELEASE_FAILED, DECOMMISSION_CAPACITY_RESERVATION_TTL,
22718 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
22719 DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP, DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP, DECOMMISSION_ENTRY_QUEUE_HARD_CAP,
22720 DECOMMISSION_META_PREFIXES, DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
22721 DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT, DecomBucketInfo, DecommissionCanceler, DecommissionCapacityTarget,
22722 DecommissionDurableIlmReceipt, DecommissionEntryEnqueueResult, DecommissionErasureLayout, DecommissionPoolCapacityInfo,
22723 DecommissionStartPoolState, DecommissionTargetConsumption, DecommissionTerminalState, DecommissionUnresolvedEntry,
22724 ListCallback, POOL_META_GENERATION_VERSION, POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_V1_VERSION,
22725 POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolMetaCasToken, PoolMetaPersistenceFence, PoolSpaceInfo, PoolStatus,
22726 QueuedDecommissionEntry, REBAL_META_NAME, acquire_pool_rebalance_activation_locks, active_decommission_source_indices,
22727 apply_decommission_status_space_info, await_decommission_worker, bind_decommission_cancelers,
22728 bind_missing_decommission_cancelers, build_decommission_capacity_reservation,
22729 build_decommission_capacity_reservation_with_model, cancel_decommission_canceler, clamp_decommission_entry_concurrency,
22730 classify_decommission_terminal_state, count_decommission_item, decommission_cancel_signal_result,
22731 decommission_durable_ilm_receipt_path, decommission_durable_ilm_receipt_run_prefix,
22732 decommission_durable_ilm_receipt_run_token, decommission_entry_queue_capacity, decommission_item_size,
22733 decommission_meta_bucket_options, decommission_physical_pool_capacity, decommission_retry_backoff_delay,
22734 decommission_start_pool_state, decommission_unresolved_listing_error, dedup_indices,
22735 default_decommission_bucket_concurrency, default_decommission_entry_concurrency, drain_decommission_entry_queue,
22736 enqueue_decommission_entry, ensure_decommission_cancel_allowed, ensure_decommission_capacity_reservations_available,
22737 ensure_decommission_clear_allowed, ensure_decommission_generation, ensure_decommission_listing_disks_available,
22738 ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool,
22739 ensure_decommission_start_local_leader, ensure_decommission_start_pool_states,
22740 ensure_decommission_start_rebalance_meta_allowed, ensure_decommission_start_target_capacity,
22741 ensure_decommission_terminal_operation_supported, ensure_decommission_unresolved_verification_disk_count,
22742 ensure_local_decommission_pool_leaders, ensure_pool_meta_write_fence, ensure_valid_decommission_pool_index, get_by_index,
22743 guard_decommission_cancelers, has_active_decommission_canceler, is_decommission_active, is_decommission_cancel_requested,
22744 load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done,
22745 merge_decommission_durable_ilm_receipts, merge_pool_meta_updates_for_save, merge_pool_status_refresh,
22746 missing_decommission_worker_prefix, next_decommission_capacity_generation, observe_decommission_terminal_reload_result,
22747 parse_decommission_durable_ilm_receipt_path, pool_meta_has_active_decommission, publish_pool_meta_updates,
22748 read_pool_meta_replica, reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion,
22749 record_decommission_unresolved_entry, recover_decommission_capacity_reservations,
22750 renew_decommission_capacity_reservation, require_decommission_store, reserve_decommission_start_cancelers,
22751 reserve_decommission_start_target_capacity, resolve_decommission_bucket_state,
22752 resolve_decommission_check_after_list_result, resolve_decommission_entry_cleanup_delete_result,
22753 resolve_decommission_entry_exact_versions, resolve_decommission_entry_reload_result,
22754 resolve_decommission_listing_worker_result, resolve_decommission_optional_bucket_config_result,
22755 resolve_decommission_pool_meta_reload_result, resolve_decommission_preflight_heal_result,
22756 resolve_decommission_progress_save_result, resolve_decommission_terminal_mark_after_error_result,
22757 resolve_decommission_terminal_mark_result, resolve_decommission_update_after_result,
22758 resolve_start_decommission_pool_meta_reload_result, resumable_decommission_queue_indices,
22759 rollback_start_decommission_pool_meta, run_decommission_buckets_bounded, run_decommission_listing_with_retry,
22760 run_decommission_listing_with_retry_and_drain, run_decommission_phases, run_decommission_side_effect,
22761 save_pool_meta_object_cas, select_decommission_capacity_model, should_cleanup_decommission_source_entry,
22762 should_continue_decommission_queue, should_count_decommission_version_complete,
22763 should_fail_decommission_pool_after_exhausted_source_changed, should_preserve_decommission_canceled_state,
22764 should_reject_decommission_cancel_as_terminal, should_retry_decommission_cancel_reload,
22765 should_retry_decommission_listing, should_skip_canceled_decommission_routine, spawn_decommission_index_cancelers,
22766 split_decommission_buckets, take_and_cancel_decommission_canceler, take_decommission_canceler,
22767 track_decommission_current_object, track_decommission_current_object_stage, update_decommission_for_operation,
22768 validate_start_decommission_request, wait_decommission_retry_backoff, wait_decommission_worker_drain,
22769 with_decommission_entry_context,
22770 };
22771 use super::{
22772 DecommissionCapacityAdmission, DecommissionCapacityOwner, DecommissionCapacityReleaseProof,
22773 DecommissionCapacityReservation, DecommissionCapacityTemporaryMutation, decommission_capacity_mutation_id,
22774 ensure_decommission_target_owner_admission, ensure_exact_delete_capacity_namespace_fences,
22775 ensure_external_decommission_target_admission, is_decommission_capacity_blocked_error,
22776 plan_exact_delete_capacity_reconciliations, record_decommission_target_consumption, release_decommission_target_inflight,
22777 reserve_decommission_target_pending, resolve_decommission_target_pending,
22778 set_decommission_capacity_info_overrides_for_test,
22779 };
22780 use crate::bucket::lifecycle::{
22781 DurableIlmRecordCheckpoint,
22782 bucket_lifecycle_ops::{ManualTransitionQueueSnapshot, ManualTransitionRunOptions},
22783 manual_transition_job::{ManualTransitionJobRecord, manual_transition_job_record_object_name},
22784 validate_durable_ilm_record,
22785 };
22786 use crate::bucket::metadata_sys;
22787 use crate::data_movement;
22788 use crate::disk::{STORAGE_FORMAT_FILE, endpoint::Endpoint};
22789 use crate::error::{Error, StorageError};
22790 use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
22791 use crate::object_api::{ObjectInfo, ObjectOptions};
22792 use crate::runtime::instance::InstanceContext;
22793 use crate::services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats};
22794 use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
22795 use crate::storage_api_contracts::object::HTTPPreconditions;
22796 use crate::storage_api_contracts::{object::ObjectIO, range::HTTPRangeSpec};
22797 use crate::store::ECStore;
22798 use byteorder::{ByteOrder, LittleEndian};
22799 use rmp_serde::Serializer;
22800 use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntry, ObjectPartInfo};
22801 use rustfs_filemeta::{MetaCacheEntries, MetadataResolutionParams};
22802 use rustfs_lock::{GlobalLockManager, LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
22803 use rustfs_rio::Index;
22804 use serde::Serialize;
22805 use std::future::Future;
22806 use std::io::Cursor;
22807 use std::sync::{
22808 Arc, Mutex as StdMutex,
22809 atomic::{AtomicBool, AtomicUsize, Ordering},
22810 };
22811 use std::task::{Context, Poll};
22812 use std::time::Duration as StdDuration;
22813 use time::{Duration, OffsetDateTime};
22814 use tokio::io::AsyncReadExt;
22815 use tokio::sync::Semaphore;
22816 use tokio_util::sync::CancellationToken;
22817
22818 #[derive(Debug)]
22819 struct ActivationLockRecorder {
22820 lock_manager: Arc<rustfs_lock::GlobalLockManager>,
22821 owner: &'static str,
22822 resources: StdMutex<Vec<String>>,
22823 }
22824
22825 #[async_trait::async_trait]
22826 impl crate::storage_api_contracts::namespace::NamespaceLocking for ActivationLockRecorder {
22827 type Error = Error;
22828 type NamespaceLock = rustfs_lock::NamespaceLockWrapper;
22829
22830 async fn new_ns_lock(&self, bucket: &str, object: &str) -> crate::error::Result<Self::NamespaceLock> {
22831 self.resources
22832 .lock()
22833 .expect("activation lock recorder should not be poisoned")
22834 .push(object.to_string());
22835 Ok(rustfs_lock::NamespaceLockWrapper::new(
22836 rustfs_lock::NamespaceLock::with_local_manager(
22837 "activation-lock-test".to_string(),
22838 Arc::clone(&self.lock_manager),
22839 ),
22840 rustfs_lock::ObjectKey::new(bucket, object),
22841 self.owner.to_string(),
22842 ))
22843 }
22844 }
22845
22846 fn noop_decommission_list_callback() -> ListCallback {
22847 Arc::new(|_| Box::pin(async {}))
22848 }
22849
22850 fn decommission_worker_test_store(pool_meta: PoolMeta, cancelers: Vec<Option<DecommissionCanceler>>) -> Arc<ECStore> {
22851 let ctx = Arc::new(InstanceContext::new());
22852 let endpoint_pools = EndpointServerPools::default();
22853 Arc::new(ECStore {
22854 id: uuid::Uuid::new_v4(),
22855 disk_map: std::collections::HashMap::new(),
22856 pools: Vec::new(),
22857 peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
22858 pool_meta: tokio::sync::RwLock::new(pool_meta),
22859 rebalance_meta: tokio::sync::RwLock::new(None),
22860 decommission_cancelers: tokio::sync::RwLock::new(cancelers),
22861 start_gate: tokio::sync::Mutex::new(()),
22862 pool_meta_save_gate: tokio::sync::Mutex::new(super::PoolMetaWriteState::for_test_bootstrap()),
22863 ctx,
22864 bucket_fence_registry: Arc::default(),
22865 })
22866 }
22867
22868 #[derive(Debug, Default)]
22869 struct PartialPoolMetaWriteStorage {
22870 fail_write: bool,
22871 fail_after_first_write: bool,
22872 pending_write: bool,
22873 write_started: tokio::sync::Notify,
22874 wrote: AtomicBool,
22875 revision: AtomicUsize,
22876 stored: StdMutex<Option<(Vec<u8>, String)>>,
22877 identity: StdMutex<Option<(Vec<u8>, String)>>,
22878 }
22879
22880 #[tokio::test]
22881 async fn pool_meta_recovery_reconciles_prepare_and_partial_commit_without_format_downgrade() {
22882 for previous_version in [POOL_META_V1_VERSION, POOL_META_VERSION, POOL_META_GENERATION_VERSION] {
22883 for commit_first in [false, true] {
22884 let cluster_id = uuid::Uuid::new_v4();
22885 let previous_meta = PoolMeta {
22886 version: previous_version,
22887 pools: vec![decommission_test_pool_status(0, None)],
22888 ..Default::default()
22889 };
22890 let previous_revision = if previous_version == POOL_META_GENERATION_VERSION {
22891 super::PoolMetaRevision {
22892 version: previous_version,
22893 cluster_id: Some(cluster_id),
22894 epoch: 1,
22895 generation: 9,
22896 transaction_id: Some(uuid::Uuid::new_v4()),
22897 }
22898 } else {
22899 super::PoolMetaRevision::legacy(previous_version)
22900 };
22901 let previous_data = if previous_revision.is_generation_protocol() {
22902 super::encode_pool_meta_v3_envelope(&previous_meta, previous_revision, true, None).unwrap()
22903 } else {
22904 previous_meta.encode_config_data_for_v2_gate(true).unwrap()
22905 };
22906 let previous = super::PoolMetaCommittedCandidate {
22907 canonical: previous_data,
22908 meta: previous_meta.clone(),
22909 revision: previous_revision,
22910 };
22911 let mut next = previous_meta.clone();
22912 next.version = POOL_META_GENERATION_VERSION;
22913 next.pools[0].last_update += Duration::seconds(1);
22914 let revision = super::PoolMetaRevision {
22915 version: POOL_META_GENERATION_VERSION,
22916 cluster_id: Some(cluster_id),
22917 epoch: 1,
22918 generation: previous_revision.generation + 1,
22919 transaction_id: Some(uuid::Uuid::new_v4()),
22920 };
22921 let pending = super::encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous)).unwrap();
22922 let durable = super::encode_pool_meta_v3_envelope(&next, revision, true, None).unwrap();
22923 let identity = super::initialized_pool_meta_identity_for_test(cluster_id, 1).unwrap();
22924 let pools = [if commit_first { durable } else { pending.clone() }, pending]
22925 .into_iter()
22926 .map(|data| {
22927 Arc::new(PartialPoolMetaWriteStorage {
22928 stored: StdMutex::new(Some((data, "initial".to_owned()))),
22929 identity: StdMutex::new(Some((identity.clone(), "identity".to_owned()))),
22930 ..Default::default()
22931 })
22932 })
22933 .collect::<Vec<_>>();
22934 let mut state = super::PoolMetaWriteState::for_startup(cluster_id, false);
22935 let selection = super::load_pool_meta_for_transaction_recovery(pools.clone(), &mut state)
22936 .await
22937 .unwrap();
22938 assert_eq!(
22939 selection.meta.pools[0].last_update,
22940 if commit_first {
22941 next.pools[0].last_update
22942 } else {
22943 previous_meta.pools[0].last_update
22944 }
22945 );
22946 let outcome = super::repair_pool_meta_transaction(
22947 pools.clone(),
22948 &mut state,
22949 selection,
22950 &PoolMetaPersistenceFence::Distributed(None),
22951 )
22952 .await
22953 .unwrap();
22954 assert_eq!(outcome.committed.version, POOL_META_GENERATION_VERSION);
22955 assert_eq!(
22956 outcome.committed.pools[0].last_update,
22957 if commit_first {
22958 next.pools[0].last_update
22959 } else {
22960 previous_meta.pools[0].last_update
22961 }
22962 );
22963 outcome.disarm();
22964 let confirmed = super::load_pool_meta_for_transaction_recovery(pools, &mut state)
22965 .await
22966 .unwrap();
22967 assert!(!confirmed.replica_state.needs_repair);
22968 state.ensure_write_safe("repaired").unwrap();
22969 }
22970 }
22971 }
22972
22973 #[tokio::test]
22974 async fn pool_meta_recovery_rejects_missing_corrupt_and_incompatible_replicas_without_writes() {
22975 let cluster_id = uuid::Uuid::new_v4();
22976 let identity = super::initialized_pool_meta_identity_for_test(cluster_id, 1).unwrap();
22977 for payload in [None, Some(Vec::new()), Some(vec![1, 0, 255, 255, 0])] {
22978 let pool = Arc::new(PartialPoolMetaWriteStorage {
22979 stored: StdMutex::new(payload.map(|data| (data, "initial".to_owned()))),
22980 identity: StdMutex::new(Some((identity.clone(), "identity".to_owned()))),
22981 ..Default::default()
22982 });
22983 let mut state = super::PoolMetaWriteState::for_startup(cluster_id, false);
22984 super::load_pool_meta_for_transaction_recovery(vec![pool.clone()], &mut state)
22985 .await
22986 .unwrap_err();
22987 assert!(!pool.wrote.load(Ordering::SeqCst));
22988 }
22989 }
22990
22991 #[tokio::test]
22992 async fn pool_meta_first_cas_rejection_is_retryable_but_later_rejection_keeps_prior_write_armed() {
22993 let pool = Arc::new(PartialPoolMetaWriteStorage::default());
22994 let state = super::PoolMetaWriteState::default();
22995 let fence = PoolMetaPersistenceFence::Distributed(None);
22996 let mut arm = state.arm_transaction();
22997 save_pool_meta_object_cas(
22998 pool.clone(),
22999 POOL_META_NAME,
23000 vec![1],
23001 &PoolMetaCasToken::Existing("missing".to_owned()),
23002 &fence,
23003 "prepare_cas",
23004 &mut arm,
23005 )
23006 .await
23007 .unwrap_err();
23008 drop(arm);
23009 state.ensure_write_safe("first rejection").unwrap();
23010 let mut arm = state.arm_transaction();
23011 save_pool_meta_object_cas(
23012 pool.clone(),
23013 POOL_META_NAME,
23014 vec![1],
23015 &PoolMetaCasToken::Missing,
23016 &fence,
23017 "prepare_cas",
23018 &mut arm,
23019 )
23020 .await
23021 .unwrap();
23022 save_pool_meta_object_cas(pool, POOL_META_NAME, vec![2], &PoolMetaCasToken::Missing, &fence, "commit_cas", &mut arm)
23023 .await
23024 .unwrap_err();
23025 drop(arm);
23026 let err = state.ensure_write_safe("later rejection").unwrap_err();
23027 assert_eq!(err.pool_metadata_failure().unwrap().phase, "prepare_cas");
23028 assert!(matches!(
23029 err.pool_metadata_failure().unwrap().source.as_deref(),
23030 Some(Error::PreconditionFailed)
23031 ));
23032 }
23033
23034 #[tokio::test]
23035 async fn pool_meta_recovery_cas_conflict_after_repair_write_keeps_transaction_blocked() {
23036 let base = PoolMeta {
23037 version: POOL_META_VERSION,
23038 pools: vec![decommission_test_pool_status(0, None)],
23039 ..Default::default()
23040 };
23041 let mut current = base.clone();
23042 current.pools[0].last_update += Duration::seconds(1);
23043 let first = Arc::new(PartialPoolMetaWriteStorage {
23044 stored: StdMutex::new(Some((current.encode_config_data_for_v2_gate(true).unwrap(), "first".to_owned()))),
23045 ..Default::default()
23046 });
23047 let second = Arc::new(PartialPoolMetaWriteStorage {
23048 stored: StdMutex::new(Some((base.encode_config_data_for_v2_gate(true).unwrap(), "second".to_owned()))),
23049 ..Default::default()
23050 });
23051 let mut state = super::PoolMetaWriteState::default();
23052 let selection = super::load_pool_meta_for_transaction_recovery(vec![first.clone(), second.clone()], &mut state)
23053 .await
23054 .unwrap();
23055 let raced = base.encode_config_data_for_v2_gate(true).unwrap();
23056 *second.stored.lock().unwrap() = Some((raced.clone(), "raced".to_owned()));
23057 let result = super::repair_pool_meta_transaction(
23058 vec![first.clone(), second.clone()],
23059 &mut state,
23060 selection,
23061 &PoolMetaPersistenceFence::Distributed(None),
23062 )
23063 .await;
23064 assert!(matches!(result, Err(Error::PreconditionFailed)));
23065 assert!(first.wrote.load(Ordering::SeqCst));
23066 assert!(!second.wrote.load(Ordering::SeqCst));
23067 assert_eq!(second.stored.lock().unwrap().as_ref().unwrap().0, raced);
23068 let err = state.ensure_write_safe("recovery retry").unwrap_err();
23069 assert_eq!(err.pool_metadata_failure().unwrap().phase, "recovery_cas");
23070 }
23071
23072 #[tokio::test]
23073 async fn pool_meta_cas_failure_retains_the_original_source_for_caller_and_latch() {
23074 let pool = Arc::new(PartialPoolMetaWriteStorage {
23075 fail_write: true,
23076 ..Default::default()
23077 });
23078 let state = super::PoolMetaWriteState::default();
23079 let mut arm = state.arm_transaction();
23080 let err = save_pool_meta_object_cas(
23081 pool,
23082 POOL_META_NAME,
23083 Vec::new(),
23084 &PoolMetaCasToken::Missing,
23085 &PoolMetaPersistenceFence::Distributed(None),
23086 "commit_cas",
23087 &mut arm,
23088 )
23089 .await
23090 .unwrap_err();
23091 let source = err.pool_metadata_failure().unwrap().source.as_ref().unwrap();
23092 assert!(matches!(source.as_ref(), Error::Timeout));
23093 assert!(Arc::ptr_eq(source, arm.source.as_ref().unwrap()));
23094 drop(arm);
23095 let blocked = state.ensure_write_safe("after CAS failure").unwrap_err();
23096 assert!(Arc::ptr_eq(source, blocked.pool_metadata_failure().unwrap().source.as_ref().unwrap()));
23097 }
23098
23099 #[tokio::test]
23100 async fn pool_meta_cancelled_identity_dispatch_is_armed_before_pool_bin_writes() {
23101 let pool = Arc::new(PartialPoolMetaWriteStorage {
23102 pending_write: true,
23103 ..Default::default()
23104 });
23105 let state = Arc::new(tokio::sync::Mutex::new(super::PoolMetaWriteState::for_startup(
23106 uuid::Uuid::new_v4(),
23107 true,
23108 )));
23109 let task = tokio::spawn({
23110 let pool = pool.clone();
23111 let state = state.clone();
23112 async move {
23113 let mut state = state.lock().await;
23114 super::persist_pool_meta_identity_for_startup(vec![pool], &mut state, false).await
23115 }
23116 });
23117 pool.write_started.notified().await;
23118 task.abort();
23119 assert!(task.await.unwrap_err().is_cancelled());
23120 let state = state.lock().await;
23121 let err = state.ensure_write_safe("identity retry").unwrap_err();
23122 assert_eq!(err.pool_metadata_failure().unwrap().phase, "identity_cas");
23123 assert_eq!(
23124 err.pool_metadata_failure().unwrap().kind,
23125 crate::error::PoolMetadataFailure::TransactionUnknown
23126 );
23127 assert!(pool.stored.lock().unwrap().is_none());
23128 }
23129
23130 #[async_trait::async_trait]
23131 impl ObjectIO for PartialPoolMetaWriteStorage {
23132 type Error = Error;
23133 type RangeSpec = HTTPRangeSpec;
23134 type HeaderMap = http::HeaderMap;
23135 type ObjectOptions = crate::object_api::ObjectOptions;
23136 type ObjectInfo = crate::object_api::ObjectInfo;
23137 type GetObjectReader = crate::object_api::GetObjectReader;
23138 type PutObjectReader = crate::object_api::PutObjReader;
23139
23140 async fn get_object_reader(
23141 &self,
23142 bucket: &str,
23143 object: &str,
23144 _range: Option<Self::RangeSpec>,
23145 _h: Self::HeaderMap,
23146 _opts: &Self::ObjectOptions,
23147 ) -> std::result::Result<Self::GetObjectReader, Error> {
23148 let stored = if object == POOL_META_IDENTITY_NAME {
23149 &self.identity
23150 } else {
23151 &self.stored
23152 };
23153 let Some((data, etag)) = stored
23154 .lock()
23155 .expect("pool metadata test storage should not be poisoned")
23156 .clone()
23157 else {
23158 return Err(Error::FileNotFound);
23159 };
23160 Ok(crate::object_api::GetObjectReader {
23161 stream: Box::new(Cursor::new(data.clone())),
23162 object_info: crate::object_api::ObjectInfo {
23163 bucket: bucket.to_string(),
23164 name: object.to_string(),
23165 size: data.len() as i64,
23166 etag: Some(etag),
23167 ..Default::default()
23168 },
23169 buffered_body: None,
23170 body_source: Default::default(),
23171 })
23172 }
23173
23174 async fn put_object(
23175 &self,
23176 _bucket: &str,
23177 object: &str,
23178 data: &mut Self::PutObjectReader,
23179 opts: &Self::ObjectOptions,
23180 ) -> std::result::Result<Self::ObjectInfo, Error> {
23181 self.write_started.notify_one();
23182 if self.pending_write {
23183 std::future::pending().await
23184 }
23185 if object == POOL_META_NAME
23186 && (self.fail_write || (self.fail_after_first_write && self.revision.load(Ordering::SeqCst) > 0))
23187 {
23188 return Err(Error::Timeout);
23189 }
23190 let stored = if object == POOL_META_IDENTITY_NAME {
23191 &self.identity
23192 } else {
23193 &self.stored
23194 };
23195 let current_etag = stored
23196 .lock()
23197 .expect("pool metadata test storage should not be poisoned")
23198 .as_ref()
23199 .map(|(_, etag)| etag.clone());
23200 if opts
23201 .http_preconditions
23202 .as_ref()
23203 .and_then(HTTPPreconditions::if_none_match_value)
23204 == Some("*")
23205 && current_etag.is_some()
23206 {
23207 return Err(Error::PreconditionFailed);
23208 }
23209 if let Some(expected) = opts.http_preconditions.as_ref().and_then(HTTPPreconditions::if_match_value)
23210 && current_etag.as_deref() != Some(expected)
23211 {
23212 return Err(Error::PreconditionFailed);
23213 }
23214 let mut payload = Vec::new();
23215 data.stream.read_to_end(&mut payload).await?;
23216 let etag = format!("pool-meta-test-{}", self.revision.fetch_add(1, Ordering::SeqCst) + 1);
23217 *stored.lock().expect("pool metadata test storage should not be poisoned") = Some((payload, etag.clone()));
23218 self.wrote.store(true, Ordering::SeqCst);
23219 Ok(crate::object_api::ObjectInfo {
23220 etag: Some(etag),
23221 ..Default::default()
23222 })
23223 }
23224 }
23225
23226 mod phase_tests {
23227 use super::super as implementation;
23228 use super::*;
23229
23230 #[derive(Clone, Copy, Debug, Default)]
23231 enum WriteFault {
23232 #[default]
23233 None,
23234 Reject,
23235 Timeout,
23236 TimeoutAfterWrite,
23237 EmptyEtag,
23238 OmitWrite,
23239 }
23240
23241 #[derive(Clone, Debug, Default)]
23242 struct WriteStep {
23243 gate: Option<Arc<tokio::sync::Semaphore>>,
23244 fault: WriteFault,
23245 }
23246
23247 #[derive(Clone, Debug)]
23248 struct WriteCall {
23249 pool: usize,
23250 phase: usize,
23251 if_match: Option<String>,
23252 }
23253
23254 #[derive(Debug, Default)]
23255 struct WriteTrace {
23256 calls: StdMutex<Vec<WriteCall>>,
23257 active: AtomicUsize,
23258 maximum: AtomicUsize,
23259 finished: AtomicUsize,
23260 changed: tokio::sync::Notify,
23261 }
23262
23263 impl WriteTrace {
23264 async fn wait_for(&self, calls: usize, finished: usize) {
23265 tokio::time::timeout(StdDuration::from_secs(3), async {
23266 loop {
23267 let changed = self.changed.notified();
23268 if self.calls.lock().expect("trace lock").len() >= calls
23269 && self.finished.load(Ordering::SeqCst) >= finished
23270 {
23271 return;
23272 }
23273 changed.await;
23274 }
23275 })
23276 .await
23277 .expect("phase must reach the requested barrier");
23278 }
23279
23280 fn commit_count(&self) -> usize {
23281 self.calls
23282 .lock()
23283 .expect("trace lock")
23284 .iter()
23285 .filter(|call| call.phase == 1)
23286 .count()
23287 }
23288 }
23289
23290 struct ActiveWrite<'a>(&'a WriteTrace);
23291 impl Drop for ActiveWrite<'_> {
23292 fn drop(&mut self) {
23293 self.0.active.fetch_sub(1, Ordering::SeqCst);
23294 self.0.finished.fetch_add(1, Ordering::SeqCst);
23295 self.0.changed.notify_one();
23296 }
23297 }
23298
23299 #[derive(Debug)]
23300 struct PhaseStorage {
23301 inner: PartialPoolMetaWriteStorage,
23302 pool: usize,
23303 steps: [WriteStep; 2],
23304 trace: Arc<WriteTrace>,
23305 }
23306
23307 #[async_trait::async_trait]
23308 impl ObjectIO for PhaseStorage {
23309 type Error = Error;
23310 type RangeSpec = HTTPRangeSpec;
23311 type HeaderMap = http::HeaderMap;
23312 type ObjectOptions = crate::object_api::ObjectOptions;
23313 type ObjectInfo = crate::object_api::ObjectInfo;
23314 type GetObjectReader = crate::object_api::GetObjectReader;
23315 type PutObjectReader = crate::object_api::PutObjReader;
23316
23317 async fn get_object_reader(
23318 &self,
23319 bucket: &str,
23320 object: &str,
23321 range: Option<Self::RangeSpec>,
23322 headers: Self::HeaderMap,
23323 opts: &Self::ObjectOptions,
23324 ) -> Result<Self::GetObjectReader, Error> {
23325 self.inner.get_object_reader(bucket, object, range, headers, opts).await
23326 }
23327
23328 async fn put_object(
23329 &self,
23330 bucket: &str,
23331 object: &str,
23332 data: &mut Self::PutObjectReader,
23333 opts: &Self::ObjectOptions,
23334 ) -> Result<Self::ObjectInfo, Error> {
23335 if object != POOL_META_NAME {
23336 return self.inner.put_object(bucket, object, data, opts).await;
23337 }
23338 let mut payload = Vec::new();
23339 data.stream.read_to_end(&mut payload).await?;
23340 let phase = match implementation::decode_pool_meta_replica(payload.clone()) {
23341 implementation::PoolMetaReplica::Valid { committed, .. } => usize::from(committed),
23342 _ => panic!("phase fixture must receive valid pool metadata"),
23343 };
23344 assert!(opts.max_parity && opts.no_lock);
23345 assert_eq!(opts.write_completion, crate::object_api::WriteCompletion::TailDrained);
23346 self.trace.calls.lock().expect("trace lock").push(WriteCall {
23347 pool: self.pool,
23348 phase,
23349 if_match: opts
23350 .http_preconditions
23351 .as_ref()
23352 .and_then(HTTPPreconditions::if_match_value)
23353 .map(str::to_owned),
23354 });
23355 let active = self.trace.active.fetch_add(1, Ordering::SeqCst) + 1;
23356 self.trace.maximum.fetch_max(active, Ordering::SeqCst);
23357 let _active = ActiveWrite(&self.trace);
23358 self.trace.changed.notify_one();
23359 let step = &self.steps[phase];
23360 if let Some(gate) = &step.gate {
23361 gate.acquire().await.expect("phase gate stays open").forget();
23362 }
23363 match step.fault {
23364 WriteFault::Reject => return Err(Error::PreconditionFailed),
23365 WriteFault::Timeout => return Err(Error::Timeout),
23366 WriteFault::OmitWrite => {
23367 return Ok(crate::object_api::ObjectInfo {
23368 etag: Some("uncommitted".to_owned()),
23369 ..Default::default()
23370 });
23371 }
23372 _ => {}
23373 }
23374 let mut data = crate::object_api::PutObjReader::from_vec(payload);
23375 let mut result = self.inner.put_object(bucket, object, &mut data, opts).await?;
23376 match step.fault {
23377 WriteFault::TimeoutAfterWrite => return Err(Error::Timeout),
23378 WriteFault::EmptyEtag => result.etag = Some(" ".to_owned()),
23379 _ => {}
23380 }
23381 Ok(result)
23382 }
23383 }
23384
23385 struct Fixture {
23386 pools: Vec<Arc<PhaseStorage>>,
23387 trace: Arc<WriteTrace>,
23388 previous: implementation::PoolMetaCommittedCandidate,
23389 desired: PoolMeta,
23390 pending: Vec<u8>,
23391 durable: Vec<u8>,
23392 revision: implementation::PoolMetaRevision,
23393 cluster_id: uuid::Uuid,
23394 }
23395
23396 impl Fixture {
23397 fn new(steps: Vec<[WriteStep; 2]>) -> Self {
23398 let cluster_id = uuid::Uuid::new_v4();
23399 let meta = PoolMeta {
23400 version: POOL_META_GENERATION_VERSION,
23401 pools: (0..steps.len())
23402 .map(|index| {
23403 decommission_test_pool_status(
23404 index,
23405 (index == 0).then(|| PoolDecommissionInfo {
23406 start_time: Some(OffsetDateTime::UNIX_EPOCH),
23407 start_size: 4096,
23408 total_size: 16384,
23409 current_size: 8192,
23410 items_decommissioned: 7,
23411 items_decommission_failed: 1,
23412 bytes_done: 4096,
23413 bytes_failed: 512,
23414 queued_buckets: vec!["remaining-a".to_owned(), "remaining-b".to_owned()],
23415 decommissioned_buckets: vec!["completed".to_owned()],
23416 bucket: "remaining-a".to_owned(),
23417 prefix: "objects/".to_owned(),
23418 object: "objects/007".to_owned(),
23419 ..Default::default()
23420 }),
23421 )
23422 })
23423 .collect(),
23424 ..Default::default()
23425 };
23426 let previous_revision = implementation::PoolMetaRevision {
23427 version: POOL_META_GENERATION_VERSION,
23428 cluster_id: Some(cluster_id),
23429 epoch: 1,
23430 generation: 7,
23431 transaction_id: Some(uuid::Uuid::new_v4()),
23432 };
23433 let previous = implementation::PoolMetaCommittedCandidate {
23434 canonical: implementation::encode_pool_meta_v3_envelope(&meta, previous_revision, true, None)
23435 .expect("committed predecessor"),
23436 meta: meta.clone(),
23437 revision: previous_revision,
23438 };
23439 let mut desired = meta;
23440 desired.pools[0].last_update += Duration::seconds(1);
23441 let progress = desired.pools[0].decommission.as_mut().expect("active decommission");
23442 progress.items_decommissioned += 1;
23443 progress.bytes_done += 512;
23444 progress.current_size += 512;
23445 progress.object = "objects/008".to_owned();
23446 let revision = implementation::PoolMetaRevision {
23447 generation: 8,
23448 transaction_id: Some(uuid::Uuid::new_v4()),
23449 ..previous_revision
23450 };
23451 let pending = implementation::encode_pool_meta_v3_envelope(&desired, revision, false, Some(&previous))
23452 .expect("pending candidate");
23453 let durable =
23454 implementation::encode_pool_meta_v3_envelope(&desired, revision, true, None).expect("committed candidate");
23455 let identity = implementation::initialized_pool_meta_identity_for_test(cluster_id, 1).expect("identity");
23456 let trace = Arc::new(WriteTrace::default());
23457 let pools = steps
23458 .into_iter()
23459 .enumerate()
23460 .map(|(index, steps)| {
23461 Arc::new(PhaseStorage {
23462 inner: PartialPoolMetaWriteStorage {
23463 stored: StdMutex::new(Some((previous.canonical.clone(), format!("initial-{index}")))),
23464 identity: StdMutex::new(Some((identity.clone(), "identity".to_owned()))),
23465 revision: AtomicUsize::new(index * 10),
23466 ..Default::default()
23467 },
23468 pool: index,
23469 steps,
23470 trace: trace.clone(),
23471 })
23472 })
23473 .collect();
23474 Self {
23475 pools,
23476 trace,
23477 previous,
23478 desired,
23479 pending,
23480 durable,
23481 revision,
23482 cluster_id,
23483 }
23484 }
23485
23486 fn state(&self) -> implementation::PoolMetaWriteState {
23487 implementation::PoolMetaWriteState::for_startup(self.cluster_id, false)
23488 }
23489
23490 fn tokens(&self) -> Vec<PoolMetaCasToken> {
23491 self.pools
23492 .iter()
23493 .map(|pool| {
23494 PoolMetaCasToken::Existing(
23495 pool.inner
23496 .stored
23497 .lock()
23498 .expect("fixture storage")
23499 .as_ref()
23500 .expect("existing replica")
23501 .1
23502 .clone(),
23503 )
23504 })
23505 .collect()
23506 }
23507 }
23508
23509 fn steps(count: usize) -> Vec<[WriteStep; 2]> {
23510 (0..count).map(|_| [WriteStep::default(), WriteStep::default()]).collect()
23511 }
23512
23513 fn signal() -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
23514 Some(Arc::default())
23515 }
23516
23517 async fn prepare(
23518 fixture: &Fixture,
23519 fence: &PoolMetaPersistenceFence<'_>,
23520 arm: &mut implementation::PoolMetaTransactionArm,
23521 ) -> Vec<Result<crate::object_api::ObjectInfo, Error>> {
23522 implementation::save_pool_meta_phase(
23523 &fixture.pools,
23524 &fixture.pending,
23525 &fixture.tokens(),
23526 fence,
23527 "prepare_cas",
23528 arm,
23529 #[cfg(feature = "e2e-test-hooks")]
23530 None,
23531 )
23532 .await
23533 }
23534
23535 #[tokio::test]
23536 async fn pool_meta_v3_phase_concurrency_is_bounded() {
23537 let gate = Arc::new(tokio::sync::Semaphore::new(0));
23538 let mut config = steps(6);
23539 for step in &mut config {
23540 step[0].gate = Some(gate.clone());
23541 }
23542 let fixture = Fixture::new(config);
23543 let state = fixture.state();
23544 let mut arm = state.arm_transaction();
23545 let fence = PoolMetaPersistenceFence::Distributed(signal());
23546 let mut pending = Box::pin(prepare(&fixture, &fence, &mut arm));
23547 tokio::select! { _ = fixture.trace.wait_for(4, 0) => {}, _ = &mut pending => panic!("blocked writes finished") }
23548 assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 4);
23549 gate.add_permits(1);
23550 tokio::select! { _ = fixture.trace.wait_for(5, 1) => {}, _ = &mut pending => panic!("unreleased writes finished") }
23551 assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 5);
23552 gate.add_permits(5);
23553 assert!(pending.await.into_iter().all(|result| result.is_ok()));
23554 assert_eq!(fixture.trace.maximum.load(Ordering::SeqCst), 4);
23555 assert_eq!(fixture.trace.active.load(Ordering::SeqCst), 0);
23556 arm.disarm();
23557 }
23558
23559 #[tokio::test]
23560 async fn pool_meta_v3_commit_waits_for_all_prepare_etags() {
23561 let gate = Arc::new(tokio::sync::Semaphore::new(0));
23562 let mut config = steps(4);
23563 config[3][0].gate = Some(gate.clone());
23564 let fixture = Fixture::new(config);
23565 let mut state = fixture.state();
23566 let mut save = Box::pin(
23567 fixture
23568 .desired
23569 .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]),
23570 );
23571 tokio::select! { _ = fixture.trace.wait_for(4, 3) => {}, _ = &mut save => panic!("commit crossed the last prepare") }
23572 assert_eq!(fixture.trace.commit_count(), 0);
23573 gate.add_permits(1);
23574 save.await.expect("all prepares and commits succeed").disarm();
23575 let calls = fixture.trace.calls.lock().expect("trace");
23576 for call in calls.iter().filter(|call| call.phase == 1) {
23577 assert_eq!(call.if_match, Some(format!("pool-meta-test-{}", call.pool * 10 + 1)));
23578 }
23579 assert_eq!(calls.iter().filter(|call| call.phase == 1).count(), 4);
23580 }
23581
23582 #[tokio::test]
23583 async fn pool_meta_v3_empty_prepare_etag_never_starts_commit() {
23584 let mut config = steps(4);
23585 config[3][0].fault = WriteFault::EmptyEtag;
23586 let fixture = Fixture::new(config);
23587 let mut state = fixture.state();
23588 let err = fixture
23589 .desired
23590 .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0])
23591 .await
23592 .expect_err("a missing conditional-write revision must fail");
23593 assert!(err.to_string().contains("without a conditional-write revision"));
23594 assert_eq!(fixture.trace.finished.load(Ordering::SeqCst), 4);
23595 assert_eq!(fixture.trace.commit_count(), 0);
23596 assert!(state.ensure_write_safe("empty etag wrote pending data").is_err());
23597 }
23598
23599 #[tokio::test]
23600 async fn pool_meta_v3_prepare_failure_drains_late_writes() {
23601 let gate = Arc::new(tokio::sync::Semaphore::new(0));
23602 let mut config = steps(4);
23603 config[1][0].fault = WriteFault::Timeout;
23604 config[3][0].gate = Some(gate.clone());
23605 let fixture = Fixture::new(config);
23606 let mut state = fixture.state();
23607 let mut save = Box::pin(
23608 fixture
23609 .desired
23610 .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]),
23611 );
23612 tokio::select! { _ = fixture.trace.wait_for(4, 3) => {}, _ = &mut save => panic!("phase returned with a live replica") }
23613 assert_eq!(fixture.trace.commit_count(), 0);
23614 gate.add_permits(1);
23615 assert!(save.await.is_err());
23616 assert_eq!(fixture.trace.finished.load(Ordering::SeqCst), 4);
23617 assert!(fixture.pools[3].inner.wrote.load(Ordering::SeqCst));
23618 assert!(state.ensure_write_safe("ambiguous prepare").is_err());
23619 }
23620
23621 #[tokio::test]
23622 async fn pool_meta_v3_rejections_restore_only_an_unmutated_phase() {
23623 for previous in [None, Some("identity_cas")] {
23624 let mut config = steps(4);
23625 for step in &mut config {
23626 step[0].fault = WriteFault::Reject;
23627 }
23628 let fixture = Fixture::new(config);
23629 let state = fixture.state();
23630 let mut arm = state.arm_transaction();
23631 arm.phase = previous;
23632 let results = prepare(&fixture, &PoolMetaPersistenceFence::Distributed(signal()), &mut arm).await;
23633 assert!(
23634 results
23635 .into_iter()
23636 .all(|result| matches!(result, Err(Error::PreconditionFailed)))
23637 );
23638 assert_eq!(arm.phase, previous);
23639 drop(arm);
23640 assert_eq!(state.ensure_write_safe("all rejected").is_ok(), previous.is_none());
23641 }
23642 let mut config = steps(4);
23643 for step in &mut config[1..] {
23644 step[0].fault = WriteFault::Reject;
23645 }
23646 let fixture = Fixture::new(config);
23647 let state = fixture.state();
23648 let mut arm = state.arm_transaction();
23649 let results = prepare(&fixture, &PoolMetaPersistenceFence::Distributed(signal()), &mut arm).await;
23650 assert!(results[0].is_ok());
23651 assert_eq!(arm.phase, Some("prepare_cas"));
23652 drop(arm);
23653 assert!(state.ensure_write_safe("one sibling wrote").is_err());
23654 }
23655
23656 #[tokio::test]
23657 async fn pool_meta_v3_parallel_errors_keep_original_source_pointers() {
23658 let mut config = steps(4);
23659 config[0][0].fault = WriteFault::Timeout;
23660 config[1][0].fault = WriteFault::Timeout;
23661 config[3][0].fault = WriteFault::Reject;
23662 let fixture = Fixture::new(config);
23663 let state = fixture.state();
23664 let mut arm = state.arm_transaction();
23665 let results = prepare(&fixture, &PoolMetaPersistenceFence::Distributed(signal()), &mut arm).await;
23666 let err = results[0].as_ref().expect_err("first pool error stays first");
23667 let source = err
23668 .pool_metadata_failure()
23669 .expect("typed error")
23670 .source
23671 .as_ref()
23672 .expect("original source");
23673 assert!(matches!(source.as_ref(), Error::Timeout));
23674 assert!(Arc::ptr_eq(source, arm.source.as_ref().expect("arm source")));
23675 drop(arm);
23676 let blocked = state
23677 .ensure_write_safe("after parallel errors")
23678 .expect_err("recovery required");
23679 assert!(Arc::ptr_eq(
23680 source,
23681 blocked
23682 .pool_metadata_failure()
23683 .expect("typed latch")
23684 .source
23685 .as_ref()
23686 .expect("latch source")
23687 ));
23688 }
23689
23690 #[tokio::test]
23691 async fn pool_meta_v3_retry_rejections_keep_the_previous_io_source() {
23692 let mut config = steps(4);
23693 config[0][0].fault = WriteFault::Reject;
23694 config[1][0].fault = WriteFault::Timeout;
23695 let first = Fixture::new(config);
23696 let state = first.state();
23697 let mut arm = state.arm_transaction();
23698 let fence = PoolMetaPersistenceFence::Distributed(signal());
23699 let results = prepare(&first, &fence, &mut arm).await;
23700 assert!(matches!(&results[0], Err(Error::PreconditionFailed)));
23701 let original = arm.source.as_ref().expect("first attempt I/O source").clone();
23702 assert!(matches!(original.as_ref(), Error::Timeout));
23703 let mut config = steps(4);
23704 for step in &mut config {
23705 step[0].fault = WriteFault::Reject;
23706 }
23707 let retry = Fixture::new(config);
23708 assert!(
23709 prepare(&retry, &fence, &mut arm)
23710 .await
23711 .into_iter()
23712 .all(|result| matches!(result, Err(Error::PreconditionFailed)))
23713 );
23714 assert!(Arc::ptr_eq(&original, arm.source.as_ref().expect("source survives phase retry")));
23715 implementation::save_pool_meta_object_cas(
23716 retry.pools[0].clone(),
23717 POOL_META_NAME,
23718 retry.pending.clone(),
23719 &retry.tokens()[0],
23720 &fence,
23721 "prepare_cas",
23722 &mut arm,
23723 )
23724 .await
23725 .expect_err("serial retry rejected");
23726 assert!(Arc::ptr_eq(&original, arm.source.as_ref().expect("source survives serial retry")));
23727 drop(arm);
23728 let blocked = state
23729 .ensure_write_safe("cancelled after retry rejection")
23730 .expect_err("recovery required");
23731 assert!(Arc::ptr_eq(
23732 &original,
23733 blocked
23734 .pool_metadata_failure()
23735 .expect("typed latch")
23736 .source
23737 .as_ref()
23738 .expect("source")
23739 ));
23740 }
23741
23742 #[tokio::test]
23743 async fn pool_meta_v3_cancelled_drain_does_not_replace_io_error_with_cas_rejection() {
23744 let gate = Arc::new(tokio::sync::Semaphore::new(0));
23745 let mut config = steps(4);
23746 config[0][0].gate = Some(gate.clone());
23747 config[2][0].gate = Some(gate);
23748 config[1][0].fault = WriteFault::Timeout;
23749 config[3][0].fault = WriteFault::Reject;
23750 let fixture = Fixture::new(config);
23751 let state = fixture.state();
23752 let mut arm = state.arm_transaction();
23753 let fence = PoolMetaPersistenceFence::Distributed(signal());
23754 let mut phase = Box::pin(prepare(&fixture, &fence, &mut arm));
23755 tokio::select! { _ = fixture.trace.wait_for(4, 2) => {}, _ = &mut phase => panic!("blocked phase finished") }
23756 assert!(futures::poll!(phase.as_mut()).is_pending(), "drain remains parked on the two slow pools");
23757 drop(phase);
23758 let source = arm.source.as_ref().expect("observed I/O source").clone();
23759 assert!(matches!(source.as_ref(), Error::Timeout));
23760 drop(arm);
23761 let blocked = state
23762 .ensure_write_safe("cancelled after mixed results")
23763 .expect_err("recovery required");
23764 assert!(Arc::ptr_eq(
23765 &source,
23766 blocked
23767 .pool_metadata_failure()
23768 .expect("typed latch")
23769 .source
23770 .as_ref()
23771 .expect("source")
23772 ));
23773 assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
23774 }
23775
23776 #[tokio::test]
23777 async fn pool_meta_v3_cancelled_prepare_commit_and_publication_keep_gate_blocked() {
23778 for phase in [0, 1] {
23779 let gate = Arc::new(tokio::sync::Semaphore::new(0));
23780 let mut config = steps(4);
23781 for step in &mut config {
23782 step[phase].gate = Some(gate.clone());
23783 }
23784 let fixture = Fixture::new(config);
23785 let mut state = fixture.state();
23786 let mut save = Box::pin(
23787 fixture
23788 .desired
23789 .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]),
23790 );
23791 tokio::select! { _ = fixture.trace.wait_for(4 * (phase + 1), 4 * phase) => {}, _ = &mut save => panic!("blocked phase finished") }
23792 drop(save);
23793 assert!(state.ensure_write_safe("cancelled phase").is_err());
23794 assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
23795 assert_eq!(fixture.trace.active.load(Ordering::SeqCst), 0);
23796 }
23797 let fixture = Fixture::new(steps(4));
23798 let mut state = fixture.state();
23799 let outcome = fixture
23800 .desired
23801 .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0])
23802 .await
23803 .expect("durable save before publication");
23804 drop(outcome);
23805 assert!(state.ensure_write_safe("publication was cancelled").is_err());
23806 assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
23807 }
23808
23809 #[tokio::test]
23810 async fn pool_meta_v3_fence_loss_blocks_queued_dispatch_and_marks_late_writes() {
23811 let lock = NamespaceLock::with_clients_and_quorum(
23812 "parallel-fence".to_owned(),
23813 vec![Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())))],
23814 1,
23815 );
23816 let request = LockRequest::new(
23817 ObjectKey::new(super::super::RUSTFS_META_BUCKET, POOL_META_NAME),
23818 LockType::Exclusive,
23819 "phase-test",
23820 )
23821 .with_acquire_timeout(StdDuration::from_secs(1))
23822 .with_ttl(StdDuration::from_secs(1))
23823 .with_refresh_interval(StdDuration::from_secs(1));
23824 let guard = lock
23825 .acquire_guard(&request)
23826 .await
23827 .expect("lease request")
23828 .expect("lease acquired");
23829 let gate = Arc::new(tokio::sync::Semaphore::new(0));
23830 let mut config = steps(6);
23831 for step in &mut config {
23832 step[0].gate = Some(gate.clone());
23833 }
23834 let fixture = Fixture::new(config);
23835 let state = fixture.state();
23836 let mut arm = state.arm_transaction();
23837 let fence = PoolMetaPersistenceFence::Distributed(guard.lock_lost_signal());
23838 let mut pending = Box::pin(prepare(&fixture, &fence, &mut arm));
23839 tokio::select! { _ = fixture.trace.wait_for(4, 0) => {}, _ = &mut pending => panic!("gate ignored") }
23840 tokio::time::timeout(StdDuration::from_secs(3), guard.lock_lost_notified())
23841 .await
23842 .expect("lease expires");
23843 gate.add_permits(6);
23844 assert!(pending.await.into_iter().all(|result| result.is_err()));
23845 assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 4, "queued pools must not reach storage");
23846 assert!(fixture.pools[..4].iter().all(|pool| pool.inner.wrote.load(Ordering::SeqCst)));
23847 assert_eq!(arm.phase, Some("prepare_cas"));
23848 drop(arm);
23849 assert!(state.ensure_write_safe("writes landed after lease loss").is_err());
23850 }
23851
23852 #[tokio::test]
23853 async fn pool_meta_v3_all_commit_errors_still_require_canonical_reread() {
23854 for fault in [WriteFault::TimeoutAfterWrite, WriteFault::OmitWrite] {
23855 let mut config = steps(4);
23856 for step in &mut config {
23857 step[1].fault = fault;
23858 }
23859 let fixture = Fixture::new(config);
23860 let mut state = fixture.state();
23861 let result = fixture
23862 .desired
23863 .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0])
23864 .await;
23865 assert_eq!(fixture.trace.commit_count(), 4);
23866 if matches!(fault, WriteFault::TimeoutAfterWrite) {
23867 let outcome = result.expect("reread proves commits whose replies timed out");
23868 assert_eq!(outcome.committed.pools[0].last_update, fixture.desired.pools[0].last_update);
23869 outcome.disarm();
23870 state
23871 .ensure_write_safe("confirmed ambiguous commits")
23872 .expect("confirmed transaction is safe");
23873 } else {
23874 assert!(result.is_err(), "successful replies cannot replace canonical proof");
23875 assert!(state.ensure_write_safe("no committed canonical generation").is_err());
23876 }
23877 }
23878 }
23879
23880 #[tokio::test]
23881 async fn pool_meta_v3_none_and_legacy_paths_remain_serial() {
23882 for (legacy, needs_repair) in [(false, false), (true, false), (false, true)] {
23883 let gate = Arc::new(tokio::sync::Semaphore::new(0));
23884 let mut config = steps(4);
23885 for step in &mut config {
23886 step[0].gate = Some(gate.clone());
23887 step[1].gate = Some(gate.clone());
23888 }
23889 let mut fixture = Fixture::new(config);
23890 if legacy {
23891 let mut meta = fixture.previous.meta.clone();
23892 meta.version = POOL_META_VERSION;
23893 let data = meta.encode_config_data_for_v2_gate(true).expect("legacy metadata");
23894 for pool in &fixture.pools {
23895 pool.inner.stored.lock().expect("fixture").as_mut().expect("replica").0 = data.clone();
23896 }
23897 fixture.desired.version = POOL_META_VERSION;
23898 }
23899 if needs_repair {
23900 fixture.pools[3]
23901 .inner
23902 .stored
23903 .lock()
23904 .expect("fixture")
23905 .as_mut()
23906 .expect("replica")
23907 .0 = fixture.pending.clone();
23908 }
23909 let mut state = fixture.state();
23910 let mut save = Box::pin(fixture.desired.save_no_lock_armed(
23911 fixture.pools.clone(),
23912 &mut state,
23913 if legacy || needs_repair { signal() } else { None },
23914 &[0],
23915 ));
23916 tokio::select! { _ = fixture.trace.wait_for(1, 0) => {}, _ = &mut save => panic!("serial write gate ignored") }
23917 assert!(futures::poll!(save.as_mut()).is_pending());
23918 assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 1);
23919 gate.add_permits(16);
23920 save.await.expect("serial save").disarm();
23921 assert_eq!(fixture.trace.maximum.load(Ordering::SeqCst), 1);
23922 }
23923 }
23924
23925 #[cfg(feature = "e2e-test-hooks")]
23926 #[tokio::test]
23927 async fn pool_meta_phase_barrier_leaves_noop_arm_for_the_next_changed_transaction() {
23928 for case in [
23929 implementation::PoolMetaPhaseBarrierCase::PrepareSubset,
23930 implementation::PoolMetaPhaseBarrierCase::PreparedAll,
23931 implementation::PoolMetaPhaseBarrierCase::CommitOne,
23932 implementation::PoolMetaPhaseBarrierCase::BeforePublish,
23933 ] {
23934 let fixture = Fixture::new(steps(4));
23935 let directory = tempfile::TempDir::new().expect("barrier directory");
23936 tokio::fs::write(
23937 directory.path().join("arm.json"),
23938 serde_json::to_vec(&serde_json::json!({"nonce": uuid::Uuid::new_v4(), "case": case})).expect("arm"),
23939 )
23940 .await
23941 .expect("write arm");
23942 let noop = implementation::encode_pool_meta_v3_envelope(&fixture.previous.meta, fixture.revision, true, None)
23943 .expect("new revision with unchanged pools");
23944 assert_ne!(noop, fixture.previous.canonical);
23945 let barrier = implementation::PoolMetaPhaseBarrier::bind_in_directory(
23946 directory.path().to_path_buf(),
23947 &fixture.previous,
23948 &fixture.previous.meta,
23949 fixture.revision,
23950 &noop,
23951 )
23952 .await
23953 .expect("unchanged pools must not consume the arm");
23954 assert!(barrier.is_none());
23955 assert!(!directory.path().join("claimed").exists());
23956 assert!(!directory.path().join("events.jsonl").exists());
23957
23958 let previous = implementation::PoolMetaCommittedCandidate {
23959 meta: fixture.previous.meta.clone(),
23960 revision: fixture.revision,
23961 canonical: noop,
23962 };
23963 let mut candidate = previous.meta.clone();
23964 let progress = candidate.pools[0].decommission.as_mut().expect("progress");
23965 progress.items_decommissioned += 1;
23966 progress.bytes_done += 512;
23967 let revision = implementation::PoolMetaRevision {
23968 generation: previous.revision.generation + 1,
23969 transaction_id: Some(uuid::Uuid::new_v4()),
23970 ..previous.revision
23971 };
23972 let durable = implementation::encode_pool_meta_v3_envelope(&candidate, revision, true, None)
23973 .expect("changed progress without a timestamp change");
23974 let barrier = implementation::PoolMetaPhaseBarrier::bind_in_directory(
23975 directory.path().to_path_buf(),
23976 &previous,
23977 &candidate,
23978 revision,
23979 &durable,
23980 )
23981 .await
23982 .expect("next changed transaction must bind")
23983 .expect("bound barrier");
23984 assert_eq!(barrier.transaction_id, revision.transaction_id.expect("transaction"));
23985 assert!(directory.path().join("claimed").is_file());
23986 let events = tokio::fs::read_to_string(directory.path().join("events.jsonl"))
23987 .await
23988 .expect("armed event");
23989 assert_eq!(events.lines().count(), 1);
23990 let event: serde_json::Value = serde_json::from_str(events.trim()).expect("armed event JSON");
23991 assert_eq!(event["kind"], "armed");
23992 assert_eq!(event["previous"]["generation"], previous.revision.generation);
23993 assert_eq!(event["candidate"]["generation"], revision.generation);
23994 assert_ne!(event["previous"]["persisted_pools_sha256"], event["candidate"]["persisted_pools_sha256"]);
23995 assert!(
23996 implementation::PoolMetaPhaseBarrier::bind_in_directory(
23997 directory.path().to_path_buf(),
23998 &previous,
23999 &candidate,
24000 revision,
24001 &durable,
24002 )
24003 .await
24004 .expect("already claimed arm")
24005 .is_none()
24006 );
24007 }
24008 }
24009
24010 #[cfg(feature = "e2e-test-hooks")]
24011 #[tokio::test]
24012 async fn pool_meta_phase_barrier_uses_persisted_responsibility_not_runtime_progress() {
24013 let fixture = Fixture::new(steps(4));
24014 let directory = tempfile::TempDir::new().expect("barrier directory");
24015 tokio::fs::write(
24016 directory.path().join("arm.json"),
24017 serde_json::to_vec(&serde_json::json!({"nonce": uuid::Uuid::new_v4(), "case": "prepare_subset"})).expect("arm"),
24018 )
24019 .await
24020 .expect("write arm");
24021 let mut candidate = fixture.previous.meta.clone();
24022 candidate.pools[0].decommission.as_mut().expect("progress").stage = "entry_started".to_owned();
24023 for changed in [false, true] {
24024 if changed {
24025 candidate.pools[0]
24026 .decommission
24027 .as_mut()
24028 .expect("progress")
24029 .queued_buckets
24030 .push("new-responsibility".to_owned());
24031 }
24032 let durable =
24033 implementation::encode_pool_meta_v3_envelope(&candidate, fixture.revision, true, None).expect("candidate");
24034 let barrier = implementation::PoolMetaPhaseBarrier::bind_in_directory(
24035 directory.path().to_path_buf(),
24036 &fixture.previous,
24037 &candidate,
24038 fixture.revision,
24039 &durable,
24040 )
24041 .await
24042 .expect("binding by persisted state");
24043 assert_eq!(barrier.is_some(), changed);
24044 assert_eq!(directory.path().join("claimed").exists(), changed);
24045 assert_eq!(directory.path().join("events.jsonl").exists(), changed);
24046 }
24047 }
24048
24049 fn persisted_pools(meta: &PoolMeta) -> serde_json::Value {
24050 serde_json::to_value(
24051 meta.pools
24052 .iter()
24053 .map(implementation::PersistedPoolStatus::from)
24054 .collect::<Vec<_>>(),
24055 )
24056 .expect("persisted pool state")
24057 }
24058
24059 #[tokio::test]
24060 async fn pool_meta_v3_all_prepare_subsets_preserve_previous_snapshot() {
24061 for previous_version in [POOL_META_VERSION, POOL_META_GENERATION_VERSION] {
24062 for mask in 0usize..16 {
24063 let mut fixture = Fixture::new(steps(4));
24064 if previous_version != POOL_META_GENERATION_VERSION {
24065 fixture.previous.meta.version = previous_version;
24066 fixture.previous.revision = implementation::PoolMetaRevision::legacy(previous_version);
24067 fixture.revision.generation = 1;
24068 fixture.previous.canonical = fixture
24069 .previous
24070 .meta
24071 .encode_config_data_for_v2_gate(true)
24072 .expect("legacy predecessor");
24073 fixture.pending = implementation::encode_pool_meta_v3_envelope(
24074 &fixture.desired,
24075 fixture.revision,
24076 false,
24077 Some(&fixture.previous),
24078 )
24079 .expect("legacy pending envelope");
24080 }
24081 let payloads = (0..4)
24082 .map(|pool| {
24083 if mask & (1 << pool) != 0 {
24084 fixture.pending.clone()
24085 } else {
24086 fixture.previous.canonical.clone()
24087 }
24088 })
24089 .collect::<Vec<_>>();
24090 let selected = implementation::select_pool_meta_replica(
24091 payloads
24092 .iter()
24093 .cloned()
24094 .map(implementation::decode_pool_meta_replica)
24095 .collect(),
24096 )
24097 .expect("every pending subset selects its predecessor");
24098 assert_eq!(
24099 persisted_pools(&selected.meta),
24100 persisted_pools(&fixture.previous.meta),
24101 "mask={mask:04b}"
24102 );
24103 assert_eq!(selected.revision, fixture.previous.revision);
24104 if previous_version == POOL_META_GENERATION_VERSION {
24105 for (pool, data) in fixture.pools.iter().zip(payloads) {
24106 pool.inner.stored.lock().expect("fixture").as_mut().expect("replica").0 = data;
24107 }
24108 let mut state = fixture.state();
24109 let selected = implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state)
24110 .await
24111 .expect("production recovery selection");
24112 let outcome = implementation::repair_pool_meta_transaction(
24113 fixture.pools.clone(),
24114 &mut state,
24115 selected,
24116 &PoolMetaPersistenceFence::Distributed(None),
24117 )
24118 .await
24119 .expect("production recovery repair");
24120 assert_eq!(persisted_pools(&outcome.committed), persisted_pools(&fixture.previous.meta));
24121 outcome.disarm();
24122 let confirmed =
24123 implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state)
24124 .await
24125 .expect("repaired reread");
24126 assert_eq!(persisted_pools(&confirmed.meta), persisted_pools(&fixture.previous.meta));
24127 assert!(!confirmed.replica_state.needs_repair);
24128 }
24129 }
24130 }
24131 }
24132
24133 #[tokio::test]
24134 async fn pool_meta_v3_every_sole_commit_position_recovers_new_snapshot() {
24135 for committed_pool in 0..4 {
24136 let fixture = Fixture::new(steps(4));
24137 for (index, pool) in fixture.pools.iter().enumerate() {
24138 pool.inner.stored.lock().expect("fixture").as_mut().expect("replica").0 = if index == committed_pool {
24139 fixture.durable.clone()
24140 } else {
24141 fixture.pending.clone()
24142 };
24143 }
24144 let mut state = fixture.state();
24145 let selected = implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state)
24146 .await
24147 .expect("sole commit must select new generation");
24148 assert_eq!(selected.revision, fixture.revision);
24149 assert_eq!(persisted_pools(&selected.meta), persisted_pools(&fixture.desired));
24150 let outcome = implementation::repair_pool_meta_transaction(
24151 fixture.pools.clone(),
24152 &mut state,
24153 selected,
24154 &PoolMetaPersistenceFence::Distributed(None),
24155 )
24156 .await
24157 .expect("repair partial commit");
24158 assert_eq!(persisted_pools(&outcome.committed), persisted_pools(&fixture.desired));
24159 outcome.disarm();
24160 let confirmed = implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state)
24161 .await
24162 .expect("repaired sole commit reread");
24163 assert_eq!(persisted_pools(&confirmed.meta), persisted_pools(&fixture.desired));
24164 assert!(!confirmed.replica_state.needs_repair);
24165 let fork = implementation::encode_pool_meta_v3_envelope(
24166 &fixture.desired,
24167 implementation::PoolMetaRevision {
24168 transaction_id: Some(uuid::Uuid::new_v4()),
24169 ..fixture.revision
24170 },
24171 true,
24172 None,
24173 )
24174 .expect("same generation fork");
24175 assert!(
24176 implementation::select_pool_meta_replica(vec![
24177 implementation::decode_pool_meta_replica(fixture.durable.clone()),
24178 implementation::decode_pool_meta_replica(fork)
24179 ])
24180 .is_err()
24181 );
24182 }
24183 }
24184 }
24185
24186 #[tokio::test]
24187 async fn test_pool_meta_cas_deterministically_rejects_stale_writer() {
24188 let storage = Arc::new(PartialPoolMetaWriteStorage {
24189 fail_write: false,
24190 fail_after_first_write: false,
24191 pending_write: false,
24192 write_started: tokio::sync::Notify::new(),
24193 wrote: AtomicBool::new(false),
24194 revision: AtomicUsize::new(0),
24195 stored: StdMutex::new(None),
24196 identity: StdMutex::new(None),
24197 });
24198 let stale_token = read_pool_meta_replica(storage.clone(), true).await.cas;
24199 assert!(matches!(&stale_token, PoolMetaCasToken::Missing));
24200 let winner = PoolMeta {
24201 version: POOL_META_VERSION,
24202 pools: vec![decommission_test_pool_status(0, None)],
24203 ..Default::default()
24204 }
24205 .encode_config_data_for_test()
24206 .expect("winning pool metadata should encode");
24207 let stale = PoolMeta {
24208 version: POOL_META_VERSION,
24209 pools: vec![decommission_test_pool_status(
24210 0,
24211 Some(PoolDecommissionInfo {
24212 complete: true,
24213 ..Default::default()
24214 }),
24215 )],
24216 ..Default::default()
24217 }
24218 .encode_config_data_for_test()
24219 .expect("stale pool metadata should encode");
24220 let fence = PoolMetaPersistenceFence::Distributed(None);
24221
24222 let mut arm = super::PoolMetaWriteState::default().arm_transaction();
24223 save_pool_meta_object_cas(
24224 storage.clone(),
24225 POOL_META_NAME,
24226 winner.clone(),
24227 &stale_token,
24228 &fence,
24229 "prepare_cas",
24230 &mut arm,
24231 )
24232 .await
24233 .expect("the first writer should create pool metadata");
24234 let err =
24235 save_pool_meta_object_cas(storage.clone(), POOL_META_NAME, stale, &stale_token, &fence, "prepare_cas", &mut arm)
24236 .await
24237 .expect_err("the second writer must not reuse the stale missing-object revision");
24238
24239 assert_eq!(err, Error::PreconditionFailed);
24240 let stored = storage
24241 .stored
24242 .lock()
24243 .expect("pool metadata test storage should not be poisoned")
24244 .as_ref()
24245 .map(|(data, _)| data.clone())
24246 .expect("the winning pool metadata should remain stored");
24247 assert_eq!(stored, winner);
24248 }
24249
24250 #[tokio::test]
24251 async fn test_pool_meta_v3_single_replica_commit_failure_recovers_on_restart() {
24252 let committed_replica = Arc::new(PartialPoolMetaWriteStorage {
24253 fail_write: false,
24254 fail_after_first_write: false,
24255 pending_write: false,
24256 write_started: tokio::sync::Notify::new(),
24257 wrote: AtomicBool::new(false),
24258 revision: AtomicUsize::new(0),
24259 stored: StdMutex::new(None),
24260 identity: StdMutex::new(None),
24261 });
24262 let pending_replica = Arc::new(PartialPoolMetaWriteStorage {
24263 fail_write: false,
24264 fail_after_first_write: true,
24265 pending_write: false,
24266 write_started: tokio::sync::Notify::new(),
24267 wrote: AtomicBool::new(false),
24268 revision: AtomicUsize::new(0),
24269 stored: StdMutex::new(None),
24270 identity: StdMutex::new(None),
24271 });
24272 let cluster_id = uuid::Uuid::new_v4();
24273 let fresh_bootstrap_nonce = uuid::Uuid::new_v4();
24274 let identity = super::encode_pool_meta_identity(super::PersistedPoolMetaIdentity {
24275 version: super::POOL_META_IDENTITY_VERSION,
24276 cluster_id,
24277 epoch: super::POOL_META_INITIAL_EPOCH,
24278 initialized: false,
24279 fresh_bootstrap_nonce: Some(fresh_bootstrap_nonce),
24280 })
24281 .expect("pending bootstrap identity should encode");
24282 *committed_replica
24283 .identity
24284 .lock()
24285 .expect("identity storage should not be poisoned") = Some((identity.clone(), "identity-0".to_string()));
24286 *pending_replica
24287 .identity
24288 .lock()
24289 .expect("identity storage should not be poisoned") = Some((identity, "identity-0".to_string()));
24290 let mut write_state = super::PoolMetaWriteState::for_startup(cluster_id, true);
24291 let snapshot = PoolMeta {
24292 version: super::POOL_META_GENERATION_VERSION,
24293 pools: vec![decommission_test_pool_status(
24294 0,
24295 Some(PoolDecommissionInfo {
24296 queued: true,
24297 ..Default::default()
24298 }),
24299 )],
24300 ..Default::default()
24301 };
24302
24303 snapshot
24304 .save_no_lock_observing(vec![committed_replica.clone(), pending_replica.clone()], &mut write_state)
24305 .await
24306 .expect("one committed replica should durably complete the V3 transaction");
24307
24308 let mut restarted = PoolMeta::default();
24309 let replica_state = restarted
24310 .load_no_lock_from_replicas(vec![committed_replica, pending_replica.clone()])
24311 .await
24312 .expect("restart should select the committed generation over a pending replica");
24313 assert_eq!(restarted.version, super::POOL_META_GENERATION_VERSION);
24314 assert!(restarted.pools[0].decommission.as_ref().is_some_and(|info| info.queued));
24315 assert!(replica_state.needs_repair);
24316
24317 let mut pending_only = PoolMeta::default();
24318 pending_only
24319 .load_no_lock_from_replicas(vec![pending_replica])
24320 .await
24321 .expect("a prepare-only replica should expose the embedded predecessor");
24322 assert!(pending_only.pools.is_empty());
24323 }
24324
24325 #[tokio::test]
24326 async fn test_partial_pool_meta_save_failure_blocks_following_side_effect() {
24327 let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
24328 let committed = Arc::new(PartialPoolMetaWriteStorage {
24329 fail_write: false,
24330 fail_after_first_write: false,
24331 pending_write: false,
24332 write_started: tokio::sync::Notify::new(),
24333 wrote: AtomicBool::new(false),
24334 revision: AtomicUsize::new(0),
24335 stored: StdMutex::new(None),
24336 identity: StdMutex::new(None),
24337 });
24338 let failed = Arc::new(PartialPoolMetaWriteStorage {
24339 fail_write: true,
24340 fail_after_first_write: false,
24341 pending_write: false,
24342 write_started: tokio::sync::Notify::new(),
24343 wrote: AtomicBool::new(false),
24344 revision: AtomicUsize::new(0),
24345 stored: StdMutex::new(None),
24346 identity: StdMutex::new(None),
24347 });
24348 let snapshot = PoolMeta {
24349 version: super::POOL_META_VERSION,
24350 pools: vec![decommission_test_pool_status(0, None)],
24351 ..Default::default()
24352 };
24353
24354 {
24355 let mut save_guard = store.pool_meta_save_gate.lock().await;
24356 snapshot
24357 .save_no_lock_observing(vec![committed.clone(), failed], &mut save_guard)
24358 .await
24359 .expect_err("the second replica write should fail after the first commits");
24360 }
24361 assert!(committed.wrote.load(Ordering::SeqCst));
24362
24363 let ran = Arc::new(AtomicBool::new(false));
24364 let ran_by_operation = ran.clone();
24365 let movement_gate = store.ctx.data_movement_operation_gate();
24366 let result: std::result::Result<(), Error> = store
24367 .run_guarded_decommission_side_effect(&CancellationToken::new(), &movement_gate, move || async move {
24368 ran_by_operation.store(true, Ordering::SeqCst);
24369 Ok(())
24370 })
24371 .await;
24372
24373 assert!(result.is_err(), "a partial pool metadata save must latch the sticky safety gate");
24374 assert!(!ran.load(Ordering::SeqCst), "the side effect must not run after a partial save");
24375 }
24376
24377 #[tokio::test]
24378 async fn test_cancelled_pool_meta_save_blocks_following_side_effect() {
24379 let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
24380 let committed = Arc::new(PartialPoolMetaWriteStorage {
24381 fail_write: false,
24382 fail_after_first_write: false,
24383 pending_write: false,
24384 write_started: tokio::sync::Notify::new(),
24385 wrote: AtomicBool::new(false),
24386 revision: AtomicUsize::new(0),
24387 stored: StdMutex::new(None),
24388 identity: StdMutex::new(None),
24389 });
24390 let pending = Arc::new(PartialPoolMetaWriteStorage {
24391 fail_write: false,
24392 fail_after_first_write: false,
24393 pending_write: true,
24394 write_started: tokio::sync::Notify::new(),
24395 wrote: AtomicBool::new(false),
24396 revision: AtomicUsize::new(0),
24397 stored: StdMutex::new(None),
24398 identity: StdMutex::new(None),
24399 });
24400 let snapshot = PoolMeta {
24401 version: super::POOL_META_VERSION,
24402 pools: vec![decommission_test_pool_status(0, None)],
24403 ..Default::default()
24404 };
24405 let save_store = store.clone();
24406 let save_committed = committed.clone();
24407 let save_pending = pending.clone();
24408 let save_task = tokio::spawn(async move {
24409 let mut save_guard = save_store.pool_meta_save_gate.lock().await;
24410 snapshot
24411 .save_no_lock_observing(vec![save_committed, save_pending], &mut save_guard)
24412 .await
24413 });
24414
24415 tokio::time::timeout(StdDuration::from_secs(1), pending.write_started.notified())
24416 .await
24417 .expect("the second replica write should start");
24418 assert!(committed.wrote.load(Ordering::SeqCst));
24419 save_task.abort();
24420 assert!(
24421 save_task.await.expect_err("the save task should be cancelled").is_cancelled(),
24422 "the pending replica write should be aborted"
24423 );
24424
24425 {
24426 let save_guard = store.pool_meta_save_gate.lock().await;
24427 save_guard
24428 .ensure_write_safe("cancelled pool metadata save")
24429 .expect_err("a cancelled replica update must latch the sticky safety gate");
24430 }
24431
24432 let ran = Arc::new(AtomicBool::new(false));
24433 let ran_by_operation = ran.clone();
24434 let movement_gate = store.ctx.data_movement_operation_gate();
24435 let result: std::result::Result<(), Error> = store
24436 .run_guarded_decommission_side_effect(&CancellationToken::new(), &movement_gate, move || async move {
24437 ran_by_operation.store(true, Ordering::SeqCst);
24438 Ok(())
24439 })
24440 .await;
24441
24442 assert!(result.is_err(), "a cancelled pool metadata save must block following side effects");
24443 assert!(!ran.load(Ordering::SeqCst), "the side effect must not run after a cancelled save");
24444 }
24445
24446 #[tokio::test]
24447 async fn test_cancelled_pool_meta_publish_keeps_write_gate_blocked() {
24448 let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
24449 let committed = Arc::new(PartialPoolMetaWriteStorage {
24450 fail_write: false,
24451 fail_after_first_write: false,
24452 pending_write: false,
24453 write_started: tokio::sync::Notify::new(),
24454 wrote: AtomicBool::new(false),
24455 revision: AtomicUsize::new(0),
24456 stored: StdMutex::new(None),
24457 identity: StdMutex::new(None),
24458 });
24459 let snapshot = PoolMeta {
24460 version: super::POOL_META_VERSION,
24461 pools: vec![decommission_test_pool_status(0, None)],
24462 ..Default::default()
24463 };
24464 let publish_started = Arc::new(tokio::sync::Notify::new());
24465 let publish_release = Arc::new(tokio::sync::Notify::new());
24466 let save_store = store.clone();
24467 let save_committed = committed.clone();
24468 let task_publish_started = publish_started.clone();
24469 let task_publish_release = publish_release.clone();
24470 let save_task = tokio::spawn(async move {
24471 let mut save_guard = save_store.pool_meta_save_gate.lock().await;
24472 let outcome = snapshot
24473 .save_no_lock_armed(vec![save_committed], &mut save_guard, None, &[0])
24474 .await?;
24475 task_publish_started.notify_one();
24476 task_publish_release.notified().await;
24477 outcome.disarm();
24478 Ok::<(), Error>(())
24479 });
24480
24481 tokio::time::timeout(StdDuration::from_secs(1), publish_started.notified())
24482 .await
24483 .expect("the replica save should complete before publication");
24484 assert!(committed.wrote.load(Ordering::SeqCst));
24485 save_task.abort();
24486 assert!(
24487 save_task
24488 .await
24489 .expect_err("the publication task should be cancelled")
24490 .is_cancelled(),
24491 "the task should be aborted while publication is pending"
24492 );
24493
24494 let save_guard = store.pool_meta_save_gate.lock().await;
24495 save_guard
24496 .ensure_write_safe("cancelled pool metadata publication")
24497 .expect_err("cancellation after replica save but before publication must keep writes blocked");
24498 }
24499
24500 #[tokio::test]
24501 async fn test_lost_pool_meta_fence_rejects_replica_write() {
24502 let client = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
24503 let lock = NamespaceLock::with_clients_and_quorum("pool-meta-fence-loss".to_string(), vec![client], 1);
24504 let request = LockRequest::new(
24505 ObjectKey::new(super::RUSTFS_META_BUCKET, super::POOL_META_NAME),
24506 LockType::Exclusive,
24507 "stale-writer",
24508 )
24509 .with_acquire_timeout(StdDuration::from_secs(1))
24510 .with_ttl(StdDuration::from_millis(50))
24511 .with_refresh_interval(StdDuration::from_millis(50));
24512 let guard = lock
24513 .acquire_guard(&request)
24514 .await
24515 .expect("pool metadata fence acquisition should not error")
24516 .expect("the stale writer should acquire the pool metadata fence");
24517 tokio::time::timeout(StdDuration::from_secs(2), guard.lock_lost_notified())
24518 .await
24519 .expect("the stale writer lease should expire");
24520
24521 let storage = Arc::new(PartialPoolMetaWriteStorage {
24522 fail_write: false,
24523 fail_after_first_write: false,
24524 pending_write: false,
24525 write_started: tokio::sync::Notify::new(),
24526 wrote: AtomicBool::new(false),
24527 revision: AtomicUsize::new(0),
24528 stored: StdMutex::new(None),
24529 identity: StdMutex::new(None),
24530 });
24531 let snapshot = PoolMeta {
24532 version: super::POOL_META_VERSION,
24533 pools: vec![decommission_test_pool_status(0, None)],
24534 ..Default::default()
24535 };
24536 let mut write_state = super::PoolMetaWriteState::default();
24537 let err = snapshot
24538 .save_no_lock_armed(vec![storage.clone()], &mut write_state, guard.lock_lost_signal(), &[0])
24539 .await
24540 .expect_err("a writer must not persist after losing the distributed pool metadata fence");
24541
24542 assert!(err.to_string().contains("distributed fence was lost"));
24543 assert!(!storage.wrote.load(Ordering::SeqCst), "the stale writer must not reach replica storage");
24544 write_state
24545 .ensure_write_safe("lost pool metadata fence")
24546 .expect("fence loss before dispatch has no unknown durable write and must remain retryable");
24547 }
24548
24549 #[tokio::test]
24550 async fn test_pool_meta_fence_loss_after_publish_keeps_write_gate_blocked() {
24551 let client = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
24552 let lock = NamespaceLock::with_clients_and_quorum("pool-meta-publish-fence-loss".to_string(), vec![client], 1);
24553 let request = LockRequest::new(
24554 ObjectKey::new(super::RUSTFS_META_BUCKET, super::POOL_META_NAME),
24555 LockType::Exclusive,
24556 "publishing-writer",
24557 )
24558 .with_acquire_timeout(StdDuration::from_secs(1))
24559 .with_ttl(StdDuration::from_secs(1))
24560 .with_refresh_interval(StdDuration::from_secs(1));
24561 let guard = lock
24562 .acquire_guard(&request)
24563 .await
24564 .expect("pool metadata fence acquisition should not error")
24565 .expect("the publishing writer should acquire the pool metadata fence");
24566 let storage = Arc::new(PartialPoolMetaWriteStorage {
24567 fail_write: false,
24568 fail_after_first_write: false,
24569 pending_write: false,
24570 write_started: tokio::sync::Notify::new(),
24571 wrote: AtomicBool::new(false),
24572 revision: AtomicUsize::new(0),
24573 stored: StdMutex::new(None),
24574 identity: StdMutex::new(None),
24575 });
24576 let mut saved = PoolMeta {
24577 version: super::POOL_META_VERSION,
24578 pools: vec![decommission_test_pool_status(0, None)],
24579 ..Default::default()
24580 };
24581 saved.pools[0].last_update = OffsetDateTime::UNIX_EPOCH + Duration::seconds(1);
24582 let mut current = saved.clone();
24583 current.pools[0].last_update = OffsetDateTime::UNIX_EPOCH;
24584 let mut write_state = super::PoolMetaWriteState::for_test_bootstrap();
24585 let outcome = saved
24586 .save_no_lock_armed(vec![storage], &mut write_state, guard.lock_lost_signal(), &[0])
24587 .await
24588 .expect("the replica save should finish while the fence is valid");
24589
24590 ensure_pool_meta_write_fence(&guard, "test pool metadata publish")
24591 .expect("the fence should remain valid before publication");
24592 publish_pool_meta_updates(&mut current, &saved, &[0]);
24593 tokio::time::timeout(StdDuration::from_secs(2), guard.lock_lost_notified())
24594 .await
24595 .expect("the fence should expire after publication");
24596 ensure_pool_meta_write_fence(&guard, "test pool metadata publish")
24597 .expect_err("the post-publication fence check must observe the loss");
24598
24599 assert_eq!(current.pools[0].last_update, saved.pools[0].last_update);
24600 drop(outcome);
24601 write_state
24602 .ensure_write_safe("lost pool metadata publish fence")
24603 .expect_err("the sticky write gate must remain armed after post-publication fence loss");
24604 }
24605
24606 #[tokio::test]
24607 async fn test_clear_decommission_transaction_survives_caller_abort() {
24608 let pool_meta = PoolMeta {
24609 version: super::POOL_META_VERSION,
24610 pools: vec![decommission_test_pool_status(
24611 0,
24612 Some(PoolDecommissionInfo {
24613 failed: true,
24614 ..Default::default()
24615 }),
24616 )],
24617 ..Default::default()
24618 };
24619 let store = decommission_worker_test_store(pool_meta, vec![None]);
24620 let save_started = Arc::new(tokio::sync::Notify::new());
24621 let save_release = Arc::new(tokio::sync::Notify::new());
24622 let save_completed = Arc::new(AtomicBool::new(false));
24623 let caller_store = store.clone();
24624 let caller_save_started = save_started.clone();
24625 let caller_save_release = save_release.clone();
24626 let caller_save_completed = save_completed.clone();
24627 let caller_task = tokio::spawn(async move {
24628 caller_store
24629 .clear_decommission_with_save(0, move || async move {
24630 caller_save_started.notify_one();
24631 caller_save_release.notified().await;
24632 caller_save_completed.store(true, Ordering::SeqCst);
24633 Ok(())
24634 })
24635 .await
24636 });
24637
24638 tokio::time::timeout(StdDuration::from_secs(1), save_started.notified())
24639 .await
24640 .expect("the clear transaction should reach persistence");
24641 caller_task.abort();
24642 assert!(
24643 caller_task
24644 .await
24645 .expect_err("the RPC waiter should be cancelled")
24646 .is_cancelled(),
24647 "the clear caller should be aborted while persistence is pending"
24648 );
24649 save_release.notify_one();
24650
24651 let _start_guard = tokio::time::timeout(StdDuration::from_secs(1), store.start_gate.lock())
24652 .await
24653 .expect("the detached clear transaction should finish");
24654 assert!(
24655 save_completed.load(Ordering::SeqCst),
24656 "the detached transaction must finish persistence after the caller is aborted"
24657 );
24658 let pool_meta = store.pool_meta.read().await;
24659 assert!(
24660 pool_meta.pools[0]
24661 .decommission
24662 .as_ref()
24663 .is_some_and(|info| !info.has_decommission_state())
24664 );
24665 }
24666
24667 fn decommission_test_pool_endpoint(idx: usize, is_local: bool) -> PoolEndpoints {
24668 let port = 9000usize + idx;
24669 let mut endpoint =
24670 Endpoint::try_from(format!("http://127.0.0.1:{port}/disk").as_str()).expect("test endpoint should parse");
24671 endpoint.is_local = is_local;
24672 endpoint.pool_idx = i32::try_from(idx).expect("test pool index should fit i32");
24673
24674 PoolEndpoints {
24675 legacy: false,
24676 set_count: 1,
24677 drives_per_set: 1,
24678 endpoints: Endpoints::from(vec![endpoint]),
24679 cmd_line: format!("pool-{idx}"),
24680 platform: String::new(),
24681 }
24682 }
24683
24684 fn decommission_test_pool_status(idx: usize, decommission: Option<PoolDecommissionInfo>) -> PoolStatus {
24685 PoolStatus {
24686 id: idx,
24687 cmd_line: format!("pool-{idx}"),
24688 last_update: OffsetDateTime::now_utc(),
24689 decommission,
24690 }
24691 }
24692
24693 fn decommission_test_active_model_meta(model_versions: &[u16]) -> PoolMeta {
24694 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
24695 PoolMeta {
24696 version: POOL_META_VERSION,
24697 pools: model_versions
24698 .iter()
24699 .copied()
24700 .enumerate()
24701 .map(|(idx, model_version)| {
24702 let reservation = build_decommission_capacity_reservation_with_model(
24703 DecommissionPoolCapacityInfo::for_test(idx, layout, 0, 10, 10),
24704 layout,
24705 uuid::Uuid::new_v4(),
24706 u64::try_from(idx).unwrap_or_default() + 1,
24707 OffsetDateTime::UNIX_EPOCH,
24708 model_version,
24709 )
24710 .expect("test capacity model should be supported");
24711 decommission_test_pool_status(
24712 idx,
24713 Some(PoolDecommissionInfo {
24714 start_time: Some(OffsetDateTime::UNIX_EPOCH),
24715 capacity_reservation: Some(reservation),
24716 ..Default::default()
24717 }),
24718 )
24719 })
24720 .collect(),
24721 ..Default::default()
24722 }
24723 }
24724
24725 fn decommission_test_cleanup_meta(
24726 model_version: u16,
24727 temporary_mutations: Vec<DecommissionCapacityTemporaryMutation>,
24728 inflight_physical_bytes: usize,
24729 pending: Option<(uuid::Uuid, usize)>,
24730 ) -> PoolMeta {
24731 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
24732 let mut reservation = build_decommission_capacity_reservation_with_model(
24733 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
24734 layout,
24735 uuid::Uuid::new_v4(),
24736 1,
24737 OffsetDateTime::UNIX_EPOCH,
24738 model_version,
24739 )
24740 .expect("test cleanup reservation should be valid");
24741 let (pending_mutation_id, pending_physical_bytes) = pending.unzip();
24742 let pending_physical_bytes = pending_physical_bytes.unwrap_or_default();
24743 reservation.observed_target_physical_bytes = inflight_physical_bytes;
24744 reservation.inflight_target_physical_bytes = inflight_physical_bytes;
24745 reservation.pending_target_physical_bytes = pending_physical_bytes;
24746 reservation.targets.push(DecommissionCapacityTarget {
24747 pool_index: 1,
24748 layout,
24749 physical_total_at_reservation: 200,
24750 physical_free_at_reservation: 200,
24751 reserved_physical_bytes: reservation.peak_physical_bytes,
24752 consumed_physical_bytes: 0,
24753 observed_physical_bytes: inflight_physical_bytes,
24754 inflight_physical_bytes,
24755 pending_physical_bytes,
24756 pending_mutation_id,
24757 temporary_mutations,
24758 });
24759 PoolMeta {
24760 version: POOL_META_VERSION,
24761 pools: vec![
24762 decommission_test_pool_status(
24763 0,
24764 Some(PoolDecommissionInfo {
24765 start_time: Some(OffsetDateTime::UNIX_EPOCH),
24766 capacity_reservation: Some(reservation),
24767 ..Default::default()
24768 }),
24769 ),
24770 decommission_test_pool_status(1, None),
24771 ],
24772 ..Default::default()
24773 }
24774 }
24775
24776 #[tokio::test]
24777 async fn test_activation_fence_uses_one_lock_order_and_serializes_callers() {
24778 let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
24779 let first = Arc::new(ActivationLockRecorder {
24780 lock_manager: Arc::clone(&manager),
24781 owner: "first",
24782 resources: StdMutex::new(Vec::new()),
24783 });
24784 let second = Arc::new(ActivationLockRecorder {
24785 lock_manager: manager,
24786 owner: "second",
24787 resources: StdMutex::new(Vec::new()),
24788 });
24789
24790 let first_guards = acquire_pool_rebalance_activation_locks(first.clone(), None)
24791 .await
24792 .expect("first activation should acquire both locks");
24793 assert_eq!(
24794 *first
24795 .resources
24796 .lock()
24797 .expect("activation lock recorder should not be poisoned"),
24798 vec![REBAL_META_NAME.to_string(), POOL_META_NAME.to_string()]
24799 );
24800
24801 let mut second_acquire = Box::pin(acquire_pool_rebalance_activation_locks(second.clone(), None));
24802 let mut context = Context::from_waker(futures::task::noop_waker_ref());
24803 assert!(matches!(second_acquire.as_mut().poll(&mut context), Poll::Pending));
24804
24805 drop(first_guards);
24806 second_acquire
24807 .await
24808 .expect("second activation should acquire both locks after the first releases them");
24809 assert_eq!(
24810 *second
24811 .resources
24812 .lock()
24813 .expect("activation lock recorder should not be poisoned"),
24814 vec![REBAL_META_NAME.to_string(), POOL_META_NAME.to_string()]
24815 );
24816 }
24817
24818 #[tokio::test]
24819 async fn test_activation_cancellation_releases_rebalance_fence_while_pool_fence_is_contended() {
24820 use crate::storage_api_contracts::namespace::NamespaceLocking as _;
24821 let pool = Arc::new(ActivationLockRecorder {
24822 lock_manager: Arc::new(rustfs_lock::GlobalLockManager::new()),
24823 owner: "activation-cancellation",
24824 resources: StdMutex::new(Vec::new()),
24825 });
24826 let pool_lock = pool
24827 .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, POOL_META_NAME)
24828 .await
24829 .expect("pool lock should be created");
24830 let pool_reader = pool_lock
24831 .get_read_lock(std::time::Duration::from_secs(5))
24832 .await
24833 .expect("ordinary mutation should hold the pool read fence");
24834 pool.resources.lock().expect("recorder should not be poisoned").clear();
24835 let mut activation = Box::pin(acquire_pool_rebalance_activation_locks(Arc::clone(&pool), None));
24836 assert!(matches!(futures::poll!(&mut activation), Poll::Pending));
24837 assert_eq!(
24838 *pool.resources.lock().expect("recorder should not be poisoned"),
24839 vec![REBAL_META_NAME.to_string(), POOL_META_NAME.to_string()],
24840 "activation must hold the run fence before waiting for the pool fence",
24841 );
24842 drop(activation);
24843 let rebalance_lock = pool
24844 .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME)
24845 .await
24846 .expect("run lock should be created");
24847 let run_writer = rebalance_lock
24848 .get_write_lock(std::time::Duration::from_secs(5))
24849 .await
24850 .expect("cancelling activation must release its already-acquired run fence");
24851 assert!(
24852 !pool_reader.is_released(),
24853 "cancelling activation must not release another caller's pool fence"
24854 );
24855 assert!(!run_writer.is_lock_lost());
24856 }
24857
24858 #[test]
24859 fn decommission_receipt_run_token_changes_with_persisted_start_time() {
24860 let first = OffsetDateTime::from_unix_timestamp(1_000).expect("first run timestamp should be valid");
24861 let second = OffsetDateTime::from_unix_timestamp(2_000).expect("second run timestamp should be valid");
24862 let first_token = decommission_durable_ilm_receipt_run_token("pool-0", first);
24863 let second_token = decommission_durable_ilm_receipt_run_token("pool-0", second);
24864
24865 assert_ne!(first_token, second_token);
24866 assert_eq!(first_token, decommission_durable_ilm_receipt_run_token("pool-0", first));
24867 let operation_id = "a".repeat(64);
24868 let old_receipt = decommission_durable_ilm_receipt_path(
24869 &first_token,
24870 &format!("ilm/tier-delete-journal/{operation_id}.json"),
24871 "operation_id",
24872 &operation_id,
24873 );
24874 assert!(!old_receipt.starts_with(&decommission_durable_ilm_receipt_run_prefix(&second_token)));
24875 }
24876
24877 #[test]
24878 fn decommission_recovery_control_receipt_path_round_trips() {
24879 let run_token = "b".repeat(64);
24880 let control_id = "a".repeat(64);
24881 let source_path = format!(
24882 "ilm/recovery-controls/transition_transaction/{}/{}/{}.json",
24883 &control_id[..2],
24884 &control_id[2..4],
24885 control_id
24886 );
24887 let path = decommission_durable_ilm_receipt_path(&run_token, &source_path, "control_id", &control_id);
24888
24889 let locator = parse_decommission_durable_ilm_receipt_path(&path).expect("recovery control receipt path should parse");
24890
24891 assert_eq!(locator.run_token, run_token);
24892 assert_eq!(locator.source_path, source_path);
24893 assert_eq!(locator.id_kind, "control_id");
24894 assert_eq!(locator.id, control_id);
24895 }
24896
24897 #[test]
24898 fn decommission_receipt_merge_preserves_terminal_proof() {
24899 let operation_id = "a".repeat(64);
24900 let source_path = format!("ilm/tier-delete-journal/{operation_id}.json");
24901 let checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal {
24902 content_sha256: "b".repeat(64),
24903 identity_sha256: "c".repeat(64),
24904 committed: false,
24905 dispatch_identity_sha256: None,
24906 state: None,
24907 };
24908 let terminal_checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal {
24909 content_sha256: "d".repeat(64),
24910 identity_sha256: "c".repeat(64),
24911 committed: true,
24912 dispatch_identity_sha256: None,
24913 state: None,
24914 };
24915 let incoming = DecommissionDurableIlmReceipt {
24916 source_path,
24917 namespace: "tier-delete-journal".to_string(),
24918 id_kind: "operation_id".to_string(),
24919 id: operation_id,
24920 checkpoint: checkpoint.clone(),
24921 terminal_checkpoint: None,
24922 fleet_topology_generation: None,
24923 };
24924 let existing = DecommissionDurableIlmReceipt {
24925 terminal_checkpoint: Some(terminal_checkpoint.clone()),
24926 ..incoming.clone()
24927 };
24928
24929 let merged = merge_decommission_durable_ilm_receipts(&existing, &incoming)
24930 .expect("retry receipt must merge with a terminal receipt");
24931
24932 assert_eq!(merged.checkpoint, checkpoint);
24933 assert_eq!(merged.terminal_checkpoint, Some(terminal_checkpoint.clone()));
24934
24935 let topology_bound = DecommissionDurableIlmReceipt {
24936 fleet_topology_generation: Some("e".repeat(64)),
24937 ..incoming
24938 };
24939 let mixed_error = merge_decommission_durable_ilm_receipts(&existing, &topology_bound)
24940 .expect_err("a topology-bound v6 receipt must not mask an unbound receipt")
24941 .to_string();
24942 assert!(mixed_error.contains("fleet topology conflict"));
24943
24944 let topology_existing = DecommissionDurableIlmReceipt {
24945 terminal_checkpoint: Some(terminal_checkpoint),
24946 ..topology_bound.clone()
24947 };
24948 let topology_merged = merge_decommission_durable_ilm_receipts(&topology_existing, &topology_bound)
24949 .expect("receipts bound to the same fleet topology should merge");
24950 assert_eq!(topology_merged.fleet_topology_generation, Some("e".repeat(64)));
24951 }
24952
24953 #[test]
24954 fn decommission_manual_job_receipt_compacts_large_progress() {
24955 let prefix = "p".repeat(12 * 1024);
24956 let options = ManualTransitionRunOptions {
24957 prefix,
24958 ..Default::default()
24959 };
24960 let mut job = ManualTransitionJobRecord::new(uuid::Uuid::new_v4(), "bounded-receipt-bucket", &options, "owner");
24961 let token_bytes = serde_json::to_vec(&serde_json::json!({
24962 "marker": "m".repeat(12 * 1024),
24963 "version_marker": "opaque-version"
24964 }))
24965 .expect("large continuation token should encode");
24966 let mut report = job.report.clone();
24967 report.scanned = 1;
24968 report.continuation_token = Some(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&token_bytes));
24969 job.update_running_progress(report, ManualTransitionQueueSnapshot::default());
24970 let path = manual_transition_job_record_object_name(job.job_id).expect("manual job path should build");
24971 let job_bytes = job.encode().expect("large manual job should remain within its record limit");
24972 assert!(job_bytes.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE);
24973 let record = validate_durable_ilm_record(&path, &job_bytes).expect("large manual job should validate");
24974 let expected_checkpoint = record.checkpoint.clone();
24975 let mut receipt = DecommissionDurableIlmReceipt::new(&path, &record, None);
24976 receipt.terminal_checkpoint = Some(record.checkpoint);
24977
24978 let encoded = receipt.encode().expect("bounded progress proof should fit the receipt limit");
24979 let decoded = DecommissionDurableIlmReceipt::decode(&encoded).expect("bounded receipt should round trip");
24980
24981 assert!(encoded.len() <= DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE);
24982 assert_eq!(decoded.source_path, path);
24983 assert_eq!(decoded.checkpoint, expected_checkpoint);
24984 assert_eq!(decoded.terminal_checkpoint, Some(expected_checkpoint));
24985 }
24986
24987 #[test]
24988 fn test_apply_decommission_status_space_info_adds_idle_pool_usage() {
24989 let status = apply_decommission_status_space_info(
24990 decommission_test_pool_status(0, None),
24991 PoolSpaceInfo {
24992 free: 25,
24993 total: 100,
24994 used: 75,
24995 },
24996 );
24997
24998 let decommission = status.decommission.expect("idle pool status should include usage info");
24999 assert_eq!(decommission.total_size, 100);
25000 assert_eq!(decommission.current_size, 25);
25001 assert!(decommission.start_time.is_none());
25002 assert!(!decommission.complete);
25003 assert!(!decommission.failed);
25004 assert!(!decommission.canceled);
25005 }
25006
25007 #[test]
25008 fn test_apply_decommission_status_space_info_refreshes_active_decommission_sizes() {
25009 let status = apply_decommission_status_space_info(
25010 decommission_test_pool_status(
25011 0,
25012 Some(PoolDecommissionInfo {
25013 total_size: 1,
25014 current_size: 1,
25015 ..Default::default()
25016 }),
25017 ),
25018 PoolSpaceInfo {
25019 free: 25,
25020 total: 100,
25021 used: 75,
25022 },
25023 );
25024
25025 let decommission = status.decommission.expect("active decommission info should remain present");
25026 assert_eq!(decommission.total_size, 100);
25027 assert_eq!(decommission.current_size, 25);
25028 }
25029
25030 #[test]
25031 fn test_merge_pool_status_refresh_uses_persisted_terminal_decommission() {
25032 let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
25033 let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
25034 let mut current = PoolMeta {
25035 pools: vec![PoolStatus {
25036 id: 0,
25037 cmd_line: "pool-0".to_string(),
25038 last_update: older,
25039 decommission: Some(PoolDecommissionInfo {
25040 start_time: Some(older),
25041 ..Default::default()
25042 }),
25043 }],
25044 ..Default::default()
25045 };
25046 let persisted = PoolMeta {
25047 pools: vec![PoolStatus {
25048 id: 0,
25049 cmd_line: "pool-0".to_string(),
25050 last_update: newer,
25051 decommission: Some(PoolDecommissionInfo {
25052 complete: true,
25053 ..Default::default()
25054 }),
25055 }],
25056 ..Default::default()
25057 };
25058
25059 assert!(merge_pool_status_refresh(&mut current, persisted, &[false]));
25060
25061 let info = current.pools[0]
25062 .decommission
25063 .as_ref()
25064 .expect("decommission info should be present");
25065 assert!(info.complete);
25066 assert!(!info.failed);
25067 assert!(!info.canceled);
25068 }
25069
25070 #[test]
25071 fn test_merge_pool_status_refresh_keeps_newer_local_active_progress() {
25072 let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
25073 let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
25074 let mut current = PoolMeta {
25075 pools: vec![PoolStatus {
25076 id: 0,
25077 cmd_line: "pool-0".to_string(),
25078 last_update: newer,
25079 decommission: Some(PoolDecommissionInfo {
25080 start_time: Some(older),
25081 items_decommissioned: 10,
25082 bytes_done: 1_024,
25083 ..Default::default()
25084 }),
25085 }],
25086 ..Default::default()
25087 };
25088 let persisted = PoolMeta {
25089 pools: vec![PoolStatus {
25090 id: 0,
25091 cmd_line: "pool-0".to_string(),
25092 last_update: older,
25093 decommission: Some(PoolDecommissionInfo {
25094 start_time: Some(older),
25095 items_decommissioned: 1,
25096 bytes_done: 128,
25097 ..Default::default()
25098 }),
25099 }],
25100 ..Default::default()
25101 };
25102
25103 assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
25104
25105 let info = current.pools[0]
25106 .decommission
25107 .as_ref()
25108 .expect("local decommission info should remain present");
25109 assert_eq!(info.items_decommissioned, 10);
25110 assert_eq!(info.bytes_done, 1_024);
25111 }
25112
25113 #[test]
25114 fn test_merge_pool_status_refresh_preserves_observed_v2_floor() {
25115 let timestamp = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
25116 let mut current = PoolMeta {
25117 version: POOL_META_V1_VERSION,
25118 pools: vec![PoolStatus {
25119 id: 0,
25120 cmd_line: "pool-0".to_string(),
25121 last_update: timestamp,
25122 decommission: Some(PoolDecommissionInfo::default()),
25123 }],
25124 dont_save: false,
25125 };
25126 let persisted = PoolMeta {
25127 version: POOL_META_VERSION,
25128 pools: current.pools.clone(),
25129 dont_save: false,
25130 };
25131
25132 assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
25133 assert_eq!(current.version, POOL_META_VERSION);
25134 let encoded = current
25135 .encode_config_data_for_v2_gate(false)
25136 .expect("a peer-observed v2 floor must remain sticky");
25137 assert_eq!(LittleEndian::read_u16(&encoded[2..4]), POOL_META_VERSION);
25138 }
25139
25140 #[test]
25141 fn test_merge_pool_status_refresh_keeps_newer_local_active_over_older_terminal() {
25142 let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
25143 let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
25144 let mut current = PoolMeta {
25145 pools: vec![PoolStatus {
25146 id: 0,
25147 cmd_line: "pool-0".to_string(),
25148 last_update: newer,
25149 decommission: Some(PoolDecommissionInfo {
25150 start_time: Some(older),
25151 items_decommissioned: 10,
25152 bytes_done: 1_024,
25153 ..Default::default()
25154 }),
25155 }],
25156 ..Default::default()
25157 };
25158 let persisted = PoolMeta {
25159 pools: vec![PoolStatus {
25160 id: 0,
25161 cmd_line: "pool-0".to_string(),
25162 last_update: older,
25163 decommission: Some(PoolDecommissionInfo {
25164 failed: true,
25165 ..Default::default()
25166 }),
25167 }],
25168 ..Default::default()
25169 };
25170
25171 assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
25172
25173 let info = current.pools[0]
25174 .decommission
25175 .as_ref()
25176 .expect("local active decommission info should remain present");
25177 assert!(!info.failed);
25178 assert_eq!(info.items_decommissioned, 10);
25179 assert_eq!(info.bytes_done, 1_024);
25180 }
25181
25182 #[test]
25183 fn test_merge_pool_status_refresh_fails_closed_on_missing_persisted_pools() {
25184 let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
25185 let mut current = PoolMeta {
25186 pools: vec![decommission_test_pool_status(
25187 0,
25188 Some(PoolDecommissionInfo {
25189 complete: true,
25190 ..Default::default()
25191 }),
25192 )],
25193 ..Default::default()
25194 };
25195 current.pools[0].last_update = newer;
25196
25197 assert!(
25198 !merge_pool_status_refresh(&mut current, PoolMeta::default(), &[false]),
25199 "an empty persisted snapshot must fail closed instead of replacing local state"
25200 );
25201
25202 let info = current.pools[0]
25203 .decommission
25204 .as_ref()
25205 .expect("local decommission info should survive a missing snapshot");
25206 assert!(info.complete);
25207 assert_eq!(current.pools[0].last_update, newer);
25208 }
25209
25210 #[test]
25211 fn test_merge_pool_status_refresh_ignores_mislabeled_pool_entries() {
25212 let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
25213 let mut current = PoolMeta {
25214 pools: vec![decommission_test_pool_status(0, None)],
25215 ..Default::default()
25216 };
25217 let mut persisted = PoolMeta {
25218 pools: vec![decommission_test_pool_status(0, Some(PoolDecommissionInfo::default()))],
25219 ..Default::default()
25220 };
25221 persisted.pools[0].id = 7;
25222 persisted.pools[0].last_update = older;
25223
25224 assert!(
25225 !merge_pool_status_refresh(&mut current, persisted, &[false]),
25226 "a pool entry whose id does not match its index must be ignored"
25227 );
25228 assert!(current.pools[0].decommission.is_none());
25229 }
25230
25231 #[test]
25232 fn test_pool_meta_save_merge_preserves_newer_untouched_pool() {
25233 let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
25234 let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
25235 let mut current = PoolMeta {
25236 pools: vec![
25237 PoolStatus {
25238 id: 0,
25239 cmd_line: "pool-0".to_string(),
25240 last_update: older,
25241 decommission: Some(PoolDecommissionInfo {
25242 failed: true,
25243 ..Default::default()
25244 }),
25245 },
25246 PoolStatus {
25247 id: 1,
25248 cmd_line: "pool-1".to_string(),
25249 last_update: older,
25250 decommission: None,
25251 },
25252 ],
25253 ..Default::default()
25254 };
25255 let mut persisted = current.clone();
25256 persisted.pools[1].last_update = newer;
25257 persisted.pools[1].decommission = Some(PoolDecommissionInfo {
25258 failed: true,
25259 ..Default::default()
25260 });
25261
25262 assert!(current.clear_decommission(0).expect("terminal decommission should clear"));
25263 merge_pool_meta_updates_for_save(&mut persisted, ¤t, &[0], "clear decommission")
25264 .expect("the target pool update should merge into the latest snapshot");
25265
25266 assert!(
25267 persisted.pools[0]
25268 .decommission
25269 .as_ref()
25270 .is_some_and(|info| !info.has_decommission_state())
25271 );
25272 assert_eq!(persisted.pools[1].last_update, newer);
25273 assert!(persisted.pools[1].decommission.as_ref().is_some_and(|info| info.failed));
25274 }
25275
25276 #[test]
25277 fn test_pool_meta_save_merge_rejects_stale_terminal_resurrection() {
25278 let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
25279 let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
25280 let mut persisted = PoolMeta {
25281 pools: vec![PoolStatus {
25282 id: 0,
25283 cmd_line: "pool-0".to_string(),
25284 last_update: newer,
25285 decommission: Some(PoolDecommissionInfo {
25286 canceled: true,
25287 ..Default::default()
25288 }),
25289 }],
25290 ..Default::default()
25291 };
25292 let stale = PoolMeta {
25293 pools: vec![PoolStatus {
25294 id: 0,
25295 cmd_line: "pool-0".to_string(),
25296 last_update: older,
25297 decommission: Some(PoolDecommissionInfo {
25298 start_time: Some(older),
25299 ..Default::default()
25300 }),
25301 }],
25302 ..Default::default()
25303 };
25304
25305 let err = merge_pool_meta_updates_for_save(&mut persisted, &stale, &[0], "stale writer")
25306 .expect_err("a stale active snapshot must not resurrect a canceled pool");
25307
25308 assert!(err.to_string().contains("stale pool metadata update rejected"));
25309 assert!(persisted.pools[0].decommission.as_ref().is_some_and(|info| info.canceled));
25310 }
25311
25312 #[test]
25313 fn test_pool_meta_save_merge_rejects_clear_over_completed_or_unresolved_state() {
25314 let timestamp = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
25315 for persisted_info in [
25316 PoolDecommissionInfo {
25317 complete: true,
25318 ..Default::default()
25319 },
25320 PoolDecommissionInfo {
25321 failed: true,
25322 unresolved_entries: vec![DecommissionUnresolvedEntry {
25323 bucket: "bucket-a".to_string(),
25324 object: "object-a".to_string(),
25325 pool_index: 0,
25326 set_index: 0,
25327 source_generation: timestamp,
25328 candidate_count: 1,
25329 disk_error_count: 1,
25330 observed_at: timestamp,
25331 reason: "test unresolved entry".to_string(),
25332 }],
25333 ..Default::default()
25334 },
25335 ] {
25336 let mut persisted = PoolMeta {
25337 pools: vec![PoolStatus {
25338 id: 0,
25339 cmd_line: "pool-0".to_string(),
25340 last_update: timestamp,
25341 decommission: Some(persisted_info),
25342 }],
25343 ..Default::default()
25344 };
25345 let cleared = PoolMeta {
25346 pools: vec![PoolStatus {
25347 id: 0,
25348 cmd_line: "pool-0".to_string(),
25349 last_update: OffsetDateTime::now_utc(),
25350 decommission: Some(PoolDecommissionInfo::default()),
25351 }],
25352 ..Default::default()
25353 };
25354
25355 let err = merge_pool_meta_updates_for_save(&mut persisted, &cleared, &[0], "clear decommission")
25356 .expect_err("a stale clear must not erase completed or unresolved state");
25357
25358 assert!(
25359 err.to_string()
25360 .contains("completed or unresolved decommission state cannot be cleared")
25361 );
25362 }
25363 }
25364
25365 #[test]
25366 fn test_pool_meta_publish_preserves_untouched_runtime_progress() {
25367 let mut current = PoolMeta {
25368 pools: vec![
25369 decommission_test_pool_status(0, None),
25370 decommission_test_pool_status(
25371 1,
25372 Some(PoolDecommissionInfo {
25373 items_decommissioned: 10,
25374 bytes_done: 1_024,
25375 ..Default::default()
25376 }),
25377 ),
25378 ],
25379 ..Default::default()
25380 };
25381 let mut saved = current.clone();
25382 saved.pools[0].decommission = Some(PoolDecommissionInfo {
25383 complete: true,
25384 ..Default::default()
25385 });
25386 let saved_pool_1 = saved.pools[1].decommission.as_mut().expect("pool 1 progress should exist");
25387 saved_pool_1.items_decommissioned = 1;
25388 saved_pool_1.bytes_done = 128;
25389
25390 publish_pool_meta_updates(&mut current, &saved, &[0]);
25391
25392 assert!(current.pools[0].decommission.as_ref().is_some_and(|info| info.complete));
25393 let pool_1 = current.pools[1]
25394 .decommission
25395 .as_ref()
25396 .expect("pool 1 progress should remain present");
25397 assert_eq!(pool_1.items_decommissioned, 10);
25398 assert_eq!(pool_1.bytes_done, 1_024);
25399 }
25400
25401 #[test]
25402 fn test_dedup_indices_removes_duplicates_preserving_order() {
25403 assert_eq!(dedup_indices(&[0, 2, 1, 2, 3, 0]), vec![0, 2, 1, 3]);
25404 }
25405
25406 #[test]
25407 fn test_dedup_indices_handles_empty_input() {
25408 let empty: Vec<usize> = Vec::new();
25409 assert!(dedup_indices(&empty).is_empty());
25410 }
25411
25412 #[test]
25413 fn test_default_decommission_bucket_concurrency_is_conservative() {
25414 assert_eq!(default_decommission_bucket_concurrency(0), 1);
25415 assert_eq!(default_decommission_bucket_concurrency(1), 1);
25416 assert_eq!(default_decommission_bucket_concurrency(2), 2);
25417 assert_eq!(default_decommission_bucket_concurrency(8), 4);
25418 }
25419
25420 #[test]
25421 fn test_default_decommission_entry_concurrency_is_conservative() {
25422 assert_eq!(default_decommission_entry_concurrency(0), 1);
25423 assert_eq!(default_decommission_entry_concurrency(1), 1);
25424 assert_eq!(default_decommission_entry_concurrency(4), 4);
25425 assert_eq!(default_decommission_entry_concurrency(16), DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP);
25426 }
25427
25428 #[test]
25429 fn test_decommission_entry_concurrency_clamps_operator_configuration() {
25430 assert_eq!(clamp_decommission_entry_concurrency(0), 1);
25431 assert_eq!(clamp_decommission_entry_concurrency(1), 1);
25432 assert_eq!(
25433 clamp_decommission_entry_concurrency(DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP),
25434 DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP
25435 );
25436 assert_eq!(clamp_decommission_entry_concurrency(usize::MAX), DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP);
25437 }
25438
25439 #[test]
25440 fn test_split_decommission_buckets_keeps_meta_buckets_last() {
25441 let (regular, meta) = split_decommission_buckets(vec![
25442 DecomBucketInfo {
25443 name: "bucket-a".to_string(),
25444 ..Default::default()
25445 },
25446 DecomBucketInfo {
25447 name: crate::disk::RUSTFS_META_BUCKET.to_string(),
25448 prefix: crate::config::com::CONFIG_PREFIX.to_string(),
25449 },
25450 DecomBucketInfo {
25451 name: "bucket-b".to_string(),
25452 ..Default::default()
25453 },
25454 DecomBucketInfo {
25455 name: crate::disk::RUSTFS_META_BUCKET.to_string(),
25456 prefix: crate::disk::BUCKET_META_PREFIX.to_string(),
25457 },
25458 DecomBucketInfo {
25459 name: crate::disk::RUSTFS_META_BUCKET.to_string(),
25460 prefix: crate::bucket::lifecycle::ILM_META_PREFIX.to_string(),
25461 },
25462 ]);
25463
25464 assert_eq!(
25465 regular.iter().map(|bucket| bucket.name.as_str()).collect::<Vec<_>>(),
25466 vec!["bucket-a", "bucket-b",]
25467 );
25468 assert_eq!(
25469 meta.iter().map(|bucket| bucket.prefix.as_str()).collect::<Vec<_>>(),
25470 vec![
25471 crate::config::com::CONFIG_PREFIX,
25472 crate::disk::BUCKET_META_PREFIX,
25473 crate::bucket::lifecycle::ILM_META_PREFIX,
25474 ]
25475 );
25476 }
25477
25478 #[test]
25479 fn test_resume_reconciles_missing_decommission_meta_prefixes() {
25480 let mut meta = PoolMeta {
25481 pools: vec![decommission_test_pool_status(
25482 0,
25483 Some(PoolDecommissionInfo {
25484 queued_buckets: vec![
25485 format!("{}/{}", crate::disk::RUSTFS_META_BUCKET, crate::config::com::CONFIG_PREFIX),
25486 format!("{}/{}", crate::disk::RUSTFS_META_BUCKET, crate::disk::BUCKET_META_PREFIX),
25487 ],
25488 ..Default::default()
25489 }),
25490 )],
25491 ..Default::default()
25492 };
25493
25494 assert!(reconcile_decommission_meta_buckets(&mut meta, 0));
25495 assert_eq!(
25496 meta.pending_buckets(0)
25497 .iter()
25498 .filter(|bucket| bucket.name == crate::disk::RUSTFS_META_BUCKET)
25499 .map(|bucket| bucket.prefix.as_str())
25500 .collect::<Vec<_>>(),
25501 DECOMMISSION_META_PREFIXES
25502 );
25503 assert!(!reconcile_decommission_meta_buckets(&mut meta, 0));
25504 }
25505
25506 #[tokio::test]
25507 async fn test_decommission_metadata_phase_precedes_regular_failure() {
25508 let events = Arc::new(StdMutex::new(Vec::new()));
25509 let err = run_decommission_phases(
25510 CancellationToken::new(),
25511 vec![
25512 DecomBucketInfo {
25513 name: "regular-fails".to_string(),
25514 ..Default::default()
25515 },
25516 DecomBucketInfo {
25517 name: "regular-not-started".to_string(),
25518 ..Default::default()
25519 },
25520 ],
25521 vec![
25522 DecomBucketInfo {
25523 name: crate::disk::RUSTFS_META_BUCKET.to_string(),
25524 prefix: crate::config::com::CONFIG_PREFIX.to_string(),
25525 },
25526 DecomBucketInfo {
25527 name: crate::disk::RUSTFS_META_BUCKET.to_string(),
25528 prefix: crate::disk::BUCKET_META_PREFIX.to_string(),
25529 },
25530 ],
25531 1,
25532 {
25533 let events = Arc::clone(&events);
25534 move |bucket, _rx| {
25535 let events = Arc::clone(&events);
25536 Box::pin(async move {
25537 let event = if bucket.name == crate::disk::RUSTFS_META_BUCKET {
25538 format!("meta:{}", bucket.prefix)
25539 } else {
25540 format!("regular:{}", bucket.name)
25541 };
25542 events.lock().expect("phase event lock should not be poisoned").push(event);
25543 if bucket.name == "regular-fails" {
25544 Err(Error::SlowDown)
25545 } else {
25546 Ok(())
25547 }
25548 })
25549 }
25550 },
25551 )
25552 .await
25553 .expect_err("regular failure should remain fatal after metadata completes");
25554
25555 assert!(matches!(err, Error::SlowDown));
25556 assert_eq!(
25557 *events.lock().expect("phase event lock should not be poisoned"),
25558 vec![
25559 format!("meta:{}", crate::config::com::CONFIG_PREFIX),
25560 format!("meta:{}", crate::disk::BUCKET_META_PREFIX),
25561 "regular:regular-fails".to_string(),
25562 ]
25563 );
25564 }
25565
25566 #[tokio::test]
25567 async fn test_run_decommission_buckets_bounded_respects_limit() {
25568 let rx = CancellationToken::new();
25569 let running = Arc::new(AtomicUsize::new(0));
25570 let max_running = Arc::new(AtomicUsize::new(0));
25571 let started = Arc::new(AtomicUsize::new(0));
25572 let buckets = (0..8)
25573 .map(|idx| DecomBucketInfo {
25574 name: format!("bucket-{idx}"),
25575 ..Default::default()
25576 })
25577 .collect::<Vec<_>>();
25578
25579 run_decommission_buckets_bounded(rx, buckets, 2, {
25580 let running = Arc::clone(&running);
25581 let max_running = Arc::clone(&max_running);
25582 let started = Arc::clone(&started);
25583 move |_bucket, _rx| {
25584 let running = Arc::clone(&running);
25585 let max_running = Arc::clone(&max_running);
25586 let started = Arc::clone(&started);
25587 Box::pin(async move {
25588 started.fetch_add(1, Ordering::SeqCst);
25589 let current = running.fetch_add(1, Ordering::SeqCst) + 1;
25590 max_running.fetch_max(current, Ordering::SeqCst);
25591 tokio::time::sleep(StdDuration::from_millis(10)).await;
25592 running.fetch_sub(1, Ordering::SeqCst);
25593 Ok(())
25594 })
25595 }
25596 })
25597 .await
25598 .expect("bounded bucket scheduler should complete");
25599
25600 assert_eq!(started.load(Ordering::SeqCst), 8);
25601 assert_eq!(max_running.load(Ordering::SeqCst), 2);
25602 assert_eq!(running.load(Ordering::SeqCst), 0);
25603 }
25604
25605 #[tokio::test]
25606 async fn test_run_decommission_buckets_bounded_cancels_and_stops_launching_after_failure() {
25607 let rx = CancellationToken::new();
25608 let started = Arc::new(AtomicUsize::new(0));
25609 let observed_cancel = Arc::new(AtomicBool::new(false));
25610 let buckets = (0..5)
25611 .map(|idx| DecomBucketInfo {
25612 name: format!("bucket-{idx}"),
25613 ..Default::default()
25614 })
25615 .collect::<Vec<_>>();
25616
25617 let err = tokio::time::timeout(
25618 StdDuration::from_secs(2),
25619 run_decommission_buckets_bounded(rx.clone(), buckets, 2, {
25620 let started = Arc::clone(&started);
25621 let observed_cancel = Arc::clone(&observed_cancel);
25622 move |bucket, rx| {
25623 let started = Arc::clone(&started);
25624 let observed_cancel = Arc::clone(&observed_cancel);
25625 Box::pin(async move {
25626 started.fetch_add(1, Ordering::SeqCst);
25627 if bucket.name == "bucket-0" {
25628 while started.load(Ordering::SeqCst) < 2 {
25629 tokio::task::yield_now().await;
25630 }
25631 return Err(Error::SlowDown);
25632 }
25633
25634 rx.cancelled().await;
25635 observed_cancel.store(true, Ordering::SeqCst);
25636 Ok(())
25637 })
25638 }
25639 }),
25640 )
25641 .await
25642 .expect("bucket scheduler should not hang after a bucket failure")
25643 .expect_err("first bucket failure should be returned");
25644
25645 assert!(matches!(err, Error::SlowDown));
25646 assert!(rx.is_cancelled());
25647 assert!(observed_cancel.load(Ordering::SeqCst));
25648 assert_eq!(started.load(Ordering::SeqCst), 2);
25649 }
25650
25651 #[tokio::test]
25652 async fn test_run_decommission_buckets_bounded_external_cancel_stops_pending_buckets() {
25653 let rx = CancellationToken::new();
25654 let started = Arc::new(AtomicUsize::new(0));
25655 let buckets = (0..4)
25656 .map(|idx| DecomBucketInfo {
25657 name: format!("bucket-{idx}"),
25658 ..Default::default()
25659 })
25660 .collect::<Vec<_>>();
25661
25662 let err = run_decommission_buckets_bounded(rx.clone(), buckets, 1, {
25663 let started = Arc::clone(&started);
25664 move |_bucket, rx| {
25665 let started = Arc::clone(&started);
25666 Box::pin(async move {
25667 started.fetch_add(1, Ordering::SeqCst);
25668 rx.cancel();
25669 Ok(())
25670 })
25671 }
25672 })
25673 .await
25674 .expect_err("external cancellation with pending buckets should stop the scheduler");
25675
25676 assert!(matches!(err, Error::OperationCanceled));
25677 assert!(rx.is_cancelled());
25678 assert_eq!(started.load(Ordering::SeqCst), 1);
25679 }
25680
25681 #[tokio::test]
25682 async fn test_wait_decommission_worker_drain_waits_for_entry_permit() {
25683 let workers = Arc::new(Semaphore::new(1));
25684 let permit = workers
25685 .clone()
25686 .acquire_owned()
25687 .await
25688 .expect("test worker permit should acquire");
25689
25690 let drain = tokio::spawn({
25691 let workers = workers.clone();
25692 async move { wait_decommission_worker_drain(&workers, 1).await }
25693 });
25694
25695 tokio::task::yield_now().await;
25696 assert!(!drain.is_finished(), "drain should wait while a worker permit is held");
25697
25698 drop(permit);
25699 let result = tokio::time::timeout(StdDuration::from_secs(1), drain)
25700 .await
25701 .expect("drain should finish after permit release")
25702 .expect("drain task should not panic");
25703 assert!(result.is_ok());
25704 }
25705
25706 #[test]
25707 fn test_decommission_entry_queue_capacity_is_bounded() {
25708 assert_eq!(decommission_entry_queue_capacity(0), 1);
25709 assert_eq!(decommission_entry_queue_capacity(1), 2);
25710 assert_eq!(
25711 decommission_entry_queue_capacity(DECOMMISSION_ENTRY_QUEUE_HARD_CAP),
25712 DECOMMISSION_ENTRY_QUEUE_HARD_CAP
25713 );
25714 assert_eq!(decommission_entry_queue_capacity(usize::MAX), DECOMMISSION_ENTRY_QUEUE_HARD_CAP);
25715 }
25716
25717 #[tokio::test]
25718 async fn test_drain_decommission_entry_queue_waits_for_all_outstanding_entries() {
25719 let outstanding = Arc::new(Semaphore::new(1));
25720 let held = outstanding
25721 .clone()
25722 .acquire_owned()
25723 .await
25724 .expect("test outstanding permit should acquire");
25725 let rx = CancellationToken::new();
25726 let drain = tokio::spawn({
25727 let outstanding = outstanding.clone();
25728 let rx = rx.clone();
25729 async move { drain_decommission_entry_queue(&rx, &outstanding, 1).await }
25730 });
25731
25732 tokio::task::yield_now().await;
25733 assert!(!drain.is_finished(), "queue drain must wait for active entry work");
25734 drop(held);
25735
25736 let drained = tokio::time::timeout(StdDuration::from_secs(1), drain)
25737 .await
25738 .expect("queue drain should finish after entry completion")
25739 .expect("queue drain task should not panic");
25740 assert!(!drained);
25741 }
25742
25743 #[tokio::test]
25744 async fn test_enqueue_decommission_entry_observes_cancellation_when_queue_is_full() {
25745 let outstanding = Arc::new(Semaphore::new(2));
25746 let (tx, mut queue) = tokio::sync::mpsc::channel(1);
25747 let held = outstanding
25748 .clone()
25749 .acquire_owned()
25750 .await
25751 .expect("first queue permit should acquire");
25752 tx.send(QueuedDecommissionEntry {
25753 entry: MetaCacheEntry::default(),
25754 queue_permit: held,
25755 })
25756 .await
25757 .expect("first entry should fill the queue");
25758
25759 let rx = CancellationToken::new();
25760 let enqueue = tokio::spawn({
25761 let rx = rx.clone();
25762 let outstanding = outstanding.clone();
25763 let tx = tx.clone();
25764 async move { enqueue_decommission_entry(&rx, &outstanding, &tx, MetaCacheEntry::default()).await }
25765 });
25766
25767 tokio::task::yield_now().await;
25768 rx.cancel();
25769 let result = tokio::time::timeout(StdDuration::from_secs(1), enqueue)
25770 .await
25771 .expect("full queue enqueue should observe cancellation")
25772 .expect("enqueue task should not panic");
25773 assert!(matches!(result, DecommissionEntryEnqueueResult::Canceled));
25774 drop(queue.recv().await);
25775 }
25776
25777 #[tokio::test]
25778 async fn test_decommission_side_effect_gate_quiesces_before_transition() {
25779 let operation_gate = Arc::new(tokio::sync::RwLock::new(()));
25780 let rx = CancellationToken::new();
25781 let started = Arc::new(tokio::sync::Notify::new());
25782 let release = Arc::new(tokio::sync::Notify::new());
25783 let operation = tokio::spawn({
25784 let operation_gate = operation_gate.clone();
25785 let rx = rx.clone();
25786 let started = started.clone();
25787 let release = release.clone();
25788 async move {
25789 run_decommission_side_effect(&rx, &operation_gate, || async {
25790 started.notify_one();
25791 release.notified().await;
25792 Ok::<_, Error>(())
25793 })
25794 .await
25795 }
25796 });
25797
25798 started.notified().await;
25799 rx.cancel();
25800 let transition = tokio::spawn({
25801 let operation_gate = operation_gate.clone();
25802 async move {
25803 let _guard = operation_gate.write().await;
25804 }
25805 });
25806 tokio::task::yield_now().await;
25807 assert!(!transition.is_finished(), "transition must wait for the in-flight side effect");
25808
25809 release.notify_one();
25810 let operation_result = operation.await.expect("operation task should not panic");
25811 assert!(matches!(operation_result, Err(Error::OperationCanceled)));
25812 transition.await.expect("transition task should not panic");
25813
25814 let called = Arc::new(AtomicBool::new(false));
25815 let result = run_decommission_side_effect(&rx, &operation_gate, {
25816 let called = called.clone();
25817 move || async move {
25818 called.store(true, Ordering::SeqCst);
25819 Ok::<_, Error>(())
25820 }
25821 })
25822 .await;
25823 assert!(matches!(result, Err(Error::OperationCanceled)));
25824 assert!(!called.load(Ordering::SeqCst));
25825 }
25826
25827 #[tokio::test]
25828 async fn test_decommission_side_effect_stops_after_pool_meta_write_block() {
25829 let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
25830 store
25831 .pool_meta_save_gate
25832 .lock()
25833 .await
25834 .observe_replicas(super::PoolMetaReplicaState {
25835 needs_repair: true,
25836 repair_write_safe: false,
25837 });
25838 let called = Arc::new(AtomicBool::new(false));
25839 let operation_gate = store.ctx.data_movement_operation_gate();
25840
25841 let result = store
25842 .run_guarded_decommission_side_effect(&CancellationToken::new(), &operation_gate, {
25843 let called = called.clone();
25844 move || async move {
25845 called.store(true, Ordering::SeqCst);
25846 Ok::<_, Error>(())
25847 }
25848 })
25849 .await;
25850
25851 assert!(
25852 result
25853 .expect_err("sticky pool metadata state must block new movement")
25854 .to_string()
25855 .contains("restart after all replicas are readable and consistent")
25856 );
25857 assert!(!called.load(Ordering::SeqCst));
25858 }
25859
25860 #[tokio::test]
25861 async fn test_decommission_reservation_stops_before_canceler_slot_when_pool_meta_is_blocked() {
25862 let generation = OffsetDateTime::UNIX_EPOCH;
25863 let pool_meta = PoolMeta {
25864 version: super::POOL_META_VERSION,
25865 pools: vec![decommission_test_pool_status(
25866 0,
25867 Some(PoolDecommissionInfo {
25868 start_time: Some(generation),
25869 ..Default::default()
25870 }),
25871 )],
25872 ..Default::default()
25873 };
25874 let store = decommission_worker_test_store(pool_meta, vec![None]);
25875 store
25876 .pool_meta_save_gate
25877 .lock()
25878 .await
25879 .observe_replicas(super::PoolMetaReplicaState {
25880 needs_repair: true,
25881 repair_write_safe: false,
25882 });
25883
25884 let err = store
25885 .reserve_decommission_routines(&CancellationToken::new(), &[0])
25886 .await
25887 .err()
25888 .expect("sticky pool metadata state must block worker reservation");
25889
25890 assert!(
25891 err.to_string()
25892 .contains("restart after all replicas are readable and consistent")
25893 );
25894 assert!(store.decommission_cancelers.read().await[0].is_none());
25895 }
25896
25897 #[tokio::test]
25898 async fn test_v1_unresolved_ledger_rejection_keeps_live_state_and_write_gate_safe() {
25899 let generation = OffsetDateTime::UNIX_EPOCH;
25900 let status = decommission_test_pool_status(
25901 0,
25902 Some(PoolDecommissionInfo {
25903 start_time: Some(generation),
25904 ..Default::default()
25905 }),
25906 );
25907 let last_update = status.last_update;
25908 let store = decommission_worker_test_store(
25909 PoolMeta {
25910 version: POOL_META_V1_VERSION,
25911 pools: vec![status],
25912 ..Default::default()
25913 },
25914 vec![None],
25915 );
25916 let entry = DecommissionUnresolvedEntry {
25917 bucket: "bucket-a".to_string(),
25918 object: "directory/".to_string(),
25919 pool_index: 0,
25920 set_index: 0,
25921 source_generation: generation,
25922 candidate_count: 1,
25923 disk_error_count: 0,
25924 observed_at: generation,
25925 reason: "metadata_resolution_failed".to_string(),
25926 };
25927
25928 let err = store
25929 .persist_decommission_unresolved_entry(0, generation, entry)
25930 .await
25931 .expect_err("V1 must reject the ledger before changing live state");
25932
25933 assert!(matches!(err, Error::InvalidArgument(..)));
25934 let pool_meta = store.pool_meta.read().await;
25935 let status = &pool_meta.pools[0];
25936 assert_eq!(status.last_update, last_update);
25937 assert!(
25938 status
25939 .decommission
25940 .as_ref()
25941 .expect("active decommission metadata should remain present")
25942 .unresolved_entries
25943 .is_empty()
25944 );
25945 drop(pool_meta);
25946 store
25947 .pool_meta_save_gate
25948 .lock()
25949 .await
25950 .ensure_write_safe("V1 unresolved-entry preflight")
25951 .expect("a deterministic capability rejection must not latch recovery");
25952 }
25953
25954 #[tokio::test]
25955 async fn test_v1_runtime_recovery_rejects_worker_but_keeps_cancel_persistable() {
25956 let generation = OffsetDateTime::UNIX_EPOCH;
25957 let store = decommission_worker_test_store(
25958 PoolMeta {
25959 version: POOL_META_V1_VERSION,
25960 pools: vec![decommission_test_pool_status(
25961 0,
25962 Some(PoolDecommissionInfo {
25963 start_time: Some(generation),
25964 ..Default::default()
25965 }),
25966 )],
25967 ..Default::default()
25968 },
25969 vec![None],
25970 );
25971
25972 let err = store
25973 .reserve_decommission_routines(&CancellationToken::new(), &[0])
25974 .await
25975 .err()
25976 .expect("V1 recovery must not install a worker that cannot persist an unresolved ledger");
25977 assert!(matches!(err, Error::InvalidArgument(..)));
25978 assert!(store.decommission_cancelers.read().await[0].is_none());
25979
25980 let save_called = Arc::new(AtomicBool::new(false));
25981 store
25982 .decommission_cancel_with_owner_and_save(0, None, {
25983 let save_called = save_called.clone();
25984 move |snapshot, _| async move {
25985 snapshot.encode_config_data_for_v2_gate(false)?;
25986 save_called.store(true, Ordering::SeqCst);
25987 Ok(())
25988 }
25989 })
25990 .await
25991 .expect("a rejected V1 recovery must remain cancelable without restart");
25992
25993 assert!(save_called.load(Ordering::SeqCst));
25994 let pool_meta = store.pool_meta.read().await;
25995 let info = pool_meta.pools[0]
25996 .decommission
25997 .as_ref()
25998 .expect("cancel metadata should remain present");
25999 assert!(info.canceled);
26000 assert!(!info.failed);
26001 assert!(!info.complete);
26002 }
26003
26004 #[tokio::test]
26005 async fn test_decommission_transition_waits_without_registered_canceler() {
26006 let store = decommission_worker_test_store(PoolMeta::default(), vec![None]);
26007 let operation_gate = store.ctx.data_movement_operation_gate();
26008 let operation_guard = operation_gate.read().await;
26009 let transition = tokio::spawn({
26010 let store = store.clone();
26011 async move { store.cancel_decommission_routines_and_wait(&[0]).await }
26012 });
26013
26014 tokio::task::yield_now().await;
26015 assert!(
26016 !transition.is_finished(),
26017 "a transition must wait for an in-flight side effect even after its canceler slot is gone"
26018 );
26019
26020 drop(operation_guard);
26021 tokio::time::timeout(StdDuration::from_secs(1), transition)
26022 .await
26023 .expect("transition should finish after the side effect")
26024 .expect("transition task should not panic");
26025 }
26026
26027 #[tokio::test]
26028 async fn test_join_error_quiesces_side_effects_before_failure_transition() {
26029 let canceler = DecommissionCanceler::new(CancellationToken::new());
26030 let store = decommission_worker_test_store(PoolMeta::default(), vec![Some(canceler.clone())]);
26031 let operation_gate = store.ctx.data_movement_operation_gate();
26032 let operation_guard = operation_gate.read().await;
26033 let transition = tokio::spawn({
26034 let store = store.clone();
26035 let canceler = canceler.clone();
26036 async move { store.quiesce_decommission_worker_after_join_error(&canceler).await }
26037 });
26038
26039 tokio::time::timeout(StdDuration::from_secs(1), canceler.token().cancelled())
26040 .await
26041 .expect("join error handling should cancel the detached worker");
26042 assert!(
26043 !transition.is_finished(),
26044 "failure transition must wait for the detached worker's in-flight side effect"
26045 );
26046
26047 drop(operation_guard);
26048 tokio::time::timeout(StdDuration::from_secs(1), transition)
26049 .await
26050 .expect("failure transition should finish after the side effect")
26051 .expect("failure transition task should not panic");
26052 }
26053
26054 #[tokio::test(start_paused = true)]
26055 async fn test_run_decommission_listing_with_retry_drains_before_each_retry() {
26056 let attempts = Arc::new(AtomicUsize::new(0));
26057 let drains = Arc::new(AtomicUsize::new(0));
26058 let err = run_decommission_listing_with_retry_and_drain(
26059 CancellationToken::new(),
26060 "bucket-a".to_string(),
26061 noop_decommission_list_callback(),
26062 1,
26063 2,
26064 2,
26065 {
26066 let attempts = attempts.clone();
26067 move |_| {
26068 let attempts = attempts.clone();
26069 async move {
26070 attempts.fetch_add(1, Ordering::SeqCst);
26071 Err(Error::SlowDown)
26072 }
26073 }
26074 },
26075 {
26076 let drains = drains.clone();
26077 move || {
26078 let drains = drains.clone();
26079 async move {
26080 drains.fetch_add(1, Ordering::SeqCst);
26081 false
26082 }
26083 }
26084 },
26085 )
26086 .await
26087 .expect_err("permanent listing failure must be returned");
26088
26089 assert!(err.to_string().contains("attempt 2/2"));
26090 assert_eq!(attempts.load(Ordering::SeqCst), 2);
26091 assert_eq!(drains.load(Ordering::SeqCst), 2);
26092 }
26093
26094 #[test]
26095 fn test_get_by_index_returns_value_when_in_range() {
26096 let values = vec!["a", "b", "c"];
26097 let value = get_by_index(values.as_slice(), 1, "fetch decommission status").expect("in-range index should return value");
26098 assert_eq!(*value, "b");
26099 }
26100
26101 #[test]
26102 fn test_get_by_index_returns_error_when_out_of_range() {
26103 let values = vec![1_u8];
26104 let err =
26105 get_by_index(values.as_slice(), 2, "load decommission background pool").expect_err("out-of-range index should fail");
26106 assert!(
26107 err.to_string()
26108 .contains("failed to load decommission background pool: invalid decommission pool index 2 for 1 pools")
26109 );
26110 }
26111
26112 #[test]
26113 fn test_pool_meta_is_suspended_returns_false_for_out_of_range() {
26114 let meta = PoolMeta::default();
26115 assert!(!meta.is_suspended(1));
26116 }
26117
26118 #[test]
26119 fn test_rollback_start_decommission_pool_meta_clears_active_state() {
26120 let previous = PoolMeta {
26121 pools: vec![PoolStatus {
26122 id: 0,
26123 cmd_line: "pool-0".to_string(),
26124 last_update: OffsetDateTime::UNIX_EPOCH,
26125 decommission: None,
26126 }],
26127 ..Default::default()
26128 };
26129 let mut active = previous.clone();
26130 let active_update = OffsetDateTime::UNIX_EPOCH + Duration::seconds(1);
26131 active.pools[0].last_update = active_update;
26132 active.pools[0].decommission = Some(PoolDecommissionInfo {
26133 start_time: Some(OffsetDateTime::UNIX_EPOCH),
26134 ..Default::default()
26135 });
26136 let mut peer = active.clone();
26137
26138 assert!(active.is_suspended(0));
26139 assert_eq!(
26140 decommission_start_pool_state(active.pools.first()),
26141 DecommissionStartPoolState::Decommissioning
26142 );
26143
26144 rollback_start_decommission_pool_meta(&mut active, &previous, &[0]);
26145
26146 assert!(!active.is_suspended(0));
26147 assert_eq!(decommission_start_pool_state(active.pools.first()), DecommissionStartPoolState::Active);
26148 assert!(active.pools[0].last_update > active_update);
26149 assert!(merge_pool_status_refresh(&mut peer, active, &[false]));
26150 assert!(peer.pools[0].decommission.is_none());
26151 }
26152
26153 #[test]
26154 fn test_pool_meta_queue_buckets_ignores_out_of_range_index() {
26155 let mut meta = PoolMeta {
26156 pools: vec![PoolStatus {
26157 id: 0,
26158 cmd_line: "pool-0".to_string(),
26159 last_update: OffsetDateTime::UNIX_EPOCH,
26160 decommission: Some(PoolDecommissionInfo::default()),
26161 }],
26162 ..Default::default()
26163 };
26164
26165 meta.queue_buckets(
26166 9,
26167 vec![DecomBucketInfo {
26168 name: "bucket-a".to_string(),
26169 prefix: String::new(),
26170 }],
26171 );
26172
26173 let queued = meta.pools[0]
26174 .decommission
26175 .as_ref()
26176 .expect("pool should have decommission info")
26177 .queued_buckets
26178 .clone();
26179 assert!(queued.is_empty());
26180 }
26181
26182 #[test]
26183 fn test_pool_meta_is_bucket_decommissioned_returns_false_for_out_of_range() {
26184 let meta = PoolMeta::default();
26185 assert!(!meta.is_bucket_decommissioned(7, "bucket-a".to_string()));
26186 }
26187
26188 #[test]
26189 fn test_resolve_decommission_bucket_state_rejects_out_of_range_index() {
26190 let meta = PoolMeta {
26191 pools: vec![PoolStatus {
26192 id: 0,
26193 cmd_line: "pool-0".to_string(),
26194 last_update: OffsetDateTime::UNIX_EPOCH,
26195 decommission: Some(PoolDecommissionInfo::default()),
26196 }],
26197 ..Default::default()
26198 };
26199
26200 let bucket = DecomBucketInfo {
26201 name: "bucket-a".to_string(),
26202 prefix: String::new(),
26203 };
26204 let err =
26205 resolve_decommission_bucket_state(&meta, 3, &bucket).expect_err("out-of-range index should return invalid argument");
26206 assert!(err.to_string().contains("invalid decommission pool index 3 for 1 pools"));
26207 }
26208
26209 #[test]
26210 fn test_resolve_decommission_bucket_state_rejects_missing_decommission_meta() {
26211 let meta = PoolMeta {
26212 pools: vec![PoolStatus {
26213 id: 0,
26214 cmd_line: "pool-0".to_string(),
26215 last_update: OffsetDateTime::UNIX_EPOCH,
26216 decommission: None,
26217 }],
26218 ..Default::default()
26219 };
26220
26221 let bucket = DecomBucketInfo {
26222 name: "bucket-a".to_string(),
26223 prefix: String::new(),
26224 };
26225 let err = resolve_decommission_bucket_state(&meta, 0, &bucket)
26226 .expect_err("missing decommission metadata should return explicit error");
26227 assert!(
26228 err.to_string()
26229 .contains("failed to resolve decommission bucket state: decommission metadata not initialized")
26230 );
26231 }
26232
26233 #[test]
26234 fn test_resolve_decommission_bucket_state_returns_true_for_done_bucket() {
26235 let meta = PoolMeta {
26236 pools: vec![PoolStatus {
26237 id: 0,
26238 cmd_line: "pool-0".to_string(),
26239 last_update: OffsetDateTime::UNIX_EPOCH,
26240 decommission: Some(PoolDecommissionInfo {
26241 decommissioned_buckets: vec!["bucket-a".to_string()],
26242 ..Default::default()
26243 }),
26244 }],
26245 ..Default::default()
26246 };
26247
26248 let bucket = DecomBucketInfo {
26249 name: "bucket-a".to_string(),
26250 prefix: String::new(),
26251 };
26252 let done = resolve_decommission_bucket_state(&meta, 0, &bucket).expect("valid state should resolve");
26253 assert!(done);
26254 }
26255
26256 #[test]
26257 fn test_decommission_capacity_publication_preserves_progress_and_checkpoint_deadline() {
26258 let mut current = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_MODEL_VERSION]);
26259 let start = OffsetDateTime::UNIX_EPOCH;
26260 current.pools[0].last_update = start;
26261 current.count_item(0, 17, false);
26262 let mut saved = current.clone();
26263 saved.pools[0]
26264 .decommission
26265 .as_mut()
26266 .expect("persisted source")
26267 .items_decommissioned = 0;
26268 saved.pools[0].decommission.as_mut().expect("persisted source").bytes_done = 0;
26269 for seconds in [10, 20, 29] {
26270 saved.pools[0].last_update = start + Duration::seconds(seconds);
26271 super::publish_decommission_capacity_update(&mut current, &saved, 0).expect("same-owner capacity publication");
26272 }
26273 let info = current.pools[0].decommission.as_ref().expect("live source");
26274 assert_eq!((info.items_decommissioned, info.bytes_done, info.progress_save_item_baseline), (1, 17, 0));
26275 assert_eq!(info.progress_save_last_at, Some(start));
26276 let checkpoint = current
26277 .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, start + Duration::seconds(30), None)
26278 .expect("checkpoint should remain due despite capacity writes")
26279 .expect("capacity updates must not postpone the 30-second progress checkpoint");
26280 current.count_item(0, 19, false);
26281 assert!(current.commit_decommission_progress_checkpoint(0, checkpoint));
26282 let info = current.pools[0].decommission.as_ref().expect("live source");
26283 assert_eq!(
26284 info.items_since_last_progress_save(),
26285 1,
26286 "only the committed snapshot advances the watermark"
26287 );
26288 assert_eq!(info.progress_save_last_at, Some(checkpoint.checkpoint_at));
26289 assert!(
26290 current
26291 .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, start + Duration::seconds(31), None)
26292 .expect("next checkpoint timing")
26293 .is_none()
26294 );
26295 }
26296
26297 #[test]
26298 fn test_decommission_capacity_publication_rejects_changed_identity_without_mutation() {
26299 let baseline = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_MODEL_VERSION]);
26300 for changed_field in [
26301 "layout",
26302 "start",
26303 "queued",
26304 "terminal",
26305 "operation",
26306 "generation",
26307 "owner",
26308 "model",
26309 "cleared",
26310 ] {
26311 let mut current = baseline.clone();
26312 let mut saved = baseline.clone();
26313 saved.pools[0].last_update += Duration::seconds(1);
26314 let pool = &mut current.pools[0];
26315 let info = pool.decommission.as_mut().expect("current source");
26316 match changed_field {
26317 "layout" => pool.cmd_line.push_str("-replaced"),
26318 "start" => info.start_time = Some(OffsetDateTime::UNIX_EPOCH + Duration::seconds(1)),
26319 "queued" => info.queued = true,
26320 "terminal" => info.canceled = true,
26321 "operation" => info.capacity_reservation.as_mut().expect("owner").operation_id = uuid::Uuid::new_v4(),
26322 "generation" => info.capacity_reservation.as_mut().expect("owner").generation += 1,
26323 "owner" => info.capacity_reservation.as_mut().expect("owner").owner_nonce = uuid::Uuid::new_v4(),
26324 "model" => info.capacity_reservation.as_mut().expect("owner").model_version += 1,
26325 "cleared" => pool.decommission = None,
26326 _ => unreachable!("fixed test matrix"),
26327 }
26328 let before = format!("{current:?}");
26329 let err = super::publish_decommission_capacity_update(&mut current, &saved, 0)
26330 .expect_err("changed operation identity must reject publication");
26331 assert!(matches!(err, Error::StalePoolMetadataUpdate { .. }), "{changed_field}: {err}");
26332 assert_eq!(format!("{current:?}"), before, "{changed_field}: rejected publication mutated live state");
26333 }
26334 }
26335
26336 #[test]
26337 fn test_decommission_capacity_revision_does_not_regress_when_clock_moves_backwards() {
26338 let mut meta = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_MODEL_VERSION, Vec::new(), 0, None);
26339 let revision = meta.pools[0].last_update;
26340 let mutation = uuid::Uuid::new_v4();
26341 reserve_decommission_target_pending(&mut meta, 0, 1, 1, mutation, OffsetDateTime::UNIX_EPOCH)
26342 .expect("clock rollback must not prevent a valid owned reservation");
26343 assert_eq!(meta.pools[0].last_update, revision);
26344 resolve_decommission_target_pending(&mut meta, 0, 1, 1, mutation).expect("resolve the exact intent");
26345 record_decommission_target_consumption(
26346 &mut meta,
26347 0,
26348 1,
26349 DecommissionTargetConsumption {
26350 committed_data_bytes: 1,
26351 target_physical_bytes: 1,
26352 observed_physical_bytes: 1,
26353 },
26354 mutation,
26355 OffsetDateTime::UNIX_EPOCH,
26356 )
26357 .expect("commit consumption during clock rollback");
26358 assert_eq!(meta.pools[0].last_update, revision);
26359 }
26360
26361 #[test]
26362 fn test_mark_decommission_bucket_done_is_idempotent_and_ignores_unknown_buckets() {
26363 let bucket = DecomBucketInfo {
26364 name: "bucket-a".to_string(),
26365 prefix: String::new(),
26366 };
26367 let unknown = DecomBucketInfo {
26368 name: "bucket-b".to_string(),
26369 prefix: String::new(),
26370 };
26371 let mut meta = PoolMeta {
26372 pools: vec![decommission_test_pool_status(
26373 0,
26374 Some(PoolDecommissionInfo {
26375 queued_buckets: vec![bucket.to_string()],
26376 ..Default::default()
26377 }),
26378 )],
26379 ..Default::default()
26380 };
26381 assert!(!mark_decommission_bucket_done(&mut meta, 0, &unknown).expect("an unknown bucket is not completed"));
26382 assert!(
26383 meta.pools[0]
26384 .decommission
26385 .as_ref()
26386 .expect("source progress")
26387 .decommissioned_buckets
26388 .is_empty()
26389 );
26390 assert!(mark_decommission_bucket_done(&mut meta, 0, &bucket).expect("complete the queued bucket once"));
26391 assert!(!mark_decommission_bucket_done(&mut meta, 0, &bucket).expect("repeat completion is a no-op"));
26392 let info = meta.pools[0].decommission.as_ref().expect("source progress");
26393 assert!(info.queued_buckets.is_empty());
26394 assert_eq!(info.decommissioned_buckets, vec![bucket.to_string()]);
26395 }
26396
26397 #[test]
26398 fn test_mark_decommission_bucket_done_rejects_missing_decommission_meta() {
26399 let mut meta = PoolMeta {
26400 pools: vec![PoolStatus {
26401 id: 0,
26402 cmd_line: "pool-0".to_string(),
26403 last_update: OffsetDateTime::UNIX_EPOCH,
26404 decommission: None,
26405 }],
26406 ..Default::default()
26407 };
26408
26409 let bucket = DecomBucketInfo {
26410 name: "bucket-a".to_string(),
26411 prefix: String::new(),
26412 };
26413 let err = mark_decommission_bucket_done(&mut meta, 0, &bucket)
26414 .expect_err("missing decommission metadata should return explicit error");
26415 assert!(
26416 err.to_string()
26417 .contains("failed to mark decommission bucket done: decommission metadata not initialized")
26418 );
26419 }
26420
26421 #[test]
26422 fn test_mark_decommission_bucket_done_rejects_out_of_range_index() {
26423 let mut meta = PoolMeta {
26424 pools: vec![PoolStatus {
26425 id: 0,
26426 cmd_line: "pool-0".to_string(),
26427 last_update: OffsetDateTime::UNIX_EPOCH,
26428 decommission: Some(PoolDecommissionInfo::default()),
26429 }],
26430 ..Default::default()
26431 };
26432
26433 let bucket = DecomBucketInfo {
26434 name: "bucket-a".to_string(),
26435 prefix: String::new(),
26436 };
26437 let err =
26438 mark_decommission_bucket_done(&mut meta, 1, &bucket).expect_err("out-of-range index should return invalid argument");
26439 assert!(err.to_string().contains("invalid decommission pool index 1 for 1 pools"));
26440 }
26441
26442 #[test]
26443 fn test_mark_decommission_bucket_done_pops_bucket_when_present() {
26444 let mut meta = PoolMeta {
26445 pools: vec![PoolStatus {
26446 id: 0,
26447 cmd_line: "pool-0".to_string(),
26448 last_update: OffsetDateTime::UNIX_EPOCH,
26449 decommission: Some(PoolDecommissionInfo {
26450 queued_buckets: vec!["bucket-a".to_string()],
26451 ..Default::default()
26452 }),
26453 }],
26454 ..Default::default()
26455 };
26456
26457 let bucket = DecomBucketInfo {
26458 name: "bucket-a".to_string(),
26459 prefix: String::new(),
26460 };
26461 let popped = mark_decommission_bucket_done(&mut meta, 0, &bucket).expect("valid state should mark bucket done");
26462 assert!(popped);
26463 }
26464
26465 #[test]
26466 fn test_count_decommission_item_rejects_missing_decommission_meta() {
26467 let mut meta = PoolMeta {
26468 pools: vec![PoolStatus {
26469 id: 0,
26470 cmd_line: "pool-0".to_string(),
26471 last_update: OffsetDateTime::UNIX_EPOCH,
26472 decommission: None,
26473 }],
26474 ..Default::default()
26475 };
26476
26477 let err = count_decommission_item(&mut meta, 0, 64, true)
26478 .expect_err("missing decommission metadata should return explicit error");
26479 assert!(
26480 err.to_string()
26481 .contains("failed to count decommission item: decommission metadata not initialized")
26482 );
26483 }
26484
26485 #[test]
26486 fn test_count_decommission_item_updates_done_and_failed_counters() {
26487 let mut meta = PoolMeta {
26488 pools: vec![PoolStatus {
26489 id: 0,
26490 cmd_line: "pool-0".to_string(),
26491 last_update: OffsetDateTime::UNIX_EPOCH,
26492 decommission: Some(PoolDecommissionInfo::default()),
26493 }],
26494 ..Default::default()
26495 };
26496
26497 count_decommission_item(&mut meta, 0, 32, false).expect("success counter should be updated");
26498 count_decommission_item(&mut meta, 0, 16, true).expect("failed counter should be updated");
26499
26500 let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
26501 assert_eq!(info.items_decommissioned, 1);
26502 assert_eq!(info.bytes_done, 32);
26503 assert_eq!(info.items_decommission_failed, 1);
26504 assert_eq!(info.bytes_failed, 16);
26505 }
26506
26507 #[test]
26508 fn test_track_decommission_current_object_rejects_missing_decommission_meta() {
26509 let mut meta = PoolMeta {
26510 pools: vec![PoolStatus {
26511 id: 0,
26512 cmd_line: "pool-0".to_string(),
26513 last_update: OffsetDateTime::UNIX_EPOCH,
26514 decommission: None,
26515 }],
26516 ..Default::default()
26517 };
26518
26519 let err = track_decommission_current_object(&mut meta, 0, "bucket-a", "object-a")
26520 .expect_err("missing decommission metadata should return explicit error");
26521 assert!(
26522 err.to_string()
26523 .contains("failed to track decommission current object: decommission metadata not initialized")
26524 );
26525 }
26526
26527 #[test]
26528 fn test_track_decommission_current_object_updates_bucket_and_object() {
26529 let mut meta = PoolMeta {
26530 pools: vec![PoolStatus {
26531 id: 0,
26532 cmd_line: "pool-0".to_string(),
26533 last_update: OffsetDateTime::UNIX_EPOCH,
26534 decommission: Some(PoolDecommissionInfo::default()),
26535 }],
26536 ..Default::default()
26537 };
26538
26539 track_decommission_current_object(&mut meta, 0, "bucket-a", "object-a").expect("valid state should track bucket/object");
26540
26541 let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
26542 assert_eq!(info.bucket, "bucket-a");
26543 assert_eq!(info.object, "object-a");
26544 assert!(info.stage.is_empty());
26545 }
26546
26547 #[test]
26548 fn test_track_decommission_current_object_stage_updates_stage() {
26549 let mut meta = PoolMeta {
26550 pools: vec![PoolStatus {
26551 id: 0,
26552 cmd_line: "pool-0".to_string(),
26553 last_update: OffsetDateTime::UNIX_EPOCH,
26554 decommission: Some(PoolDecommissionInfo::default()),
26555 }],
26556 ..Default::default()
26557 };
26558
26559 track_decommission_current_object_stage(&mut meta, 0, "bucket-a", "object-a", "cleanup_preflight")
26560 .expect("valid state should track bucket/object stage");
26561
26562 let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
26563 assert_eq!(info.bucket, "bucket-a");
26564 assert_eq!(info.object, "object-a");
26565 assert_eq!(info.stage, "cleanup_preflight");
26566 }
26567
26568 #[test]
26569 fn test_resolve_decommission_update_after_result_passthrough_ok() {
26570 let ok = resolve_decommission_update_after_result(Ok(true)).expect("ok value should pass through");
26571 assert!(ok);
26572 }
26573
26574 #[test]
26575 fn test_resolve_decommission_update_after_result_wraps_error_context() {
26576 let err = resolve_decommission_update_after_result(ensure_valid_decommission_pool_index(0, 0).map(|_| false))
26577 .expect_err("invalid argument should be wrapped with context");
26578 assert!(err.to_string().contains("decommission metadata update failed"));
26579 assert!(err.to_string().contains("invalid decommission pool index 0 for 0 pools"));
26580 }
26581
26582 #[test]
26583 fn test_resolve_decommission_progress_save_result_returns_none_on_success() {
26584 assert!(resolve_decommission_progress_save_result(Ok(())).is_none());
26585 }
26586
26587 #[test]
26588 fn test_resolve_decommission_progress_save_result_returns_error_for_best_effort_failure() {
26589 let err = resolve_decommission_progress_save_result(Err(Error::SlowDown))
26590 .expect("progress save failure should be returned for logging");
26591
26592 assert!(err.to_string().contains("decommission progress save failed"));
26593 assert!(err.to_string().contains(Error::SlowDown.to_string().as_str()));
26594 }
26595
26596 #[test]
26597 fn test_resolve_decommission_preflight_heal_result_passthrough_ok() {
26598 assert!(resolve_decommission_preflight_heal_result::<()>("bucket-a", Ok(())).is_ok());
26599 }
26600
26601 #[test]
26602 fn test_resolve_decommission_preflight_heal_result_wraps_error_context() {
26603 let err = resolve_decommission_preflight_heal_result::<()>("bucket-a", Err(Error::SlowDown))
26604 .expect_err("heal failure should carry preflight context");
26605 assert!(
26606 err.to_string()
26607 .contains("decommission preflight heal failed for bucket bucket-a")
26608 );
26609 }
26610
26611 #[test]
26612 fn test_resolve_decommission_optional_bucket_config_result_passthrough() {
26613 let result = resolve_decommission_optional_bucket_config_result("bucket-a", "replication", Ok(42_u8))
26614 .expect("bucket config should pass through");
26615 assert_eq!(result, Some(42));
26616 }
26617
26618 #[test]
26619 fn test_resolve_decommission_optional_bucket_config_result_returns_none_for_missing_config() {
26620 let result =
26621 resolve_decommission_optional_bucket_config_result::<()>("bucket-a", "versioning", Err(Error::ConfigNotFound))
26622 .expect("missing bucket config should map to None");
26623 assert!(result.is_none());
26624 }
26625
26626 #[test]
26627 fn test_resolve_decommission_optional_bucket_config_result_wraps_other_errors() {
26628 let err = resolve_decommission_optional_bucket_config_result::<()>("bucket-a", "replication", Err(Error::SlowDown))
26629 .expect_err("unexpected bucket config errors should be wrapped with context");
26630 assert!(
26631 err.to_string()
26632 .contains("decommission replication config load failed for bucket bucket-a")
26633 );
26634 }
26635
26636 #[test]
26637 fn test_resolve_decommission_entry_cleanup_delete_result_passthrough_ok() {
26638 assert!(resolve_decommission_entry_cleanup_delete_result(Ok(()), "bucket-a", "obj.txt").is_ok());
26639 }
26640
26641 #[test]
26642 fn test_resolve_decommission_entry_cleanup_delete_result_ignores_not_found() {
26643 assert!(resolve_decommission_entry_cleanup_delete_result::<()>(Err(Error::FileNotFound), "bucket-a", "obj.txt").is_ok());
26644 }
26645
26646 #[test]
26647 fn test_resolve_decommission_entry_cleanup_delete_result_wraps_error_context() {
26648 let err = resolve_decommission_entry_cleanup_delete_result::<()>(Err(Error::SlowDown), "bucket-a", "obj.txt")
26649 .expect_err("cleanup delete failure should be wrapped with explicit context");
26650 assert!(
26651 err.to_string()
26652 .contains("decommission cleanup_delete_object failed for bucket-a/obj.txt")
26653 );
26654 }
26655
26656 #[test]
26657 fn test_resolve_decommission_entry_reload_result_passthrough_ok() {
26658 assert!(resolve_decommission_entry_reload_result(Ok(()), "bucket-a", "obj.txt").is_ok());
26659 }
26660
26661 #[test]
26662 fn test_resolve_decommission_entry_reload_result_wraps_error_context() {
26663 let err = resolve_decommission_entry_reload_result(Err(Error::SlowDown), "bucket-a", "obj.txt")
26664 .expect_err("reload failure should be wrapped with explicit context");
26665 assert!(
26666 err.to_string()
26667 .contains("decommission reload_pool_meta failed for bucket-a/obj.txt")
26668 );
26669 }
26670
26671 #[test]
26672 fn test_resolve_decommission_terminal_mark_result_passthrough_ok() {
26673 assert!(resolve_decommission_terminal_mark_result(Ok(()), "completed", "pool-a").is_ok());
26674 }
26675
26676 #[test]
26677 fn test_resolve_decommission_terminal_mark_result_wraps_error_context() {
26678 let err = resolve_decommission_terminal_mark_result(Err(Error::SlowDown), "failed", "pool-a")
26679 .expect_err("terminal mark failure should include stage and pool context");
26680 let message = err.to_string();
26681 assert!(message.contains("decommission terminal mark failed failed for pool pool-a"));
26682 }
26683
26684 #[test]
26685 fn test_resolve_decommission_terminal_mark_after_error_result_passthrough_ok() {
26686 assert!(resolve_decommission_terminal_mark_after_error_result(Ok(()), 3, &Error::SlowDown).is_ok());
26687 }
26688
26689 #[test]
26690 fn test_resolve_decommission_terminal_mark_after_error_result_wraps_error_context() {
26691 let err = resolve_decommission_terminal_mark_after_error_result(Err(Error::OperationCanceled), 3, &Error::SlowDown)
26692 .expect_err("terminal mark after-error failure should include both errors");
26693 let message = err.to_string();
26694 assert!(message.contains("decommission terminal mark failed after background error on pool 3"));
26695 assert!(message.contains("mark error"));
26696 }
26697
26698 #[test]
26699 fn test_observe_decommission_terminal_reload_result_returns_none_on_success() {
26700 assert!(observe_decommission_terminal_reload_result(Ok(()), "complete_decommission for pool 3").is_none());
26701 }
26702
26703 #[test]
26704 fn test_observe_decommission_terminal_reload_result_keeps_failure_for_logging() {
26705 let err = observe_decommission_terminal_reload_result(Err(Error::SlowDown), "decommission_failed for pool 3")
26706 .expect("reload failure should be observable");
26707 let message = err.to_string();
26708 assert!(message.contains("decommission terminal pool meta reload failed during decommission_failed for pool 3"));
26709 assert!(message.contains(Error::SlowDown.to_string().as_str()));
26710 }
26711
26712 #[test]
26713 fn test_decommission_item_size_converts_positive_values() {
26714 assert_eq!(decommission_item_size(42_i64), 42);
26715 }
26716
26717 #[test]
26718 fn test_decommission_item_size_clamps_negative_values_to_zero() {
26719 assert_eq!(decommission_item_size(-1_i64), 0);
26720 }
26721
26722 #[test]
26723 fn test_new_multipart_abort_flag_defaults_to_abort_enabled() {
26724 let flag = data_movement::new_multipart_abort_flag();
26725 assert!(data_movement::should_abort_multipart_upload(&flag));
26726 }
26727
26728 #[test]
26729 fn test_mark_multipart_upload_completed_disables_abort_cleanup() {
26730 let flag = data_movement::new_multipart_abort_flag();
26731 data_movement::mark_multipart_upload_completed(&flag);
26732 assert!(!data_movement::should_abort_multipart_upload(&flag));
26733 }
26734
26735 #[test]
26736 fn test_decode_part_index_returns_some_for_valid_payload() {
26737 let mut index = Index::new();
26738 index.add(0, 0).expect("first index entry should be accepted");
26739 index
26740 .add(2_097_152, 2_097_152)
26741 .expect("second index entry should advance totals");
26742
26743 let encoded = index.into_vec();
26744 let decoded = data_movement::decode_part_index(Some(&encoded)).expect("valid index payload should decode");
26745
26746 assert_eq!(decoded.total_uncompressed, 2_097_152);
26747 assert_eq!(decoded.total_compressed, 2_097_152);
26748 }
26749
26750 #[test]
26751 fn test_with_decommission_entry_context_formats_stage_bucket_and_object() {
26752 let err = with_decommission_entry_context("update_after", "bucket-a", "obj.txt", Error::SlowDown);
26753 let message = err.to_string();
26754 assert!(message.contains("decommission entry update_after failed"));
26755 assert!(message.contains("bucket bucket-a"));
26756 assert!(message.contains("object obj.txt"));
26757 }
26758
26759 #[test]
26760 fn test_load_decommission_entry_versions_wraps_parse_errors_with_context() {
26761 let entry = MetaCacheEntry {
26762 name: "obj.txt".to_string(),
26763 metadata: vec![1, 2, 3],
26764 cached: None,
26765 reusable: false,
26766 };
26767
26768 let err = load_decommission_entry_versions(&entry, "bucket-a", "check_after_decommission.file_info_versions")
26769 .expect_err("invalid metadata should fail");
26770 let message = err.to_string();
26771 assert!(message.contains("decommission entry check_after_decommission.file_info_versions failed"));
26772 assert!(message.contains("bucket bucket-a"));
26773 assert!(message.contains("object obj.txt"));
26774 }
26775
26776 #[test]
26777 fn test_resolve_decommission_entry_exact_versions_preserves_full_parts() {
26778 let entry = MetaCacheEntry {
26779 name: "obj.txt".to_string(),
26780 metadata: Vec::new(),
26781 cached: None,
26782 reusable: false,
26783 };
26784 let fivs = FileInfoVersions {
26785 volume: "bucket-a".to_string(),
26786 name: "obj.txt".to_string(),
26787 versions: vec![FileInfo {
26788 name: "obj.txt".to_string(),
26789 parts: vec![ObjectPartInfo {
26790 number: 1,
26791 etag: "part-etag".to_string(),
26792 size: 128,
26793 actual_size: 128,
26794 ..Default::default()
26795 }],
26796 ..Default::default()
26797 }],
26798 ..Default::default()
26799 };
26800
26801 let resolved = resolve_decommission_entry_exact_versions(Ok(Some(fivs)), &entry, "bucket-a", "file_info_versions")
26802 .expect("exact versions should be preserved");
26803
26804 assert_eq!(resolved.versions[0].parts.len(), 1);
26805 assert_eq!(resolved.versions[0].parts[0].etag, "part-etag");
26806 }
26807
26808 #[test]
26809 fn test_resolve_decommission_entry_exact_versions_uses_empty_when_source_missing() {
26810 let entry = MetaCacheEntry {
26811 name: "obj.txt".to_string(),
26812 metadata: Vec::new(),
26813 cached: None,
26814 reusable: false,
26815 };
26816
26817 let resolved = resolve_decommission_entry_exact_versions(Ok(None), &entry, "bucket-a", "file_info_versions")
26818 .expect("missing source metadata should be treated as empty");
26819
26820 assert_eq!(resolved.volume, "bucket-a");
26821 assert_eq!(resolved.name, "obj.txt");
26822 assert!(resolved.versions.is_empty());
26823 }
26824
26825 #[test]
26826 fn test_resolve_decommission_check_after_list_result_prefers_entry_error() {
26827 let err = resolve_decommission_check_after_list_result(Err(Error::OperationCanceled), Some(Error::SlowDown))
26828 .expect_err("entry error should win over cancellation");
26829 assert!(matches!(err, Error::SlowDown));
26830 }
26831
26832 #[test]
26833 fn test_resolve_decommission_partial_listing_entry_rejects_unresolved_metadata() {
26834 let generation = OffsetDateTime::now_utc();
26835 let unresolved_entry = resolve_decommission_partial_listing_entry(
26836 MetaCacheEntries(vec![None]),
26837 MetadataResolutionParams {
26838 dir_quorum: 2,
26839 obj_quorum: 2,
26840 bucket: "bucket-a".to_string(),
26841 ..Default::default()
26842 },
26843 "bucket-a",
26844 "prefix/",
26845 1,
26846 2,
26847 3,
26848 generation,
26849 )
26850 .expect_err("unresolved partial listing must fail closed");
26851
26852 assert_eq!(unresolved_entry.bucket, "bucket-a");
26853 assert_eq!(unresolved_entry.object, "prefix/");
26854 assert_eq!(unresolved_entry.pool_index, 2);
26855 assert_eq!(unresolved_entry.set_index, 3);
26856 assert_eq!(unresolved_entry.source_generation, generation);
26857 assert_eq!(unresolved_entry.candidate_count, 0);
26858 assert_eq!(unresolved_entry.disk_error_count, 1);
26859 assert_eq!(unresolved_entry.reason, "metadata_resolution_failed");
26860
26861 let message = decommission_unresolved_listing_error(&unresolved_entry).to_string();
26862 assert!(message.contains("decommission listing could not resolve metadata"));
26863 assert!(message.contains("bucket-a/prefix/"));
26864 assert!(message.contains("pool 2 set 3"));
26865 assert!(message.contains("1 disk error(s)"));
26866 }
26867
26868 #[test]
26869 fn unresolved_entry_ledger_requires_individual_verification_before_completion() {
26870 let generation = OffsetDateTime::now_utc();
26871 let mut pool_meta = PoolMeta {
26872 version: POOL_META_VERSION,
26873 pools: vec![PoolStatus {
26874 id: 0,
26875 cmd_line: "/data/pool".to_string(),
26876 last_update: generation,
26877 decommission: Some(PoolDecommissionInfo {
26878 start_time: Some(generation),
26879 ..Default::default()
26880 }),
26881 }],
26882 dont_save: true,
26883 };
26884 let unresolved_entry = DecommissionUnresolvedEntry {
26885 bucket: "bucket-a".to_string(),
26886 object: "object-a".to_string(),
26887 pool_index: 0,
26888 set_index: 0,
26889 source_generation: generation,
26890 candidate_count: 1,
26891 disk_error_count: 0,
26892 observed_at: generation,
26893 reason: "metadata_resolution_failed".to_string(),
26894 };
26895
26896 assert!(
26897 record_decommission_unresolved_entry(
26898 &mut pool_meta,
26899 0,
26900 generation,
26901 unresolved_entry.clone(),
26902 generation + Duration::nanoseconds(1),
26903 None,
26904 )
26905 .expect("active generation should accept unresolved entry")
26906 );
26907 let err = reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, None, None)
26908 .expect_err("unverified completion must retain unresolved entries");
26909 assert!(err.to_string().contains("1 unresolved listing entries remain"));
26910 assert_eq!(
26911 pool_meta.pools[0]
26912 .decommission
26913 .as_ref()
26914 .expect("decommission should exist")
26915 .unresolved_entries
26916 .len(),
26917 1
26918 );
26919
26920 let err = reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), None)
26921 .expect_err("a same-generation sweep without entry proof must retain the ledger");
26922 assert!(err.to_string().contains("not individually verified"));
26923
26924 let stale_verified = vec![unresolved_entry.clone()];
26925 let mut replacement = unresolved_entry;
26926 replacement.disk_error_count = 2;
26927 replacement.observed_at = generation + Duration::seconds(1);
26928 assert!(
26929 record_decommission_unresolved_entry(
26930 &mut pool_meta,
26931 0,
26932 generation,
26933 replacement.clone(),
26934 generation + Duration::nanoseconds(2),
26935 None,
26936 )
26937 .expect("a newer observation should replace the ledger entry")
26938 );
26939 let err =
26940 reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), Some(&stale_verified))
26941 .expect_err("stale entry verification must not clear a concurrent replacement");
26942 assert!(err.to_string().contains("not individually verified"));
26943
26944 let verified = vec![replacement];
26945 reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), Some(&verified))
26946 .expect("individually verified entries should reconcile");
26947 assert!(
26948 pool_meta.pools[0]
26949 .decommission
26950 .as_ref()
26951 .expect("decommission should exist")
26952 .unresolved_entries
26953 .is_empty()
26954 );
26955 }
26956
26957 #[tokio::test]
26958 async fn final_sweep_persists_unresolved_entry_from_real_listing() {
26959 let (dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
26960 let bucket = "decommission-final-sweep-unresolved";
26961 let object = "corrupt-object";
26962 metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
26963 store
26964 .make_bucket(bucket, &MakeBucketOptions::default())
26965 .await
26966 .expect("test bucket should be created");
26967
26968 let generation = OffsetDateTime::now_utc();
26969 {
26970 let mut pool_meta = store.pool_meta.write().await;
26971 pool_meta.dont_save = false;
26972 pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
26973 start_time: Some(generation),
26974 ..Default::default()
26975 });
26976 }
26977 store
26978 .save_current_pool_meta_for_test(&[0])
26979 .await
26980 .expect("active decommission metadata should be persisted before the final sweep");
26981
26982 for (disk_index, dir) in dirs.iter().enumerate() {
26983 let object_dir = dir.path().join(bucket).join(object);
26984 tokio::fs::create_dir_all(&object_dir)
26985 .await
26986 .expect("corrupt object directory should be created");
26987 tokio::fs::write(object_dir.join(STORAGE_FORMAT_FILE), format!("corrupt-xl-meta-{disk_index}").into_bytes())
26988 .await
26989 .expect("divergent corrupt metadata should be written");
26990 }
26991
26992 let err = store
26993 .check_after_decommission(0, &CancellationToken::new(), generation)
26994 .await
26995 .expect_err("real final sweep must fail closed on unresolved listing metadata");
26996 assert!(
26997 err.to_string().contains("decommission listing could not resolve metadata"),
26998 "unexpected final sweep error: {err:?}"
26999 );
27000
27001 let in_memory_entries = store.pool_meta.read().await.pools[0]
27002 .decommission
27003 .as_ref()
27004 .expect("decommission should remain active")
27005 .unresolved_entries
27006 .clone();
27007 assert_eq!(in_memory_entries.len(), 1);
27008 let unresolved_entry = &in_memory_entries[0];
27009 assert_eq!(unresolved_entry.bucket, bucket);
27010 assert_eq!(unresolved_entry.object, object);
27011 assert_eq!(unresolved_entry.pool_index, 0);
27012 assert_eq!(unresolved_entry.set_index, 0);
27013 assert_eq!(unresolved_entry.source_generation, generation);
27014 assert_eq!(unresolved_entry.candidate_count, 4);
27015 assert_eq!(unresolved_entry.disk_error_count, 0);
27016 assert_eq!(unresolved_entry.reason, "metadata_resolution_failed");
27017
27018 let mut restored = PoolMeta::default();
27019 restored
27020 .load_no_lock_from_replicas(vec![store.pools[0].clone()])
27021 .await
27022 .expect("persisted pool metadata should reload after the final-sweep failure");
27023 assert_eq!(
27024 restored.pools[0]
27025 .decommission
27026 .as_ref()
27027 .expect("reloaded decommission should exist")
27028 .unresolved_entries,
27029 in_memory_entries
27030 );
27031 }
27032
27033 #[tokio::test]
27034 async fn decommission_metadata_saves_stay_monotonic_across_clock_rollback_before_terminal_restart() {
27035 let (_dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
27036 let persisted_floor = store.pool_meta.read().await.pools[0].last_update;
27037 let generation = persisted_floor
27038 .checked_add(Duration::seconds(1))
27039 .expect("test generation should advance the initialized pool metadata");
27040 let earlier_tick = generation - Duration::nanoseconds(10);
27041 {
27042 let mut pool_meta = store.pool_meta.write().await;
27043 pool_meta.dont_save = false;
27044 pool_meta.pools[0].last_update = generation;
27045 pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
27046 start_time: Some(generation),
27047 ..Default::default()
27048 });
27049 }
27050 store
27051 .save_current_pool_meta_for_test(&[0])
27052 .await
27053 .expect("active decommission metadata should persist before rollback checkpoints");
27054
27055 let unresolved_entry = DecommissionUnresolvedEntry {
27056 bucket: "bucket-a".to_string(),
27057 object: "object-a".to_string(),
27058 pool_index: 0,
27059 set_index: 0,
27060 source_generation: generation,
27061 candidate_count: 1,
27062 disk_error_count: 1,
27063 observed_at: earlier_tick,
27064 reason: "metadata_resolution_failed".to_string(),
27065 };
27066 {
27067 let mut pool_meta = store.pool_meta.write().await;
27068 assert!(
27069 record_decommission_unresolved_entry(&mut pool_meta, 0, generation, unresolved_entry, earlier_tick, None)
27070 .expect("unresolved entry should record under active generation")
27071 );
27072 assert_eq!(pool_meta.pools[0].last_update, generation + Duration::nanoseconds(1));
27073 }
27074 store
27075 .save_current_pool_meta_for_test(&[0])
27076 .await
27077 .expect("rollback unresolved-entry save should not stale-reject");
27078 store.pool_meta.write().await.pools[0]
27079 .decommission
27080 .as_mut()
27081 .expect("decommission metadata should exist")
27082 .items_decommissioned = DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD;
27083
27084 assert!(
27085 store
27086 .save_decommission_progress_checkpoint_at(0, generation, earlier_tick)
27087 .await
27088 .expect("rollback progress checkpoint should use a monotonic durable identity")
27089 );
27090 let progress_at = generation + Duration::nanoseconds(2);
27091 assert_eq!(store.pool_meta.read().await.pools[0].last_update, progress_at);
27092
27093 let terminal_at = generation + Duration::nanoseconds(3);
27094 {
27095 let mut pool_meta = store.pool_meta.write().await;
27096 assert!(pool_meta.decommission_failed_at_for_test(0, earlier_tick, None));
27097 assert_eq!(pool_meta.pools[0].last_update, terminal_at);
27098 }
27099 store
27100 .save_current_pool_meta_for_test(&[0])
27101 .await
27102 .expect("terminal failure after rollback checkpoints should persist");
27103
27104 let mut restored = PoolMeta::default();
27105 restored
27106 .load_no_lock_from_replicas(vec![store.pools[0].clone()])
27107 .await
27108 .expect("terminal metadata should reload after restart");
27109 assert_eq!(restored.pools[0].last_update, terminal_at);
27110 assert!(
27111 restored.pools[0]
27112 .decommission
27113 .as_ref()
27114 .is_some_and(|info| info.failed && !info.complete && !info.canceled)
27115 );
27116
27117 let restarted = decommission_worker_test_store(restored, Vec::new());
27118 let status = restarted.scanner_data_movement_pause_status().await;
27119 let expected_generation =
27120 u64::try_from(terminal_at.unix_timestamp_nanos()).expect("fixed positive timestamp should fit generation");
27121
27122 assert_eq!(status.movement_generation, expected_generation);
27123 assert_eq!(status.reasons, vec![crate::store::ScannerDataMovementPauseReason::DecommissionFailed]);
27124 assert_eq!(restarted.scanner_data_movement_generation(), expected_generation);
27125 }
27126
27127 #[tokio::test]
27128 async fn unresolved_entry_probe_verifies_absence_on_every_source_disk() {
27129 let (_dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
27130 let bucket = "decommission-final-sweep-absent-ledger";
27131 store
27132 .peer_sys
27133 .make_bucket(bucket, &MakeBucketOptions::default())
27134 .await
27135 .expect("test bucket should be created");
27136 metadata_sys::init_bucket_metadata_sys(store.clone(), vec![bucket.to_string()]).await;
27137
27138 let generation = OffsetDateTime::now_utc();
27139 let unresolved_entry = DecommissionUnresolvedEntry {
27140 bucket: bucket.to_string(),
27141 object: "absent-object".to_string(),
27142 pool_index: 0,
27143 set_index: 0,
27144 source_generation: generation,
27145 candidate_count: 1,
27146 disk_error_count: 1,
27147 observed_at: generation,
27148 reason: "metadata_resolution_failed".to_string(),
27149 };
27150 {
27151 let mut pool_meta = store.pool_meta.write().await;
27152 pool_meta.version = POOL_META_VERSION;
27153 pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
27154 start_time: Some(generation),
27155 unresolved_entries: vec![unresolved_entry.clone()],
27156 ..Default::default()
27157 });
27158 }
27159
27160 assert!(
27161 store.pools[0].disk_set[0]
27162 .decommission_unresolved_entry_absent_on_all_disks(0, &unresolved_entry)
27163 .await
27164 .expect("all source disks should be readable")
27165 );
27166 let verified = vec![unresolved_entry.clone()];
27167
27168 let mut pool_meta = store.pool_meta.write().await;
27169 reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), Some(&verified))
27170 .expect("the individually verified entry should reconcile");
27171 assert!(
27172 pool_meta.pools[0]
27173 .decommission
27174 .as_ref()
27175 .expect("decommission should remain present")
27176 .unresolved_entries
27177 .is_empty()
27178 );
27179 }
27180
27181 #[tokio::test]
27182 async fn test_record_decommission_entry_error_cancels_listing_and_preserves_first_error() {
27183 let entry_error = Arc::new(tokio::sync::Mutex::new(None));
27184 let rx = CancellationToken::new();
27185
27186 assert!(record_decommission_entry_error(&entry_error, &rx, Error::SlowDown).await);
27187 assert!(!record_decommission_entry_error(&entry_error, &rx, Error::OperationCanceled).await);
27188
27189 assert!(rx.is_cancelled());
27190 assert!(matches!(*entry_error.lock().await, Some(Error::SlowDown)));
27191 }
27192
27193 #[tokio::test]
27194 async fn test_record_decommission_entry_error_ignores_already_canceled_listing() {
27195 let entry_error = Arc::new(tokio::sync::Mutex::new(None));
27196 let rx = CancellationToken::new();
27197 rx.cancel();
27198
27199 assert!(!record_decommission_entry_error(&entry_error, &rx, Error::SlowDown).await);
27200
27201 assert!(entry_error.lock().await.is_none());
27202 }
27203
27204 #[test]
27205 fn unresolved_entry_verification_requires_every_source_disk() {
27206 assert!(ensure_decommission_unresolved_verification_disk_count(4, 4, 0, 1).is_ok());
27207
27208 let err = ensure_decommission_unresolved_verification_disk_count(4, 3, 0, 1)
27209 .expect_err("an offline source disk must block ledger reconciliation");
27210 assert!(err.to_string().contains("requires all 4 source disks, but only 3 are online"));
27211 }
27212
27213 #[test]
27214 fn test_resolve_decommission_listing_error_preserves_real_listing_failure() {
27215 let err = resolve_decommission_listing_error(Some(Error::SlowDown), Some(Error::OperationCanceled))
27216 .expect("listing failure should be returned");
27217 assert!(matches!(err, Error::SlowDown));
27218
27219 let err = resolve_decommission_listing_error(Some(Error::OperationCanceled), Some(Error::SlowDown))
27220 .expect("entry failure should be returned");
27221 assert!(matches!(err, Error::SlowDown));
27222 }
27223
27224 #[test]
27225 fn test_resolve_decommission_check_after_list_result_returns_list_result_without_entry_error() {
27226 let err = resolve_decommission_check_after_list_result(Err(Error::OperationCanceled), None)
27227 .expect_err("list result should be preserved without entry error");
27228 assert!(matches!(err, Error::OperationCanceled));
27229 }
27230
27231 #[test]
27232 fn test_resolve_decommission_pool_meta_reload_result_passthrough_ok() {
27233 assert!(resolve_decommission_pool_meta_reload_result(Ok(()), "start_decommission").is_ok());
27234 }
27235
27236 #[test]
27237 fn test_resolve_decommission_pool_meta_reload_result_wraps_error_context() {
27238 let err = resolve_decommission_pool_meta_reload_result(Err(Error::SlowDown), "decommission_failed for pool 3")
27239 .expect_err("reload failure should be wrapped with stage context");
27240 let message = err.to_string();
27241 assert!(message.contains("decommission pool meta reload failed during decommission_failed for pool 3"));
27242 assert!(message.contains(Error::SlowDown.to_string().as_str()));
27243 }
27244
27245 #[test]
27246 fn test_resolve_start_decommission_pool_meta_reload_result_returns_failure() {
27247 let err = resolve_start_decommission_pool_meta_reload_result(Err(Error::other(
27248 "reload_pool_meta encountered 1 failure(s): peer[0] reload_pool_meta failed",
27249 )))
27250 .expect_err("start_decommission must fail when peer pool meta reload fails");
27251 let message = err.to_string();
27252
27253 assert!(message.contains("decommission pool meta reload failed during start_decommission"));
27254 assert!(message.contains("reload_pool_meta encountered 1 failure(s)"));
27255 assert!(message.contains("peer[0]"));
27256 }
27257
27258 #[test]
27259 fn test_resolve_decommission_listing_worker_result_passthrough_ok() {
27260 assert!(resolve_decommission_listing_worker_result(2, Ok(Ok(()))).is_ok());
27261 }
27262
27263 #[test]
27264 fn test_resolve_decommission_listing_worker_result_passthrough_worker_error() {
27265 let err = resolve_decommission_listing_worker_result(2, Ok(Err(Error::SlowDown)))
27266 .expect_err("listing worker error should be returned");
27267
27268 assert!(matches!(err, Error::SlowDown));
27269 }
27270
27271 #[tokio::test]
27272 async fn test_resolve_decommission_listing_worker_result_wraps_join_error_context() {
27273 let join_error = tokio::spawn(async {
27274 panic!("listing worker panic");
27275 })
27276 .await
27277 .expect_err("panic task should return JoinError");
27278
27279 let err = resolve_decommission_listing_worker_result(4, Err(join_error))
27280 .expect_err("join error should be wrapped with context");
27281 let message = err.to_string();
27282 assert!(message.contains("decommission listing worker 4 task join error"));
27283 assert!(message.contains("panic"));
27284 }
27285
27286 #[test]
27287 fn test_should_retry_decommission_listing_respects_attempt_limit_and_bucket_missing() {
27288 assert!(should_retry_decommission_listing(&Error::SlowDown, 0, 2));
27289 assert!(!should_retry_decommission_listing(&Error::SlowDown, 1, 2));
27290 assert!(!should_retry_decommission_listing(
27291 &StorageError::BucketNotFound("bucket".to_string()),
27292 0,
27293 2
27294 ));
27295 }
27296
27297 #[tokio::test]
27298 async fn test_wait_decommission_retry_backoff_reports_canceled_without_sleeping() {
27299 let token = CancellationToken::new();
27300 token.cancel();
27301
27302 assert!(wait_decommission_retry_backoff(&token, StdDuration::from_secs(30)).await);
27303 }
27304
27305 #[test]
27306 fn test_decommission_retry_backoff_delay_grows_linearly() {
27307 let base = StdDuration::from_millis(100);
27308
27309 assert_eq!(decommission_retry_backoff_delay(base, 1), base);
27310 assert_eq!(decommission_retry_backoff_delay(base, 3), StdDuration::from_millis(300));
27311 assert_eq!(decommission_retry_backoff_delay(base, usize::MAX), base.saturating_mul(u32::MAX));
27312 }
27313
27314 #[test]
27315 fn test_source_changed_exhaustion_fails_only_after_pool_limit() {
27316 assert!(!should_fail_decommission_pool_after_exhausted_source_changed(
27317 DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT
27318 ));
27319 assert!(should_fail_decommission_pool_after_exhausted_source_changed(
27320 DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT + 1
27321 ));
27322 }
27323
27324 #[tokio::test(start_paused = true)]
27325 async fn test_run_decommission_listing_with_retry_stops_after_attempt_limit() {
27326 let attempts = Arc::new(AtomicUsize::new(0));
27327 let err = run_decommission_listing_with_retry(
27328 CancellationToken::new(),
27329 "bucket-a".to_string(),
27330 noop_decommission_list_callback(),
27331 1,
27332 2,
27333 3,
27334 {
27335 let attempts = attempts.clone();
27336 move |_| {
27337 let attempts = attempts.clone();
27338 async move {
27339 attempts.fetch_add(1, Ordering::SeqCst);
27340 Err(Error::SlowDown)
27341 }
27342 }
27343 },
27344 )
27345 .await
27346 .expect_err("permanent listing failure must not retry forever");
27347
27348 assert_eq!(attempts.load(Ordering::SeqCst), 3);
27349 assert!(err.to_string().contains("attempt 3/3"));
27350 }
27351
27352 #[tokio::test]
27353 async fn test_run_decommission_listing_with_retry_treats_bucket_missing_as_complete() {
27354 let attempts = Arc::new(AtomicUsize::new(0));
27355 run_decommission_listing_with_retry(
27356 CancellationToken::new(),
27357 "bucket-a".to_string(),
27358 noop_decommission_list_callback(),
27359 1,
27360 2,
27361 3,
27362 {
27363 let attempts = attempts.clone();
27364 move |_| {
27365 let attempts = attempts.clone();
27366 async move {
27367 attempts.fetch_add(1, Ordering::SeqCst);
27368 Err(StorageError::BucketNotFound("bucket-a".to_string()))
27369 }
27370 }
27371 },
27372 )
27373 .await
27374 .expect("missing bucket should keep previous decommission listing behavior");
27375
27376 assert_eq!(attempts.load(Ordering::SeqCst), 1);
27377 }
27378
27379 #[test]
27380 fn test_should_count_decommission_version_complete_for_cleanup_safe_ignored_result() {
27381 assert!(should_count_decommission_version_complete(true, true, false));
27382 }
27383
27384 #[test]
27385 fn test_should_count_decommission_version_complete_rejects_skip_only_ignored_result() {
27386 assert!(!should_count_decommission_version_complete(true, false, false));
27387 }
27388
27389 #[test]
27390 fn test_should_count_decommission_version_complete_for_completed_result() {
27391 assert!(should_count_decommission_version_complete(false, false, false));
27392 }
27393
27394 #[test]
27395 fn test_should_count_decommission_version_complete_rejects_failed_result() {
27396 assert!(!should_count_decommission_version_complete(false, false, true));
27397 }
27398
27399 #[test]
27400 fn test_should_cleanup_decommission_source_entry_accepts_all_versions_completed() {
27401 assert!(should_cleanup_decommission_source_entry(3, 3, 0));
27402 }
27403
27404 #[test]
27405 fn test_should_cleanup_decommission_source_entry_accepts_migrated_and_safely_expired_versions() {
27406 assert!(should_cleanup_decommission_source_entry(1, 2, 1));
27407 }
27408
27409 #[test]
27410 fn test_should_cleanup_decommission_source_entry_accepts_versions_only_safely_expired_by_lifecycle() {
27411 assert!(should_cleanup_decommission_source_entry(0, 2, 2));
27412 }
27413
27414 #[test]
27415 fn test_should_cleanup_decommission_source_entry_rejects_object_lock_retained_version() {
27416 assert!(!should_cleanup_decommission_source_entry(1, 2, 0));
27417 }
27418
27419 #[test]
27420 fn test_should_cleanup_decommission_source_entry_rejects_replication_pending_version() {
27421 assert!(!should_cleanup_decommission_source_entry(2, 3, 0));
27422 }
27423
27424 #[test]
27425 fn test_should_cleanup_decommission_source_entry_rejects_counter_overrun() {
27426 assert!(!should_cleanup_decommission_source_entry(2, 2, 1));
27427 }
27428
27429 #[test]
27430 fn test_pool_meta_update_after_rejects_out_of_range_index() {
27431 let mut meta = PoolMeta::default();
27432 let err = meta
27433 .update_after(1, Duration::seconds(1))
27434 .expect_err("out-of-range index should fail");
27435 assert!(err.to_string().contains("invalid decommission pool index 1 for 0 pools"));
27436 }
27437
27438 #[test]
27439 fn test_pool_meta_update_after_rejects_when_decommission_missing() {
27440 let mut meta = PoolMeta {
27441 pools: vec![PoolStatus {
27442 id: 0,
27443 cmd_line: "pool-0".to_string(),
27444 last_update: OffsetDateTime::UNIX_EPOCH,
27445 decommission: None,
27446 }],
27447 ..Default::default()
27448 };
27449
27450 let err = meta
27451 .update_after(0, Duration::seconds(1))
27452 .expect_err("pool without decommission should fail");
27453 assert!(
27454 err.to_string()
27455 .contains("failed to update decommission metadata timestamp: decommission metadata not initialized")
27456 );
27457 }
27458
27459 #[test]
27460 fn test_track_decommission_stage_does_not_advance_checkpoint_state() {
27461 let mut meta = PoolMeta {
27462 pools: vec![PoolStatus {
27463 id: 0,
27464 cmd_line: "pool-0".to_string(),
27465 last_update: OffsetDateTime::UNIX_EPOCH,
27466 decommission: Some(PoolDecommissionInfo {
27467 items_decommissioned: 3,
27468 items_decommission_failed: 2,
27469 ..Default::default()
27470 }),
27471 }],
27472 ..Default::default()
27473 };
27474
27475 track_decommission_current_object_stage(&mut meta, 0, "bucket", "object", "migrate_object")
27476 .expect("valid decommission progress should be tracked");
27477
27478 assert_eq!(meta.pools[0].last_update, OffsetDateTime::UNIX_EPOCH);
27479 let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
27480 assert_eq!(info.items_since_last_progress_save(), 5);
27481 assert_eq!(info.stage, "migrate_object");
27482 }
27483
27484 #[test]
27485 fn test_pool_meta_update_after_skips_before_time_and_item_thresholds() {
27486 let mut meta = PoolMeta {
27487 pools: vec![PoolStatus {
27488 id: 0,
27489 cmd_line: "pool-0".to_string(),
27490 last_update: OffsetDateTime::now_utc(),
27491 decommission: Some(PoolDecommissionInfo {
27492 items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD - 1,
27493 ..Default::default()
27494 }),
27495 }],
27496 ..Default::default()
27497 };
27498
27499 let saved = meta
27500 .update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
27501 .expect("valid decommission state should update");
27502
27503 assert!(!saved);
27504 let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
27505 assert_eq!(info.items_since_last_progress_save(), DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD - 1);
27506 }
27507
27508 #[test]
27509 fn test_pool_meta_update_after_requests_save_when_item_threshold_reached() {
27510 let mut meta = PoolMeta {
27511 pools: vec![PoolStatus {
27512 id: 0,
27513 cmd_line: "pool-0".to_string(),
27514 last_update: OffsetDateTime::now_utc(),
27515 decommission: Some(PoolDecommissionInfo {
27516 items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
27517 ..Default::default()
27518 }),
27519 }],
27520 ..Default::default()
27521 };
27522
27523 let saved = meta
27524 .update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
27525 .expect("item threshold should save progress");
27526
27527 assert!(saved);
27528 let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
27529 assert_eq!(info.items_since_last_progress_save(), DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD);
27530 }
27531
27532 #[test]
27533 fn test_pool_meta_update_after_requests_save_when_time_threshold_reached() {
27534 let mut meta = PoolMeta {
27535 pools: vec![PoolStatus {
27536 id: 0,
27537 cmd_line: "pool-0".to_string(),
27538 last_update: OffsetDateTime::now_utc() - DECOMMISSION_PROGRESS_SAVE_INTERVAL,
27539 decommission: Some(PoolDecommissionInfo {
27540 items_decommissioned: 1,
27541 ..Default::default()
27542 }),
27543 }],
27544 ..Default::default()
27545 };
27546
27547 let saved = meta
27548 .update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
27549 .expect("time threshold should save progress");
27550
27551 assert!(saved);
27552 let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
27553 assert_eq!(info.items_since_last_progress_save(), 1);
27554 }
27555
27556 #[test]
27557 fn test_pool_meta_update_after_does_not_advance_last_update_before_save() {
27558 let last_update = OffsetDateTime::UNIX_EPOCH;
27559 let mut meta = PoolMeta {
27560 pools: vec![PoolStatus {
27561 id: 0,
27562 cmd_line: "pool-0".to_string(),
27563 last_update,
27564 decommission: Some(PoolDecommissionInfo {
27565 start_time: Some(last_update),
27566 items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
27567 ..Default::default()
27568 }),
27569 }],
27570 ..Default::default()
27571 };
27572
27573 assert!(
27574 meta.update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
27575 .expect("item threshold should request a checkpoint")
27576 );
27577 assert_eq!(meta.pools[0].last_update, last_update);
27578 }
27579
27580 #[test]
27581 fn test_decommission_progress_checkpoint_commits_exact_snapshot_watermark() {
27582 let start_time = OffsetDateTime::UNIX_EPOCH;
27583 let checkpoint_at = start_time + Duration::seconds(30);
27584 let mut meta = PoolMeta {
27585 pools: vec![PoolStatus {
27586 id: 0,
27587 cmd_line: "pool-0".to_string(),
27588 last_update: start_time,
27589 decommission: Some(PoolDecommissionInfo {
27590 start_time: Some(start_time),
27591 items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
27592 ..Default::default()
27593 }),
27594 }],
27595 ..Default::default()
27596 };
27597
27598 let checkpoint = meta
27599 .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None)
27600 .expect("valid decommission state should produce a checkpoint")
27601 .expect("item threshold should produce a checkpoint");
27602 meta.count_item(0, 1, false);
27603
27604 assert!(meta.commit_decommission_progress_checkpoint(0, checkpoint));
27605 let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
27606 assert_eq!(info.progress_save_item_baseline, checkpoint.counted_items);
27607 assert_eq!(info.items_since_last_progress_save(), 1);
27608 assert_eq!(meta.pools[0].last_update, checkpoint_at);
27609 }
27610
27611 #[test]
27612 fn test_decommission_progress_checkpoint_backoff_does_not_advance_baseline() {
27613 let start_time = OffsetDateTime::UNIX_EPOCH;
27614 let checkpoint_at = start_time + Duration::seconds(30);
27615 let retry_after = checkpoint_at + DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
27616 let mut meta = PoolMeta {
27617 pools: vec![PoolStatus {
27618 id: 0,
27619 cmd_line: "pool-0".to_string(),
27620 last_update: start_time,
27621 decommission: Some(PoolDecommissionInfo {
27622 start_time: Some(start_time),
27623 items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
27624 ..Default::default()
27625 }),
27626 }],
27627 ..Default::default()
27628 };
27629
27630 let checkpoint = meta
27631 .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None)
27632 .expect("valid decommission state should produce a checkpoint")
27633 .expect("item threshold should produce a checkpoint");
27634 meta.defer_decommission_progress_checkpoint(0, checkpoint, retry_after);
27635
27636 assert!(
27637 meta.decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None)
27638 .expect("retry backoff check should succeed")
27639 .is_none()
27640 );
27641 assert_eq!(meta.pools[0].last_update, start_time);
27642 assert_eq!(
27643 meta.pools[0]
27644 .decommission
27645 .as_ref()
27646 .expect("decommission info should exist")
27647 .progress_save_item_baseline,
27648 0
27649 );
27650 }
27651
27652 #[test]
27653 fn test_decommission_progress_checkpoint_count_scales_with_threshold() {
27654 let start_time = OffsetDateTime::UNIX_EPOCH;
27655 let checkpoint_at = start_time;
27656 let mut meta = PoolMeta {
27657 pools: vec![PoolStatus {
27658 id: 0,
27659 cmd_line: "pool-0".to_string(),
27660 last_update: start_time,
27661 decommission: Some(PoolDecommissionInfo {
27662 start_time: Some(start_time),
27663 ..Default::default()
27664 }),
27665 }],
27666 ..Default::default()
27667 };
27668 let mut checkpoint_count = 0;
27669
27670 for _ in 0..(DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD * 10) {
27671 meta.count_item(0, 1, false);
27672 if let Some(checkpoint) = meta
27673 .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None)
27674 .expect("valid decommission state should produce a checkpoint")
27675 {
27676 checkpoint_count += 1;
27677 assert!(meta.commit_decommission_progress_checkpoint(0, checkpoint));
27678 }
27679 }
27680
27681 assert_eq!(checkpoint_count, 10);
27682 }
27683
27684 #[test]
27685 fn test_ensure_decommission_not_rebalancing_rejects_running_rebalance() {
27686 let err = ensure_decommission_not_rebalancing(true).expect_err("rebalance running should be rejected");
27687 assert!(matches!(err, Error::RebalanceAlreadyRunning));
27688 }
27689
27690 #[test]
27691 fn test_ensure_decommission_not_rebalancing_allows_idle() {
27692 assert!(ensure_decommission_not_rebalancing(false).is_ok());
27693 }
27694
27695 #[test]
27696 fn test_ensure_decommission_start_rebalance_meta_allowed_rejects_active_rebalance() {
27697 let meta = RebalanceMeta {
27698 pool_stats: vec![RebalanceStats {
27699 participating: true,
27700 info: RebalanceInfo {
27701 status: RebalStatus::Started,
27702 ..Default::default()
27703 },
27704 ..Default::default()
27705 }],
27706 ..Default::default()
27707 };
27708
27709 let err = ensure_decommission_start_rebalance_meta_allowed(Some(&meta))
27710 .expect_err("persisted active rebalance should block decommission start");
27711
27712 assert!(matches!(err, Error::RebalanceAlreadyRunning));
27713 }
27714
27715 #[test]
27716 fn test_ensure_decommission_start_rebalance_meta_allowed_rejects_stopping_rebalance() {
27717 let meta = RebalanceMeta {
27718 pool_stats: vec![RebalanceStats {
27719 info: RebalanceInfo {
27720 status: RebalStatus::Started,
27721 stopping: true,
27722 ..Default::default()
27723 },
27724 ..Default::default()
27725 }],
27726 ..Default::default()
27727 };
27728
27729 let err = ensure_decommission_start_rebalance_meta_allowed(Some(&meta))
27730 .expect_err("persisted stopping rebalance should block decommission start");
27731
27732 assert!(matches!(err, Error::RebalanceAlreadyRunning));
27733 }
27734
27735 #[test]
27736 fn test_ensure_decommission_start_rebalance_meta_allowed_allows_terminal_or_missing_rebalance() {
27737 let terminal_meta = RebalanceMeta {
27738 pool_stats: vec![RebalanceStats {
27739 participating: true,
27740 info: RebalanceInfo {
27741 status: RebalStatus::Completed,
27742 ..Default::default()
27743 },
27744 ..Default::default()
27745 }],
27746 ..Default::default()
27747 };
27748
27749 assert!(ensure_decommission_start_rebalance_meta_allowed(Some(&terminal_meta)).is_ok());
27750 assert!(ensure_decommission_start_rebalance_meta_allowed(None).is_ok());
27751 }
27752
27753 #[test]
27754 fn test_ensure_local_decommission_pool_leaders_allows_local_first_endpoint() {
27755 let endpoints = EndpointServerPools::from(vec![
27756 decommission_test_pool_endpoint(0, false),
27757 decommission_test_pool_endpoint(1, true),
27758 ]);
27759
27760 assert!(ensure_local_decommission_pool_leaders(&endpoints, &[1]).is_ok());
27761 }
27762
27763 #[test]
27764 fn test_ensure_local_decommission_pool_leaders_rejects_remote_first_endpoint() {
27765 let endpoints = EndpointServerPools::from(vec![decommission_test_pool_endpoint(0, false)]);
27766
27767 let err = ensure_local_decommission_pool_leaders(&endpoints, &[0])
27768 .expect_err("remote first endpoint should reject local decommission start");
27769
27770 assert!(err.to_string().contains("must run on the pool first endpoint"));
27771 }
27772
27773 #[test]
27774 fn test_ensure_local_decommission_pool_leaders_rejects_empty_endpoints() {
27775 let endpoints = EndpointServerPools::from(vec![PoolEndpoints {
27776 legacy: false,
27777 set_count: 1,
27778 drives_per_set: 1,
27779 endpoints: Endpoints::from(Vec::<Endpoint>::new()),
27780 cmd_line: "pool-0".to_string(),
27781 platform: String::new(),
27782 }]);
27783
27784 let err = ensure_local_decommission_pool_leaders(&endpoints, &[0])
27785 .expect_err("pool without endpoints should reject local decommission start");
27786
27787 assert!(err.to_string().contains("has no configured endpoints"));
27788 }
27789
27790 #[test]
27791 fn test_decommission_meta_bucket_options_are_idempotent() {
27792 let opts = decommission_meta_bucket_options();
27793
27794 assert!(opts.force_create);
27795 }
27796
27797 #[test]
27798 fn test_is_decommission_active_true_only_when_not_terminal() {
27799 assert!(is_decommission_active(false, false, false));
27800 assert!(!is_decommission_active(true, false, false));
27801 assert!(!is_decommission_active(false, true, false));
27802 assert!(!is_decommission_active(false, false, true));
27803 }
27804
27805 #[test]
27806 fn test_ensure_decommission_generation_rejects_stale_or_queued_workers() {
27807 let generation = OffsetDateTime::UNIX_EPOCH;
27808 let mut meta = PoolMeta {
27809 pools: vec![PoolStatus {
27810 id: 0,
27811 cmd_line: "pool-0".to_string(),
27812 last_update: generation,
27813 decommission: Some(PoolDecommissionInfo {
27814 start_time: Some(generation),
27815 ..Default::default()
27816 }),
27817 }],
27818 ..Default::default()
27819 };
27820
27821 assert!(ensure_decommission_generation(&meta, 0, generation).is_ok());
27822 assert!(ensure_decommission_generation(&meta, 0, generation + Duration::seconds(1)).is_err());
27823
27824 meta.pools[0]
27825 .decommission
27826 .as_mut()
27827 .expect("decommission metadata should exist")
27828 .queued = true;
27829 assert!(ensure_decommission_generation(&meta, 0, generation).is_err());
27830
27831 let replacement_generation = generation + Duration::seconds(2);
27832 let info = meta.pools[0]
27833 .decommission
27834 .as_mut()
27835 .expect("decommission metadata should exist");
27836 info.queued = false;
27837 info.start_time = Some(replacement_generation);
27838 assert!(ensure_decommission_generation(&meta, 0, generation).is_err());
27839 assert!(ensure_decommission_generation(&meta, 0, replacement_generation).is_ok());
27840 }
27841
27842 #[test]
27843 fn test_pool_meta_has_active_decommission_counts_running_and_queued_states() {
27844 let active_meta = PoolMeta {
27845 pools: vec![PoolStatus {
27846 id: 0,
27847 cmd_line: "pool-0".to_string(),
27848 last_update: OffsetDateTime::UNIX_EPOCH,
27849 decommission: Some(PoolDecommissionInfo {
27850 start_time: Some(OffsetDateTime::UNIX_EPOCH),
27851 ..Default::default()
27852 }),
27853 }],
27854 ..Default::default()
27855 };
27856 let queued_meta = PoolMeta {
27857 pools: vec![PoolStatus {
27858 id: 0,
27859 cmd_line: "pool-0".to_string(),
27860 last_update: OffsetDateTime::UNIX_EPOCH,
27861 decommission: Some(PoolDecommissionInfo {
27862 queued: true,
27863 ..Default::default()
27864 }),
27865 }],
27866 ..Default::default()
27867 };
27868
27869 assert!(pool_meta_has_active_decommission(&active_meta));
27870 assert!(pool_meta_has_active_decommission(&queued_meta));
27871 }
27872
27873 #[test]
27874 fn test_pool_meta_has_active_decommission_ignores_capacity_placeholder() {
27875 let meta = PoolMeta {
27876 pools: vec![PoolStatus {
27877 id: 0,
27878 cmd_line: "pool-0".to_string(),
27879 last_update: OffsetDateTime::UNIX_EPOCH,
27880 decommission: Some(PoolDecommissionInfo {
27881 total_size: 100,
27882 current_size: 75,
27883 ..Default::default()
27884 }),
27885 }],
27886 ..Default::default()
27887 };
27888
27889 assert!(!pool_meta_has_active_decommission(&meta));
27890 assert!(!meta.is_suspended(0));
27891 assert_eq!(decommission_start_pool_state(meta.pools.first()), DecommissionStartPoolState::Active);
27892 }
27893
27894 #[test]
27895 fn test_pool_meta_has_active_decommission_ignores_terminal_states() {
27896 let terminal_meta = PoolMeta {
27897 pools: vec![
27898 PoolStatus {
27899 id: 0,
27900 cmd_line: "pool-0".to_string(),
27901 last_update: OffsetDateTime::UNIX_EPOCH,
27902 decommission: Some(PoolDecommissionInfo {
27903 complete: true,
27904 ..Default::default()
27905 }),
27906 },
27907 PoolStatus {
27908 id: 1,
27909 cmd_line: "pool-1".to_string(),
27910 last_update: OffsetDateTime::UNIX_EPOCH,
27911 decommission: Some(PoolDecommissionInfo {
27912 failed: true,
27913 ..Default::default()
27914 }),
27915 },
27916 PoolStatus {
27917 id: 2,
27918 cmd_line: "pool-2".to_string(),
27919 last_update: OffsetDateTime::UNIX_EPOCH,
27920 decommission: Some(PoolDecommissionInfo {
27921 canceled: true,
27922 ..Default::default()
27923 }),
27924 },
27925 ],
27926 ..Default::default()
27927 };
27928
27929 assert!(!pool_meta_has_active_decommission(&terminal_meta));
27930 }
27931
27932 #[test]
27933 fn test_decommission_request_rejections_preserve_invalid_argument_type() {
27934 for result in [
27935 ensure_decommission_start_allowed(DecommissionStartPoolState::Missing),
27936 ensure_decommission_start_allowed(DecommissionStartPoolState::Decommissioned),
27937 ensure_decommission_start_allowed(DecommissionStartPoolState::Blocked),
27938 ensure_decommission_cancel_allowed(false, false, false),
27939 ensure_decommission_clear_allowed(false, false, false, false, false, 0),
27940 ensure_decommission_clear_allowed(true, true, false, true, false, 1),
27941 ensure_decommission_terminal_operation_supported(true, "cancel decommission"),
27942 validate_start_decommission_request(&[], false),
27943 validate_start_decommission_request(&[0], true),
27944 ensure_decommission_start_keeps_active_pool(&PoolMeta::default(), &[]),
27945 ] {
27946 let err = result.expect_err("invalid lifecycle requests must be rejected before mutation");
27947 assert!(matches!(&err, Error::InvalidArgument(_, _, reason) if !reason.is_empty()), "{err:?}");
27948 }
27949 }
27950
27951 #[test]
27952 fn test_ensure_decommission_start_allowed_rejects_missing_pool() {
27953 let err =
27954 ensure_decommission_start_allowed(DecommissionStartPoolState::Missing).expect_err("missing pool should be invalid");
27955 assert!(
27956 err.to_string()
27957 .contains("failed to start decommission: target pool was not found")
27958 );
27959 }
27960
27961 #[test]
27962 fn test_ensure_decommission_start_allowed_rejects_running_state() {
27963 let err = ensure_decommission_start_allowed(DecommissionStartPoolState::Decommissioning)
27964 .expect_err("active decommission should be rejected");
27965 assert!(matches!(err, Error::DecommissionAlreadyRunning));
27966 }
27967
27968 #[test]
27969 fn test_ensure_decommission_start_allowed_rejects_completed_state() {
27970 let err = ensure_decommission_start_allowed(DecommissionStartPoolState::Decommissioned)
27971 .expect_err("completed decommission should be rejected");
27972 assert!(err.to_string().contains("target pool is already decommissioned"));
27973 }
27974
27975 #[test]
27976 fn test_ensure_decommission_start_allowed_rejects_blocked_state() {
27977 let err = ensure_decommission_start_allowed(DecommissionStartPoolState::Blocked)
27978 .expect_err("blocked decommission should be rejected");
27979 assert!(err.to_string().contains("target pool decommission is blocked"));
27980 }
27981
27982 #[test]
27983 fn test_ensure_decommission_start_allowed_allows_active_state() {
27984 assert!(ensure_decommission_start_allowed(DecommissionStartPoolState::Active).is_ok());
27985 }
27986
27987 #[test]
27988 fn test_ensure_decommission_start_allowed_allows_retryable_state() {
27989 assert!(ensure_decommission_start_allowed(DecommissionStartPoolState::Retryable).is_ok());
27990 }
27991
27992 #[test]
27993 fn test_decommission_start_pool_state_reports_missing_pool() {
27994 assert_eq!(decommission_start_pool_state(None), DecommissionStartPoolState::Missing);
27995 }
27996
27997 #[test]
27998 fn test_decommission_start_pool_state_reports_idle_pool_without_decommission_info() {
27999 let pool = PoolStatus {
28000 id: 0,
28001 cmd_line: "pool-0".to_string(),
28002 last_update: OffsetDateTime::UNIX_EPOCH,
28003 decommission: None,
28004 };
28005
28006 assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Active);
28007 }
28008
28009 #[test]
28010 fn test_decommission_start_pool_state_reports_decommissioning_pool_when_not_terminal() {
28011 let pool = PoolStatus {
28012 id: 0,
28013 cmd_line: "pool-0".to_string(),
28014 last_update: OffsetDateTime::UNIX_EPOCH,
28015 decommission: Some(PoolDecommissionInfo {
28016 start_time: Some(OffsetDateTime::UNIX_EPOCH),
28017 complete: false,
28018 failed: false,
28019 canceled: false,
28020 ..Default::default()
28021 }),
28022 };
28023
28024 assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Decommissioning);
28025 }
28026
28027 #[test]
28028 fn test_decommission_start_pool_state_reports_canceled_pool_as_blocked() {
28029 let pool = PoolStatus {
28030 id: 0,
28031 cmd_line: "pool-0".to_string(),
28032 last_update: OffsetDateTime::UNIX_EPOCH,
28033 decommission: Some(PoolDecommissionInfo {
28034 complete: false,
28035 failed: false,
28036 canceled: true,
28037 ..Default::default()
28038 }),
28039 };
28040
28041 assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Blocked);
28042 }
28043
28044 #[test]
28045 fn test_decommission_start_pool_state_reports_failed_pool_as_blocked() {
28046 let pool = PoolStatus {
28047 id: 0,
28048 cmd_line: "pool-0".to_string(),
28049 last_update: OffsetDateTime::UNIX_EPOCH,
28050 decommission: Some(PoolDecommissionInfo {
28051 failed: true,
28052 ..Default::default()
28053 }),
28054 };
28055
28056 assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Blocked);
28057 }
28058
28059 #[test]
28060 fn test_decommission_start_pool_state_reports_terminal_pool_with_unresolved_entries_as_retryable() {
28061 let pool = PoolStatus {
28062 id: 0,
28063 cmd_line: "pool-0".to_string(),
28064 last_update: OffsetDateTime::UNIX_EPOCH,
28065 decommission: Some(PoolDecommissionInfo {
28066 failed: true,
28067 unresolved_entries: vec![DecommissionUnresolvedEntry {
28068 bucket: "bucket-a".to_string(),
28069 object: "object-a".to_string(),
28070 pool_index: 0,
28071 set_index: 0,
28072 source_generation: OffsetDateTime::UNIX_EPOCH,
28073 candidate_count: 1,
28074 disk_error_count: 1,
28075 observed_at: OffsetDateTime::UNIX_EPOCH,
28076 reason: "metadata_resolution_failed".to_string(),
28077 }],
28078 ..Default::default()
28079 }),
28080 };
28081
28082 assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Retryable);
28083 }
28084
28085 #[test]
28086 fn test_decommission_start_pool_state_reports_completed_pool() {
28087 let pool = PoolStatus {
28088 id: 0,
28089 cmd_line: "pool-0".to_string(),
28090 last_update: OffsetDateTime::UNIX_EPOCH,
28091 decommission: Some(PoolDecommissionInfo {
28092 complete: true,
28093 ..Default::default()
28094 }),
28095 };
28096
28097 assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Decommissioned);
28098 }
28099
28100 #[test]
28101 fn test_ensure_decommission_start_keeps_active_pool_rejects_last_active_pool() {
28102 let meta = PoolMeta {
28103 pools: vec![PoolStatus {
28104 id: 0,
28105 cmd_line: "pool-0".to_string(),
28106 last_update: OffsetDateTime::UNIX_EPOCH,
28107 decommission: None,
28108 }],
28109 ..Default::default()
28110 };
28111
28112 let err = ensure_decommission_start_keeps_active_pool(&meta, &[0]).expect_err("last active pool should be rejected");
28113
28114 assert!(err.to_string().contains("at least one active pool must remain"));
28115 }
28116
28117 #[test]
28118 fn decommission_capacity_model_selection_uses_v2_only_with_a_live_fleet_proof() {
28119 let meta = PoolMeta::default();
28120
28121 assert_eq!(
28122 select_decommission_capacity_model(&meta, false).expect("a proofless empty cohort should remain compatible"),
28123 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION
28124 );
28125 assert_eq!(
28126 select_decommission_capacity_model(&meta, true).expect("an all-v4 proof should authorize the target fence"),
28127 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
28128 );
28129 }
28130
28131 #[test]
28132 fn decommission_capacity_model_selection_keeps_an_active_v1_cohort_sticky() {
28133 let meta = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION]);
28134
28135 assert_eq!(
28136 select_decommission_capacity_model(&meta, true).expect("a fleet upgrade must not change an active v1 lock model"),
28137 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION
28138 );
28139 }
28140
28141 #[test]
28142 fn decommission_capacity_model_selection_never_downgrades_an_active_v2_cohort() {
28143 let meta = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION]);
28144
28145 assert_eq!(
28146 select_decommission_capacity_model(&meta, true).expect("a live proof should admit another v2 reservation"),
28147 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
28148 );
28149 let err = select_decommission_capacity_model(&meta, false)
28150 .expect_err("proof loss must block new admission instead of falling back to the global lock model");
28151 assert!(err.to_string().contains("active per-target capacity cohort"));
28152 }
28153
28154 #[test]
28155 fn decommission_capacity_model_selection_rejects_a_mixed_active_cohort() {
28156 let meta = decommission_test_active_model_meta(&[
28157 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION,
28158 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
28159 ]);
28160
28161 let err = select_decommission_capacity_model(&meta, true)
28162 .expect_err("mixed global and per-target lock models cannot be made safe by a fleet proof");
28163 assert!(err.to_string().contains("mixed lock models"));
28164 }
28165
28166 #[test]
28167 fn test_ensure_decommission_start_target_capacity_allows_sufficient_free_space() {
28168 let meta = PoolMeta {
28169 version: POOL_META_VERSION,
28170 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
28171 ..Default::default()
28172 };
28173 let capacity_infos = vec![
28174 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 100, 700, 600),
28175 DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 1_600, 2_000, 400),
28176 ];
28177
28178 assert!(ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos, true).is_ok());
28179 }
28180
28181 #[test]
28182 fn test_ensure_decommission_start_target_capacity_rejects_insufficient_free_space() {
28183 let meta = PoolMeta {
28184 version: POOL_META_VERSION,
28185 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
28186 ..Default::default()
28187 };
28188 let capacity_infos = vec![
28189 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 100, 700, 600),
28190 DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 1_599, 2_000, 401),
28191 ];
28192
28193 let err = ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos, true)
28194 .expect_err("target physical free capacity below the modeled peak should be rejected");
28195
28196 assert!(err.to_string().contains("insufficient reserved physical target capacity"));
28197 assert!(err.to_string().contains("requires 1600 bytes, but 1599 bytes are available"));
28198 }
28199
28200 #[test]
28201 fn test_ensure_decommission_start_target_capacity_ignores_non_active_target_pool() {
28202 let meta = PoolMeta {
28203 version: POOL_META_VERSION,
28204 pools: vec![
28205 decommission_test_pool_status(0, None),
28206 decommission_test_pool_status(
28207 1,
28208 Some(PoolDecommissionInfo {
28209 complete: true,
28210 ..Default::default()
28211 }),
28212 ),
28213 decommission_test_pool_status(2, None),
28214 ],
28215 ..Default::default()
28216 };
28217 let capacity_infos = vec![
28218 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 100, 700, 600),
28219 DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 10_000, 10_000, 0),
28220 DecommissionPoolCapacityInfo::for_test(2, DecommissionErasureLayout { data: 4, parity: 4 }, 1_599, 2_000, 401),
28221 ];
28222
28223 let err = ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos, true)
28224 .expect_err("completed pools must not contribute target free capacity");
28225
28226 assert!(err.to_string().contains("requires 1600 bytes, but 1599 bytes are available"));
28227 }
28228
28229 #[test]
28230 fn decommission_capacity_model_converts_different_source_and_target_parity() {
28231 let now = OffsetDateTime::UNIX_EPOCH;
28232 let high_target_parity = build_decommission_capacity_reservation(
28233 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 0, 600, 600),
28234 DecommissionErasureLayout { data: 4, parity: 4 },
28235 uuid::Uuid::new_v4(),
28236 1,
28237 now,
28238 )
28239 .expect("the source and target layouts should be valid");
28240 assert_eq!(high_target_parity.source_data_equivalent_bytes, 400);
28241 assert_eq!(high_target_parity.predicted_physical_bytes, 800);
28242 assert_eq!(high_target_parity.temporary_physical_bytes, 800);
28243 assert_eq!(high_target_parity.peak_physical_bytes, 1_600);
28244
28245 let low_target_parity = build_decommission_capacity_reservation(
28246 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 4 }, 0, 800, 800),
28247 DecommissionErasureLayout { data: 4, parity: 2 },
28248 uuid::Uuid::new_v4(),
28249 2,
28250 now,
28251 )
28252 .expect("the inverse source and target layouts should be valid");
28253 assert_eq!(low_target_parity.source_data_equivalent_bytes, 400);
28254 assert_eq!(low_target_parity.predicted_physical_bytes, 600);
28255 assert_eq!(low_target_parity.peak_physical_bytes, 1_200);
28256 }
28257
28258 #[test]
28259 fn decommission_batch_persists_one_replayable_operation_identity_and_generation() {
28260 let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(1);
28261 let operation_id = uuid::Uuid::new_v4();
28262 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
28263 let capacity_infos = vec![
28264 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
28265 DecommissionPoolCapacityInfo::for_test(1, layout, 0, 20, 20),
28266 DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
28267 ];
28268 let mut meta = PoolMeta {
28269 version: POOL_META_VERSION,
28270 pools: vec![
28271 decommission_test_pool_status(0, None),
28272 decommission_test_pool_status(1, None),
28273 decommission_test_pool_status(2, None),
28274 ],
28275 ..Default::default()
28276 };
28277 meta.decommission(0, capacity_infos[0].space).unwrap();
28278 meta.queue_decommission(1, capacity_infos[1].space).unwrap();
28279
28280 reserve_decommission_start_target_capacity(
28281 &mut meta,
28282 &[0, 1],
28283 &capacity_infos,
28284 operation_id,
28285 17,
28286 now,
28287 DECOMMISSION_CAPACITY_MODEL_VERSION,
28288 )
28289 .expect("the batch reservation should fit exactly");
28290 for idx in [0, 1] {
28291 let reservation = meta.pools[idx]
28292 .decommission
28293 .as_ref()
28294 .and_then(|info| info.capacity_reservation.as_ref())
28295 .expect("each source pool should carry the batch reservation identity");
28296 assert_eq!(reservation.operation_id, operation_id);
28297 assert_eq!(reservation.generation, 17);
28298 }
28299 }
28300
28301 #[test]
28302 fn decommission_capacity_model_reserves_versions_delete_markers_and_temporary_copies_from_physical_usage() {
28303 let reservation = build_decommission_capacity_reservation(
28304 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 2, parity: 2 }, 0, 1_024, 1_024),
28305 DecommissionErasureLayout { data: 2, parity: 2 },
28306 uuid::Uuid::new_v4(),
28307 1,
28308 OffsetDateTime::UNIX_EPOCH,
28309 )
28310 .expect("physical source usage should produce a reservation");
28311
28312 assert_eq!(reservation.source_physical_bytes, 1_024);
28313 assert_eq!(reservation.predicted_physical_bytes, 1_024);
28314 assert_eq!(reservation.temporary_copies, 1);
28315 assert_eq!(reservation.peak_physical_bytes, 2_048);
28316 }
28317
28318 #[test]
28319 fn decommission_runtime_capacity_tracks_own_target_consumption_but_rejects_external_drop() {
28320 let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
28321 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
28322 let initial = vec![
28323 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
28324 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
28325 ];
28326 let mut meta = PoolMeta {
28327 version: POOL_META_VERSION,
28328 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
28329 ..Default::default()
28330 };
28331 meta.decommission(0, initial[0].space).unwrap();
28332 reserve_decommission_start_target_capacity(
28333 &mut meta,
28334 &[0],
28335 &initial,
28336 uuid::Uuid::new_v4(),
28337 1,
28338 now,
28339 DECOMMISSION_CAPACITY_MODEL_VERSION,
28340 )
28341 .expect("the initial target capacity should fit exactly");
28342
28343 let mutation_id = uuid::Uuid::from_u128(1);
28344 reserve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id, now + Duration::seconds(1))
28345 .expect("the target intent should be durable before its write");
28346 let own_consumption = vec![initial[0], DecommissionPoolCapacityInfo::for_test(1, layout, 50, 60, 10)];
28347 ensure_decommission_capacity_reservations_available(&meta, &own_consumption, "restart")
28348 .expect("a persisted target intent must recognize its own write after restart");
28349 resolve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id)
28350 .expect("the persisted target intent should resolve at commit");
28351 record_decommission_target_consumption(
28352 &mut meta,
28353 0,
28354 1,
28355 DecommissionTargetConsumption {
28356 committed_data_bytes: 10,
28357 target_physical_bytes: 10,
28358 observed_physical_bytes: 10,
28359 },
28360 mutation_id,
28361 now + Duration::seconds(1),
28362 )
28363 .expect("the operation's own target consumption should be persisted");
28364 ensure_decommission_capacity_reservations_available(&meta, &own_consumption, "migration")
28365 .expect("the operation's own committed target bytes must not look like external capacity loss");
28366
28367 let dropped = vec![initial[0], DecommissionPoolCapacityInfo::for_test(1, layout, 39, 60, 21)];
28368 let err = ensure_decommission_capacity_reservations_available(&meta, &dropped, "migration")
28369 .expect_err("runtime migration must stop after a sudden competing write consumes reserved capacity");
28370 assert!(
28371 err.to_string()
28372 .contains("requires 40 physical bytes, but only 39 bytes remain")
28373 );
28374 assert!(is_decommission_capacity_blocked_error(&err));
28375 }
28376
28377 #[test]
28378 fn decommission_target_capacity_reuses_only_matching_pending_intent() {
28379 let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
28380 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
28381 let initial = vec![
28382 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
28383 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
28384 ];
28385 let mut meta = PoolMeta {
28386 version: POOL_META_VERSION,
28387 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
28388 ..Default::default()
28389 };
28390 meta.decommission(0, initial[0].space).unwrap();
28391 reserve_decommission_start_target_capacity(
28392 &mut meta,
28393 &[0],
28394 &initial,
28395 uuid::Uuid::new_v4(),
28396 1,
28397 now,
28398 DECOMMISSION_CAPACITY_MODEL_VERSION,
28399 )
28400 .expect("the initial target capacity should fit exactly");
28401
28402 let first_mutation_id = uuid::Uuid::from_u128(1);
28403 let second_mutation_id = uuid::Uuid::from_u128(2);
28404 reserve_decommission_target_pending(&mut meta, 0, 1, 10, first_mutation_id, now + Duration::seconds(1))
28405 .expect("the first mutation should persist its target intent");
28406 assert_eq!(
28407 reserve_decommission_target_pending(&mut meta, 0, 1, 10, first_mutation_id, now + Duration::seconds(2))
28408 .expect("the same mutation should reuse its persisted target intent"),
28409 0
28410 );
28411 let err = reserve_decommission_target_pending(&mut meta, 0, 1, 10, second_mutation_id, now + Duration::seconds(2))
28412 .expect_err("a second mutation must not reuse the first mutation's pending intent");
28413 assert!(is_decommission_capacity_blocked_error(&err));
28414 assert!(err.to_string().contains("unresolved target capacity intent"));
28415
28416 resolve_decommission_target_pending(&mut meta, 0, 1, 10, first_mutation_id)
28417 .expect("the first mutation should finalize its intent");
28418 record_decommission_target_consumption(
28419 &mut meta,
28420 0,
28421 1,
28422 DecommissionTargetConsumption {
28423 committed_data_bytes: 10,
28424 target_physical_bytes: 10,
28425 observed_physical_bytes: 10,
28426 },
28427 first_mutation_id,
28428 now + Duration::seconds(2),
28429 )
28430 .expect("the first mutation's consumption should be durable");
28431 reserve_decommission_target_pending(&mut meta, 0, 1, 10, second_mutation_id, now + Duration::seconds(3))
28432 .expect("the second mutation should retry only after the first intent is finalized");
28433 assert_eq!(
28434 meta.pools[0]
28435 .decommission
28436 .as_ref()
28437 .and_then(|info| info.capacity_reservation.as_ref())
28438 .expect("the reservation should remain present")
28439 .pending_target_physical_bytes,
28440 10
28441 );
28442 }
28443
28444 #[test]
28445 fn decommission_target_capacity_allows_same_mutation_to_grow_its_pending_stage() {
28446 let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
28447 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
28448 let initial = vec![
28449 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
28450 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
28451 ];
28452 let mut meta = PoolMeta {
28453 version: POOL_META_VERSION,
28454 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
28455 ..Default::default()
28456 };
28457 meta.decommission(0, initial[0].space).unwrap();
28458 reserve_decommission_start_target_capacity(
28459 &mut meta,
28460 &[0],
28461 &initial,
28462 uuid::Uuid::new_v4(),
28463 1,
28464 now,
28465 DECOMMISSION_CAPACITY_MODEL_VERSION,
28466 )
28467 .expect("the initial target capacity should fit exactly");
28468
28469 let mutation_id = uuid::Uuid::from_u128(1);
28470 assert_eq!(
28471 reserve_decommission_target_pending(&mut meta, 0, 1, 1, mutation_id, now + Duration::seconds(1))
28472 .expect("the initial multipart stage should persist its one-byte intent"),
28473 1
28474 );
28475 assert_eq!(
28476 reserve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id, now + Duration::seconds(2))
28477 .expect("the same mutation should grow its intent for the commit stage"),
28478 9
28479 );
28480 assert_eq!(
28481 meta.pools[0]
28482 .decommission
28483 .as_ref()
28484 .and_then(|info| info.capacity_reservation.as_ref())
28485 .expect("the reservation should remain present")
28486 .targets[0]
28487 .pending_physical_bytes,
28488 10
28489 );
28490 assert_eq!(
28491 reserve_decommission_target_pending(&mut meta, 0, 1, 1, mutation_id, now + Duration::seconds(3))
28492 .expect("a retry of an earlier stage must reuse the larger same-mutation intent"),
28493 0
28494 );
28495 }
28496
28497 #[test]
28498 fn decommission_capacity_mutation_identity_survives_lease_nonce_rotation() {
28499 let owner = DecommissionCapacityOwner {
28500 source_pool_index: 0,
28501 operation_id: uuid::Uuid::from_u128(1),
28502 generation: 2,
28503 owner_nonce: uuid::Uuid::from_u128(3),
28504 mutation_id: None,
28505 };
28506 let recovered_owner = DecommissionCapacityOwner {
28507 owner_nonce: uuid::Uuid::from_u128(4),
28508 ..owner
28509 };
28510 let first = decommission_capacity_mutation_id(owner, "bucket", "object", Some("version"), false, None);
28511 let recovered = decommission_capacity_mutation_id(recovered_owner, "bucket", "object", Some("version"), false, None);
28512 assert_eq!(first, recovered, "a lease nonce rotation must not change the mutation identity");
28513 }
28514
28515 #[test]
28516 fn exact_delete_capacity_plan_requires_identity_and_exact_size() {
28517 let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
28518 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
28519 let capacity_infos = vec![
28520 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
28521 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
28522 ];
28523 let mut meta = PoolMeta {
28524 version: POOL_META_VERSION,
28525 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
28526 ..Default::default()
28527 };
28528 meta.decommission(0, capacity_infos[0].space).unwrap();
28529 reserve_decommission_start_target_capacity(
28530 &mut meta,
28531 &[0],
28532 &capacity_infos,
28533 uuid::Uuid::new_v4(),
28534 1,
28535 now,
28536 DECOMMISSION_CAPACITY_MODEL_VERSION,
28537 )
28538 .expect("the exact-delete test reservation should fit");
28539 let exact = ObjectInfo {
28540 bucket: "bucket".to_string(),
28541 name: "object".to_string(),
28542 version_id: Some(uuid::Uuid::from_u128(7)),
28543 mod_time: Some(now),
28544 size: 10,
28545 ..Default::default()
28546 };
28547 let owner = {
28548 let reservation = meta.pools[0]
28549 .decommission
28550 .as_ref()
28551 .and_then(|info| info.capacity_reservation.as_ref())
28552 .expect("the exact-delete test reservation should exist");
28553 DecommissionCapacityOwner {
28554 source_pool_index: 0,
28555 operation_id: reservation.operation_id,
28556 generation: reservation.generation,
28557 owner_nonce: reservation.owner_nonce,
28558 mutation_id: None,
28559 }
28560 };
28561 let version_id = exact.version_id.map(|version_id| version_id.to_string());
28562 let mutation_id = decommission_capacity_mutation_id(
28563 owner,
28564 &exact.bucket,
28565 &exact.name,
28566 version_id.as_deref(),
28567 exact.delete_marker,
28568 exact.mod_time,
28569 );
28570 reserve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id, now + Duration::seconds(1))
28571 .expect("the exact-delete test intent should be reserved");
28572
28573 let plan = plan_exact_delete_capacity_reconciliations(&meta, &exact.name, &exact)
28574 .expect("the exact identity should match the pending intent");
28575 assert_eq!(plan.len(), 1);
28576 assert_eq!(plan[0].source_pool_index, 0);
28577 assert_eq!(plan[0].target_pool_index, 1);
28578 assert_eq!(plan[0].expected_data_bytes, 10);
28579 assert_eq!(plan[0].expected_target_physical_bytes, 10);
28580
28581 let mismatched_size = ObjectInfo {
28582 size: 9,
28583 ..exact.clone()
28584 };
28585 let mismatched_size = plan_exact_delete_capacity_reconciliations(&meta, &mismatched_size.name, &mismatched_size)
28586 .expect_err("a different exact size must not consume the pending intent");
28587 assert!(mismatched_size.to_string().contains("does not match the exact object size"));
28588
28589 let directory_exact = ObjectInfo {
28590 name: "directory/".to_string(),
28591 ..exact.clone()
28592 };
28593 let internal_directory = rustfs_utils::path::encode_dir_object(&directory_exact.name);
28594 let internal_directory_mutation_id = decommission_capacity_mutation_id(
28595 owner,
28596 &directory_exact.bucket,
28597 &internal_directory,
28598 version_id.as_deref(),
28599 directory_exact.delete_marker,
28600 directory_exact.mod_time,
28601 );
28602 meta.pools[0]
28603 .decommission
28604 .as_mut()
28605 .and_then(|info| info.capacity_reservation.as_mut())
28606 .expect("the exact-delete test reservation should exist")
28607 .targets[0]
28608 .pending_mutation_id = Some(internal_directory_mutation_id);
28609 let directory_plan = plan_exact_delete_capacity_reconciliations(&meta, &internal_directory, &directory_exact)
28610 .expect("an internally encoded directory intent should match its logical exact object");
28611 assert_eq!(directory_plan[0].mutation_id, internal_directory_mutation_id);
28612
28613 let logical_directory_mutation_id = decommission_capacity_mutation_id(
28614 owner,
28615 &directory_exact.bucket,
28616 &directory_exact.name,
28617 version_id.as_deref(),
28618 directory_exact.delete_marker,
28619 directory_exact.mod_time,
28620 );
28621 meta.pools[0]
28622 .decommission
28623 .as_mut()
28624 .and_then(|info| info.capacity_reservation.as_mut())
28625 .expect("the exact-delete test reservation should exist")
28626 .targets[0]
28627 .pending_mutation_id = Some(logical_directory_mutation_id);
28628 let directory_plan = plan_exact_delete_capacity_reconciliations(&meta, &internal_directory, &directory_exact)
28629 .expect("a logical directory intent should match its internally encoded delete path");
28630 assert_eq!(directory_plan[0].mutation_id, logical_directory_mutation_id);
28631
28632 meta.pools[0]
28633 .decommission
28634 .as_mut()
28635 .and_then(|info| info.capacity_reservation.as_mut())
28636 .expect("the exact-delete test reservation should exist")
28637 .targets[0]
28638 .pending_mutation_id = None;
28639 let unidentified = plan_exact_delete_capacity_reconciliations(&meta, &exact.name, &exact)
28640 .expect_err("an unidentified pending intent must fail closed");
28641 assert!(unidentified.to_string().contains("without an object identity"));
28642
28643 let mut opts = ObjectOptions::default();
28644 let unfenced = ensure_exact_delete_capacity_namespace_fences(&opts, &exact.bucket, &exact.name)
28645 .expect_err("capacity reconciliation must reject a missing object namespace fence");
28646 assert!(unfenced.to_string().contains("requires an object namespace fence"));
28647 opts.ensure_namespace_lock_fence();
28648 ensure_exact_delete_capacity_namespace_fences(&opts, &exact.bucket, &exact.name)
28649 .expect("a live object namespace fence should admit capacity reconciliation");
28650 }
28651
28652 #[test]
28653 fn ordinary_write_admission_shares_a_reserved_target_without_becoming_its_owner() {
28654 let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
28655 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
28656 let capacity_infos = vec![
28657 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
28658 DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
28659 ];
28660 let mut meta = PoolMeta {
28661 version: POOL_META_VERSION,
28662 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
28663 ..Default::default()
28664 };
28665 meta.decommission(0, capacity_infos[0].space).unwrap();
28666 reserve_decommission_start_target_capacity(
28667 &mut meta,
28668 &[0],
28669 &capacity_infos,
28670 uuid::Uuid::new_v4(),
28671 1,
28672 now,
28673 DECOMMISSION_CAPACITY_MODEL_VERSION,
28674 )
28675 .expect("the decommission reservation should fit");
28676
28677 for admission in [
28678 DecommissionCapacityAdmission::Mutation,
28679 DecommissionCapacityAdmission::BatchDelete,
28680 DecommissionCapacityAdmission::ScannerBacklog,
28681 ] {
28682 ensure_external_decommission_target_admission(&meta, 1, admission)
28683 .expect("a healthy target must remain writable while sharing capacity with migration");
28684 }
28685 assert!(matches!(
28686 ensure_external_decommission_target_admission(&meta, 1, DecommissionCapacityAdmission::Heal),
28687 Err(Error::SlowDown)
28688 ));
28689 let rebalance_opts = ObjectOptions {
28690 data_movement: true,
28691 src_pool_idx: 0,
28692 ..Default::default()
28693 };
28694 assert!(
28695 DecommissionCapacityOwner::from_options(&rebalance_opts).is_none(),
28696 "rebalance data movement must remain an external capacity consumer"
28697 );
28698 let reservation = meta.pools[0]
28699 .decommission
28700 .as_ref()
28701 .and_then(|info| info.capacity_reservation.as_ref())
28702 .expect("the active reservation should remain available");
28703 let expected_owner = DecommissionCapacityOwner {
28704 source_pool_index: 0,
28705 operation_id: reservation.operation_id,
28706 generation: reservation.generation,
28707 owner_nonce: reservation.owner_nonce,
28708 mutation_id: None,
28709 };
28710 ensure_decommission_target_owner_admission(&meta, expected_owner, 1, 10, now)
28711 .expect("the reservation owner should consume its own allocation under the shared boundary");
28712 let mut decommission_opts = rebalance_opts;
28713 expected_owner.apply_to(&mut decommission_opts);
28714 assert_eq!(DecommissionCapacityOwner::from_options(&decommission_opts), Some(expected_owner));
28715
28716 meta.pools[0]
28717 .decommission
28718 .as_mut()
28719 .expect("active source")
28720 .capacity_reservation = None;
28721 for admission in [
28722 DecommissionCapacityAdmission::Mutation,
28723 DecommissionCapacityAdmission::BatchDelete,
28724 DecommissionCapacityAdmission::ScannerBacklog,
28725 ] {
28726 assert!(
28727 matches!(ensure_external_decommission_target_admission(&meta, 1, admission), Err(Error::SlowDown)),
28728 "shared capacity must not bypass an active source's missing durable ledger"
28729 );
28730 }
28731 }
28732
28733 #[test]
28734 fn external_decommission_admission_fences_suspended_sources_but_preserves_repair() {
28735 let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
28736 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
28737 let capacity_infos = vec![
28738 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
28739 DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
28740 ];
28741 let mut active = PoolMeta {
28742 version: POOL_META_VERSION,
28743 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
28744 ..Default::default()
28745 };
28746 active
28747 .decommission(0, capacity_infos[0].space)
28748 .expect("start the source admission fixture");
28749 reserve_decommission_start_target_capacity(
28750 &mut active,
28751 &[0],
28752 &capacity_infos,
28753 uuid::Uuid::new_v4(),
28754 1,
28755 now,
28756 DECOMMISSION_CAPACITY_MODEL_VERSION,
28757 )
28758 .expect("the active source must have a valid reservation to isolate its write fence");
28759
28760 for (state, queued, failed, canceled, complete) in [
28761 ("running", false, false, false, false),
28762 ("queued", true, false, false, false),
28763 ("failed", false, true, false, false),
28764 ("canceled", false, false, true, false),
28765 ("completed", false, false, false, true),
28766 ] {
28767 let mut meta = active.clone();
28768 let info = meta.pools[0].decommission.as_mut().expect("the source fixture must exist");
28769 info.queued = queued;
28770 info.failed = failed;
28771 info.canceled = canceled;
28772 info.complete = complete;
28773 if queued || failed || canceled || complete {
28774 info.start_time = None;
28775 }
28776 for admission in [
28777 DecommissionCapacityAdmission::Mutation,
28778 DecommissionCapacityAdmission::BatchDelete,
28779 ] {
28780 assert!(
28781 matches!(ensure_external_decommission_target_admission(&meta, 0, admission), Err(Error::SlowDown)),
28782 "{state} source must reject new publication until its decommission metadata is cleared"
28783 );
28784 }
28785 let existing_multipart =
28786 ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::ExistingMultipart);
28787 if active_decommission_source_indices(&meta).contains(&0) {
28788 existing_multipart
28789 .unwrap_or_else(|err| panic!("{state} source must allow an existing multipart upload to drain: {err}"));
28790 } else {
28791 assert!(
28792 matches!(existing_multipart, Err(Error::SlowDown)),
28793 "{state} terminal source must reject an existing multipart publication"
28794 );
28795 }
28796 ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::Heal)
28797 .unwrap_or_else(|err| panic!("{state} source repair must retain its capacity-only admission: {err}"));
28798 let scanner_result =
28799 ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::ScannerBacklog);
28800 assert_eq!(
28801 meta.scanner_pause_backlog_pool_writable(0),
28802 failed || canceled,
28803 "{state} scanner selection"
28804 );
28805 if failed || canceled {
28806 scanner_result.unwrap_or_else(|err| panic!("{state} scanner membership repair must remain writable: {err}"));
28807 } else {
28808 assert!(
28809 matches!(scanner_result, Err(Error::SlowDown)),
28810 "{state} scanner publication must reject its source"
28811 );
28812 }
28813 meta.pools[0].decommission = None;
28814 ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::Mutation)
28815 .unwrap_or_else(|err| panic!("cleared {state} source must become writable again: {err}"));
28816 ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::ScannerBacklog)
28817 .unwrap_or_else(|err| panic!("cleared {state} scanner source must rejoin membership: {err}"));
28818 }
28819 assert!(!active.scanner_pause_backlog_pool_writable(active.pools.len()));
28820 assert!(matches!(
28821 ensure_external_decommission_target_admission(
28822 &active,
28823 active.pools.len(),
28824 DecommissionCapacityAdmission::ScannerBacklog
28825 ),
28826 Err(Error::SlowDown)
28827 ));
28828 }
28829
28830 #[test]
28831 fn decommission_physical_capacity_uses_per_set_bottleneck_and_widest_usage() {
28832 let disks = [10_u64, 20, 30, 40]
28833 .into_iter()
28834 .enumerate()
28835 .map(|(disk_index, available_space)| rustfs_madmin::Disk {
28836 endpoint: format!("http://node-{disk_index}"),
28837 drive_path: format!("/disk-{disk_index}"),
28838 state: "ok".to_string(),
28839 total_space: 100,
28840 used_space: 100 - available_space,
28841 available_space,
28842 pool_index: 0,
28843 set_index: 0,
28844 disk_index: disk_index as i32,
28845 ..Default::default()
28846 })
28847 .collect::<Vec<_>>();
28848 let capacity = decommission_physical_pool_capacity(
28849 &disks,
28850 0,
28851 DecommissionErasureLayout { data: 2, parity: 2 },
28852 PoolSpaceInfo {
28853 free: 0,
28854 total: 0,
28855 used: 0,
28856 },
28857 );
28858
28859 assert_eq!(capacity, (400, 40, 360));
28860 }
28861
28862 #[test]
28863 fn decommission_terminal_transitions_release_capacity_reservations() {
28864 let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(1);
28865 let reservation = build_decommission_capacity_reservation(
28866 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 1, parity: 0 }, 0, 50, 50),
28867 DecommissionErasureLayout { data: 1, parity: 0 },
28868 uuid::Uuid::new_v4(),
28869 7,
28870 now,
28871 )
28872 .expect("test reservation should be valid");
28873 let active = PoolMeta {
28874 version: POOL_META_VERSION,
28875 pools: vec![decommission_test_pool_status(
28876 0,
28877 Some(PoolDecommissionInfo {
28878 start_time: Some(now),
28879 capacity_reservation: Some(reservation),
28880 ..Default::default()
28881 }),
28882 )],
28883 ..Default::default()
28884 };
28885
28886 for (reason, transition) in [
28887 (
28888 DECOMMISSION_CAPACITY_RELEASE_CANCELED,
28889 PoolMeta::decommission_cancel as fn(&mut PoolMeta, usize) -> bool,
28890 ),
28891 (DECOMMISSION_CAPACITY_RELEASE_FAILED, PoolMeta::decommission_failed),
28892 (DECOMMISSION_CAPACITY_RELEASE_COMPLETED, PoolMeta::decommission_complete),
28893 ] {
28894 let mut meta = active.clone();
28895 assert!(transition(&mut meta, 0));
28896 let released = meta.pools[0]
28897 .decommission
28898 .as_ref()
28899 .and_then(|info| info.capacity_reservation.as_ref())
28900 .expect("terminal metadata should retain reservation observability");
28901 assert!(released.released_at.is_some());
28902 assert_eq!(released.release_reason.as_deref(), Some(reason));
28903 assert_eq!(
28904 released.operation_id,
28905 active.pools[0]
28906 .decommission
28907 .as_ref()
28908 .unwrap()
28909 .capacity_reservation
28910 .as_ref()
28911 .unwrap()
28912 .operation_id
28913 );
28914 }
28915 }
28916
28917 #[test]
28918 fn decommission_capacity_target_round_trip_preserves_temporary_mutation_without_pending_intent() {
28919 let mutation_id = uuid::Uuid::new_v4();
28920 let target = DecommissionCapacityTarget {
28921 pool_index: 1,
28922 layout: DecommissionErasureLayout { data: 2, parity: 2 },
28923 physical_total_at_reservation: 200,
28924 physical_free_at_reservation: 200,
28925 reserved_physical_bytes: 200,
28926 consumed_physical_bytes: 0,
28927 observed_physical_bytes: 1,
28928 inflight_physical_bytes: 1,
28929 pending_physical_bytes: 0,
28930 pending_mutation_id: None,
28931 temporary_mutations: vec![DecommissionCapacityTemporaryMutation {
28932 mutation_id,
28933 physical_bytes: 1,
28934 }],
28935 };
28936 let mut encoded = Vec::new();
28937 target
28938 .serialize(&mut Serializer::new(&mut encoded))
28939 .expect("capacity target should serialize");
28940
28941 let restored: DecommissionCapacityTarget =
28942 rmp_serde::from_slice(&encoded).expect("capacity target with a released pending intent should deserialize");
28943
28944 assert_eq!(restored, target);
28945 }
28946
28947 #[test]
28948 fn temporary_cleanup_releases_only_its_exact_scoped_mutation_and_is_idempotent() {
28949 let mutation_id = uuid::Uuid::new_v4();
28950 let foreign_id = uuid::Uuid::new_v4();
28951 let mut meta = decommission_test_cleanup_meta(
28952 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
28953 vec![
28954 DecommissionCapacityTemporaryMutation {
28955 mutation_id,
28956 physical_bytes: 10,
28957 },
28958 DecommissionCapacityTemporaryMutation {
28959 mutation_id: foreign_id,
28960 physical_bytes: 20,
28961 },
28962 ],
28963 30,
28964 None,
28965 );
28966
28967 assert!(
28968 release_decommission_target_inflight(
28969 &mut meta,
28970 0,
28971 1,
28972 0,
28973 mutation_id,
28974 DecommissionCapacityReleaseProof::confirmed_absence(true),
28975 OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
28976 )
28977 .expect("confirmed absence should release the exact mutation")
28978 );
28979 let reservation = meta.pools[0]
28980 .decommission
28981 .as_ref()
28982 .and_then(|info| info.capacity_reservation.as_ref())
28983 .expect("cleanup reservation should remain present");
28984 assert_eq!(reservation.inflight_target_physical_bytes, 20);
28985 assert_eq!(
28986 reservation.targets[0].temporary_mutations,
28987 vec![DecommissionCapacityTemporaryMutation {
28988 mutation_id: foreign_id,
28989 physical_bytes: 20,
28990 }]
28991 );
28992
28993 let before_replay = reservation.clone();
28994 assert!(
28995 !release_decommission_target_inflight(
28996 &mut meta,
28997 0,
28998 1,
28999 0,
29000 mutation_id,
29001 DecommissionCapacityReleaseProof::confirmed_absence(true),
29002 OffsetDateTime::UNIX_EPOCH + Duration::seconds(2),
29003 )
29004 .expect("repeated confirmed absence should be a metadata no-op")
29005 );
29006 let after_replay = meta.pools[0]
29007 .decommission
29008 .as_ref()
29009 .and_then(|info| info.capacity_reservation.as_ref())
29010 .expect("cleanup reservation should remain present");
29011 assert_eq!(after_replay, &before_replay);
29012 }
29013
29014 #[test]
29015 fn temporary_cleanup_uses_aggregate_fallback_only_for_legacy_unscoped_state() {
29016 let mutation_id = uuid::Uuid::new_v4();
29017 let mut meta = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, Vec::new(), 30, None);
29018
29019 assert!(
29020 release_decommission_target_inflight(
29021 &mut meta,
29022 0,
29023 1,
29024 12,
29025 mutation_id,
29026 DecommissionCapacityReleaseProof::confirmed_absence(true),
29027 OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
29028 )
29029 .expect("legacy unscoped cleanup should use its observed release delta")
29030 );
29031 let reservation = meta.pools[0]
29032 .decommission
29033 .as_ref()
29034 .and_then(|info| info.capacity_reservation.as_ref())
29035 .expect("legacy cleanup reservation should remain present");
29036 assert_eq!(reservation.inflight_target_physical_bytes, 18);
29037 assert_eq!(reservation.targets[0].inflight_physical_bytes, 18);
29038 }
29039
29040 #[test]
29041 fn temporary_cleanup_clears_only_a_matching_pending_identity_after_confirmed_absence() {
29042 let mutation_id = uuid::Uuid::new_v4();
29043 let mut meta = decommission_test_cleanup_meta(
29044 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
29045 Vec::new(),
29046 0,
29047 Some((mutation_id, 15)),
29048 );
29049
29050 assert!(
29051 release_decommission_target_inflight(
29052 &mut meta,
29053 0,
29054 1,
29055 7,
29056 mutation_id,
29057 DecommissionCapacityReleaseProof::confirmed_absence(true),
29058 OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
29059 )
29060 .expect("a confirmed absent upload should clear its matching pending identity")
29061 );
29062 let reservation = meta.pools[0]
29063 .decommission
29064 .as_ref()
29065 .and_then(|info| info.capacity_reservation.as_ref())
29066 .expect("cleanup reservation should remain present");
29067 assert_eq!(reservation.pending_target_physical_bytes, 0);
29068 assert_eq!(reservation.targets[0].pending_mutation_id, None);
29069 assert_eq!(reservation.targets[0].pending_physical_bytes, 0);
29070 }
29071
29072 #[test]
29073 fn temporary_cleanup_preserves_pending_for_an_already_published_target() {
29074 let mutation_id = uuid::Uuid::new_v4();
29075 let mut meta = decommission_test_cleanup_meta(
29076 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
29077 vec![DecommissionCapacityTemporaryMutation {
29078 mutation_id,
29079 physical_bytes: 5,
29080 }],
29081 5,
29082 Some((mutation_id, 15)),
29083 );
29084
29085 assert!(
29086 release_decommission_target_inflight(
29087 &mut meta,
29088 0,
29089 1,
29090 0,
29091 mutation_id,
29092 DecommissionCapacityReleaseProof::confirmed_absence(false),
29093 OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
29094 )
29095 .expect("published-target cleanup should release only its temporary staging state")
29096 );
29097 let reservation = meta.pools[0]
29098 .decommission
29099 .as_ref()
29100 .and_then(|info| info.capacity_reservation.as_ref())
29101 .expect("cleanup reservation should remain present");
29102 assert_eq!(reservation.inflight_target_physical_bytes, 0);
29103 assert_eq!(reservation.pending_target_physical_bytes, 15);
29104 assert_eq!(reservation.targets[0].pending_mutation_id, Some(mutation_id));
29105 assert_eq!(reservation.targets[0].pending_physical_bytes, 15);
29106 assert!(reservation.targets[0].temporary_mutations.is_empty());
29107 }
29108
29109 #[test]
29110 fn published_pending_ignores_a_delayed_release_observation() {
29111 let mutation_id = uuid::Uuid::new_v4();
29112 let mut meta = decommission_test_cleanup_meta(
29113 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
29114 Vec::new(),
29115 0,
29116 Some((mutation_id, 15)),
29117 );
29118 let before = meta.pools[0]
29119 .decommission
29120 .as_ref()
29121 .and_then(|info| info.capacity_reservation.as_ref())
29122 .expect("published-target reservation should remain present")
29123 .clone();
29124
29125 assert!(
29126 !release_decommission_target_inflight(
29127 &mut meta,
29128 0,
29129 1,
29130 7,
29131 mutation_id,
29132 DecommissionCapacityReleaseProof::confirmed_absence(false),
29133 OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
29134 )
29135 .expect("a delayed statfs release must not block published-target reconciliation")
29136 );
29137 let after = meta.pools[0]
29138 .decommission
29139 .as_ref()
29140 .and_then(|info| info.capacity_reservation.as_ref())
29141 .expect("published-target reservation should remain present");
29142 assert_eq!(after, &before);
29143 }
29144
29145 #[test]
29146 fn temporary_cleanup_fails_closed_on_a_foreign_scoped_release_delta() {
29147 let foreign_id = uuid::Uuid::new_v4();
29148 let mut meta = decommission_test_cleanup_meta(
29149 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
29150 vec![DecommissionCapacityTemporaryMutation {
29151 mutation_id: foreign_id,
29152 physical_bytes: 20,
29153 }],
29154 20,
29155 None,
29156 );
29157 let before = meta.pools[0]
29158 .decommission
29159 .as_ref()
29160 .and_then(|info| info.capacity_reservation.as_ref())
29161 .expect("cleanup reservation should remain present")
29162 .clone();
29163
29164 let err = release_decommission_target_inflight(
29165 &mut meta,
29166 0,
29167 1,
29168 5,
29169 uuid::Uuid::new_v4(),
29170 DecommissionCapacityReleaseProof::confirmed_absence(true),
29171 OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
29172 )
29173 .expect_err("an observed release cannot be charged to a foreign scoped mutation");
29174 assert!(err.to_string().contains("cannot be attributed"));
29175 let after = meta.pools[0]
29176 .decommission
29177 .as_ref()
29178 .and_then(|info| info.capacity_reservation.as_ref())
29179 .expect("cleanup reservation should remain present");
29180 assert_eq!(after, &before);
29181 }
29182
29183 #[test]
29184 fn temporary_cleanup_missing_identity_and_nonzero_delta_is_a_metadata_noop() {
29185 let mut meta = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, Vec::new(), 0, None);
29186 let before = meta.pools[0]
29187 .decommission
29188 .as_ref()
29189 .and_then(|info| info.capacity_reservation.as_ref())
29190 .expect("cleanup reservation should remain present")
29191 .clone();
29192
29193 assert!(
29194 !release_decommission_target_inflight(
29195 &mut meta,
29196 0,
29197 1,
29198 7,
29199 uuid::Uuid::new_v4(),
29200 DecommissionCapacityReleaseProof::confirmed_absence(true),
29201 OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
29202 )
29203 .expect("confirmed absence without tracked state should ignore a delayed capacity observation")
29204 );
29205 let after = meta.pools[0]
29206 .decommission
29207 .as_ref()
29208 .and_then(|info| info.capacity_reservation.as_ref())
29209 .expect("cleanup reservation should remain present");
29210 assert_eq!(after, &before);
29211 }
29212
29213 #[test]
29214 fn temporary_cleanup_missing_identity_fails_closed_on_foreign_pending_state() {
29215 let foreign_id = uuid::Uuid::new_v4();
29216 let mut meta = decommission_test_cleanup_meta(
29217 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
29218 Vec::new(),
29219 0,
29220 Some((foreign_id, 11)),
29221 );
29222 let before = meta.pools[0]
29223 .decommission
29224 .as_ref()
29225 .and_then(|info| info.capacity_reservation.as_ref())
29226 .expect("cleanup reservation should remain present")
29227 .clone();
29228
29229 let err = release_decommission_target_inflight(
29230 &mut meta,
29231 0,
29232 1,
29233 7,
29234 uuid::Uuid::new_v4(),
29235 DecommissionCapacityReleaseProof::confirmed_absence(true),
29236 OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
29237 )
29238 .expect_err("a delayed release observation must not clear a foreign pending mutation");
29239 assert!(err.to_string().contains("cannot be attributed"));
29240 let after = meta.pools[0]
29241 .decommission
29242 .as_ref()
29243 .and_then(|info| info.capacity_reservation.as_ref())
29244 .expect("cleanup reservation should remain present");
29245 assert_eq!(after, &before);
29246 }
29247
29248 #[test]
29249 fn pool_meta_v2_round_trip_preserves_each_supported_capacity_lock_model() {
29250 for model_version in [
29251 DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION,
29252 DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
29253 ] {
29254 let meta = decommission_test_cleanup_meta(model_version, Vec::new(), 0, None);
29255 let encoded = meta
29256 .encode_config_data_for_test()
29257 .expect("supported capacity lock model should encode in pool metadata V2");
29258 let mut restored = PoolMeta::default();
29259 restored
29260 .load_from_config_data(encoded)
29261 .expect("supported capacity lock model should decode from pool metadata V2");
29262 assert_eq!(
29263 restored.pools[0]
29264 .decommission
29265 .as_ref()
29266 .and_then(|info| info.capacity_reservation.as_ref())
29267 .expect("round-tripped reservation should remain present")
29268 .model_version,
29269 model_version
29270 );
29271 }
29272 }
29273
29274 #[test]
29275 fn pool_meta_v2_decode_rejects_mixed_active_capacity_lock_models() {
29276 let mut mixed = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, Vec::new(), 0, None);
29277 let mut target_fence_source =
29278 decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, Vec::new(), 0, None)
29279 .pools
29280 .remove(0);
29281 target_fence_source.id = 2;
29282 target_fence_source.cmd_line = "pool-2".to_string();
29283 target_fence_source
29284 .decommission
29285 .as_mut()
29286 .and_then(|info| info.capacity_reservation.as_mut())
29287 .expect("second active reservation should remain present")
29288 .source_pool_index = 2;
29289 mixed.pools.push(target_fence_source);
29290
29291 let encoded = mixed
29292 .encode_config_data_for_test()
29293 .expect("the decode test must be able to construct a mixed persisted payload");
29294 let mut restored = PoolMeta::default();
29295 let err = restored
29296 .load_from_config_data(encoded)
29297 .expect_err("mixed active lock models must fail closed while loading pool metadata");
29298 assert!(err.to_string().contains("mixed lock models"));
29299 }
29300
29301 #[test]
29302 fn decommission_capacity_reservation_recovers_expired_lease_after_restart_round_trip() {
29303 let created_at = OffsetDateTime::UNIX_EPOCH + Duration::hours(1);
29304 let recovered_at = created_at + DECOMMISSION_CAPACITY_RESERVATION_TTL + Duration::seconds(1);
29305 let mut reservation = build_decommission_capacity_reservation(
29306 DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 2, parity: 2 }, 0, 100, 100),
29307 DecommissionErasureLayout { data: 2, parity: 2 },
29308 uuid::Uuid::new_v4(),
29309 41,
29310 created_at,
29311 )
29312 .expect("test reservation should be valid");
29313 reservation.targets.push(DecommissionCapacityTarget {
29314 pool_index: 1,
29315 layout: DecommissionErasureLayout { data: 2, parity: 2 },
29316 physical_total_at_reservation: 200,
29317 physical_free_at_reservation: 200,
29318 reserved_physical_bytes: 200,
29319 consumed_physical_bytes: 0,
29320 observed_physical_bytes: 0,
29321 inflight_physical_bytes: 0,
29322 pending_physical_bytes: 0,
29323 pending_mutation_id: None,
29324 temporary_mutations: Vec::new(),
29325 });
29326 let operation_id = reservation.operation_id;
29327 let owner_nonce = reservation.owner_nonce;
29328 let mut meta = PoolMeta {
29329 version: POOL_META_VERSION,
29330 pools: vec![decommission_test_pool_status(
29331 0,
29332 Some(PoolDecommissionInfo {
29333 start_time: Some(created_at),
29334 capacity_reservation: Some(reservation),
29335 ..Default::default()
29336 }),
29337 )],
29338 ..Default::default()
29339 };
29340
29341 let encoded = meta
29342 .encode_config_data_for_test()
29343 .expect("capacity reservation should persist in V2");
29344 let mut restored = PoolMeta::default();
29345 restored
29346 .load_from_config_data(encoded)
29347 .expect("capacity reservation should survive a restart round trip");
29348 let info = restored.pools[0]
29349 .decommission
29350 .as_mut()
29351 .expect("active decommission should restore");
29352 let recovered = info.capacity_reservation.as_mut().expect("reservation should remain present");
29353 assert!(renew_decommission_capacity_reservation(recovered, recovered_at, true));
29354 assert_eq!(recovered.operation_id, operation_id);
29355 assert_eq!(recovered.generation, 41);
29356 assert_ne!(recovered.owner_nonce, owner_nonce);
29357 assert_eq!(recovered.recovered_at, Some(recovered_at));
29358 assert_eq!(recovered.expires_at, recovered_at + DECOMMISSION_CAPACITY_RESERVATION_TTL);
29359 assert_eq!(next_decommission_capacity_generation(&restored).unwrap(), 42);
29360
29361 meta.decommission_failed(0);
29362 let terminal = meta.pools[0]
29363 .decommission
29364 .as_ref()
29365 .and_then(|info| info.capacity_reservation.as_ref())
29366 .expect("failed metadata should retain the released reservation");
29367 assert!(!terminal.active());
29368 }
29369
29370 #[test]
29371 fn decommission_restart_reconstructs_a_missing_capacity_reservation_fail_closed() {
29372 let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(2);
29373 let mut meta = PoolMeta {
29374 version: POOL_META_VERSION,
29375 pools: vec![
29376 decommission_test_pool_status(
29377 0,
29378 Some(PoolDecommissionInfo {
29379 start_time: Some(now - Duration::minutes(1)),
29380 ..Default::default()
29381 }),
29382 ),
29383 decommission_test_pool_status(1, None),
29384 ],
29385 ..Default::default()
29386 };
29387 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
29388 let capacity_infos = vec![
29389 DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
29390 DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
29391 ];
29392
29393 let recovered_indices = recover_decommission_capacity_reservations(&mut meta, &capacity_infos, now, true)
29394 .expect("restart recovery should rebuild the missing reservation while capacity still fits");
29395 assert_eq!(recovered_indices, vec![0]);
29396 let reservation = meta.pools[0]
29397 .decommission
29398 .as_ref()
29399 .and_then(|info| info.capacity_reservation.as_ref())
29400 .expect("restart recovery should persist a replacement reservation");
29401 assert_eq!(reservation.peak_physical_bytes, 60);
29402 assert_eq!(reservation.recovered_at, Some(now));
29403 assert_eq!(reservation.generation, 1);
29404 }
29405
29406 #[test]
29407 fn test_ensure_decommission_start_pool_states_rejects_blocked_pool() {
29408 let meta = PoolMeta {
29409 pools: vec![
29410 PoolStatus {
29411 id: 0,
29412 cmd_line: "pool-0".to_string(),
29413 last_update: OffsetDateTime::UNIX_EPOCH,
29414 decommission: Some(PoolDecommissionInfo {
29415 failed: true,
29416 ..Default::default()
29417 }),
29418 },
29419 PoolStatus {
29420 id: 1,
29421 cmd_line: "pool-1".to_string(),
29422 last_update: OffsetDateTime::UNIX_EPOCH,
29423 decommission: None,
29424 },
29425 ],
29426 ..Default::default()
29427 };
29428
29429 let err = ensure_decommission_start_pool_states(&meta, &[0]).expect_err("blocked pool should be rejected");
29430
29431 assert!(err.to_string().contains("target pool decommission is blocked"));
29432 }
29433
29434 #[test]
29435 fn test_ensure_decommission_start_pool_states_allows_active_pool_with_remaining_active_pool() {
29436 let meta = PoolMeta {
29437 pools: vec![
29438 PoolStatus {
29439 id: 0,
29440 cmd_line: "pool-0".to_string(),
29441 last_update: OffsetDateTime::UNIX_EPOCH,
29442 decommission: None,
29443 },
29444 PoolStatus {
29445 id: 1,
29446 cmd_line: "pool-1".to_string(),
29447 last_update: OffsetDateTime::UNIX_EPOCH,
29448 decommission: None,
29449 },
29450 ],
29451 ..Default::default()
29452 };
29453
29454 assert!(ensure_decommission_start_pool_states(&meta, &[0]).is_ok());
29455 }
29456
29457 #[test]
29458 fn test_ensure_decommission_start_pool_states_allows_retryable_pool_with_active_peer() {
29459 let meta = PoolMeta {
29460 pools: vec![
29461 PoolStatus {
29462 id: 0,
29463 cmd_line: "pool-0".to_string(),
29464 last_update: OffsetDateTime::UNIX_EPOCH,
29465 decommission: Some(PoolDecommissionInfo {
29466 failed: true,
29467 unresolved_entries: vec![DecommissionUnresolvedEntry {
29468 bucket: "bucket-a".to_string(),
29469 object: "object-a".to_string(),
29470 pool_index: 0,
29471 set_index: 0,
29472 source_generation: OffsetDateTime::UNIX_EPOCH,
29473 candidate_count: 1,
29474 disk_error_count: 1,
29475 observed_at: OffsetDateTime::UNIX_EPOCH,
29476 reason: "metadata_resolution_failed".to_string(),
29477 }],
29478 ..Default::default()
29479 }),
29480 },
29481 PoolStatus {
29482 id: 1,
29483 cmd_line: "pool-1".to_string(),
29484 last_update: OffsetDateTime::UNIX_EPOCH,
29485 decommission: None,
29486 },
29487 ],
29488 ..Default::default()
29489 };
29490
29491 assert!(ensure_decommission_start_pool_states(&meta, &[0]).is_ok());
29492 }
29493
29494 #[test]
29495 fn test_ensure_valid_decommission_pool_index_accepts_in_range_index() {
29496 assert!(ensure_valid_decommission_pool_index(4, 3).is_ok());
29497 }
29498
29499 #[test]
29500 fn test_ensure_valid_decommission_pool_index_rejects_out_of_range_index() {
29501 let err = ensure_valid_decommission_pool_index(2, 2).expect_err("out-of-range index should fail");
29502 assert!(err.to_string().contains("invalid decommission pool index 2 for 2 pools"));
29503 }
29504
29505 #[test]
29506 fn test_ensure_valid_decommission_pool_index_rejects_when_pool_count_zero() {
29507 let err = ensure_valid_decommission_pool_index(0, 0).expect_err("empty pool list should reject all indices");
29508 assert!(err.to_string().contains("invalid decommission pool index 0 for 0 pools"));
29509 }
29510
29511 #[test]
29512 fn test_classify_decommission_terminal_state_completed_when_no_failures() {
29513 assert_eq!(classify_decommission_terminal_state(false), DecommissionTerminalState::Completed);
29514 }
29515
29516 #[test]
29517 fn test_classify_decommission_terminal_state_failed_when_failures_present() {
29518 assert_eq!(classify_decommission_terminal_state(true), DecommissionTerminalState::Failed);
29519 }
29520
29521 #[test]
29522 fn test_should_preserve_decommission_canceled_state_when_meta_canceled() {
29523 assert!(should_preserve_decommission_canceled_state(true, false));
29524 }
29525
29526 #[test]
29527 fn test_should_preserve_decommission_canceled_state_when_signal_canceled() {
29528 assert!(!should_preserve_decommission_canceled_state(false, true));
29529 }
29530
29531 #[test]
29532 fn test_should_preserve_decommission_canceled_state_when_not_canceled() {
29533 assert!(!should_preserve_decommission_canceled_state(false, false));
29534 }
29535
29536 #[test]
29537 fn test_should_continue_decommission_queue_requires_clean_completion() {
29538 let meta = PoolMeta {
29539 pools: vec![
29540 PoolStatus {
29541 id: 0,
29542 cmd_line: "pool-0".to_string(),
29543 last_update: OffsetDateTime::UNIX_EPOCH,
29544 decommission: Some(PoolDecommissionInfo {
29545 complete: true,
29546 ..Default::default()
29547 }),
29548 },
29549 PoolStatus {
29550 id: 1,
29551 cmd_line: "pool-1".to_string(),
29552 last_update: OffsetDateTime::UNIX_EPOCH,
29553 decommission: Some(PoolDecommissionInfo::default()),
29554 },
29555 PoolStatus {
29556 id: 2,
29557 cmd_line: "pool-2".to_string(),
29558 last_update: OffsetDateTime::UNIX_EPOCH,
29559 decommission: Some(PoolDecommissionInfo {
29560 failed: true,
29561 ..Default::default()
29562 }),
29563 },
29564 PoolStatus {
29565 id: 3,
29566 cmd_line: "pool-3".to_string(),
29567 last_update: OffsetDateTime::UNIX_EPOCH,
29568 decommission: Some(PoolDecommissionInfo {
29569 canceled: true,
29570 ..Default::default()
29571 }),
29572 },
29573 PoolStatus {
29574 id: 4,
29575 cmd_line: "pool-4".to_string(),
29576 last_update: OffsetDateTime::UNIX_EPOCH,
29577 decommission: None,
29578 },
29579 ],
29580 ..Default::default()
29581 };
29582
29583 assert!(should_continue_decommission_queue(&meta, 0));
29584 assert!(!should_continue_decommission_queue(&meta, 1));
29585 assert!(!should_continue_decommission_queue(&meta, 2));
29586 assert!(!should_continue_decommission_queue(&meta, 3));
29587 assert!(!should_continue_decommission_queue(&meta, 4));
29588 assert!(!should_continue_decommission_queue(&meta, 5));
29589 }
29590
29591 #[test]
29592 fn test_decommission_cancel_signal_result_returns_err_when_canceled() {
29593 let err = decommission_cancel_signal_result(true).expect_err("canceled signal should return operation-canceled");
29594 assert!(matches!(err, Error::OperationCanceled));
29595 }
29596
29597 #[test]
29598 fn test_decommission_cancel_signal_result_returns_ok_when_not_canceled() {
29599 assert!(decommission_cancel_signal_result(false).is_ok());
29600 }
29601
29602 #[test]
29603 fn test_is_decommission_cancel_requested_accepts_signal_or_metadata() {
29604 let pool = PoolStatus {
29605 id: 0,
29606 cmd_line: "pool-0".to_string(),
29607 last_update: OffsetDateTime::UNIX_EPOCH,
29608 decommission: Some(PoolDecommissionInfo {
29609 canceled: true,
29610 ..Default::default()
29611 }),
29612 };
29613
29614 assert!(is_decommission_cancel_requested(false, Some(&pool)));
29615 assert!(is_decommission_cancel_requested(true, None));
29616 }
29617
29618 #[test]
29619 fn test_is_decommission_cancel_requested_rejects_active_without_signal() {
29620 let pool = PoolStatus {
29621 id: 0,
29622 cmd_line: "pool-0".to_string(),
29623 last_update: OffsetDateTime::UNIX_EPOCH,
29624 decommission: Some(PoolDecommissionInfo::default()),
29625 };
29626
29627 assert!(!is_decommission_cancel_requested(false, Some(&pool)));
29628 assert!(!is_decommission_cancel_requested(false, None));
29629 }
29630
29631 #[test]
29632 fn test_skip_canceled_decommission_routine_only_for_terminal_canceled_state() {
29633 let canceled = PoolStatus {
29634 id: 0,
29635 cmd_line: "pool-0".to_string(),
29636 last_update: OffsetDateTime::UNIX_EPOCH,
29637 decommission: Some(PoolDecommissionInfo {
29638 canceled: true,
29639 ..Default::default()
29640 }),
29641 };
29642 let active = PoolStatus {
29643 id: 1,
29644 cmd_line: "pool-1".to_string(),
29645 last_update: OffsetDateTime::UNIX_EPOCH,
29646 decommission: Some(PoolDecommissionInfo::default()),
29647 };
29648
29649 assert!(should_skip_canceled_decommission_routine(true, Some(&canceled)));
29650 assert!(!should_skip_canceled_decommission_routine(false, Some(&canceled)));
29651 assert!(!should_skip_canceled_decommission_routine(true, Some(&active)));
29652 assert!(!should_skip_canceled_decommission_routine(true, None));
29653 }
29654
29655 #[test]
29656 fn test_ensure_decommission_cancel_allowed_rejects_missing_pool() {
29657 let err = ensure_decommission_cancel_allowed(false, false, false).expect_err("missing pool should be invalid");
29658 assert!(
29659 err.to_string()
29660 .contains("failed to cancel decommission: target pool was not found")
29661 );
29662 }
29663
29664 #[test]
29665 fn test_should_reject_decommission_cancel_as_terminal_true_when_completed() {
29666 assert!(should_reject_decommission_cancel_as_terminal(true, false));
29667 }
29668
29669 #[test]
29670 fn test_should_reject_decommission_cancel_as_terminal_true_when_failed() {
29671 assert!(should_reject_decommission_cancel_as_terminal(false, true));
29672 }
29673
29674 #[test]
29675 fn test_should_reject_decommission_cancel_as_terminal_false_when_active_or_canceled() {
29676 assert!(!should_reject_decommission_cancel_as_terminal(false, false));
29677 }
29678
29679 #[test]
29680 fn test_should_retry_decommission_cancel_reload_when_changed_or_already_canceled() {
29681 assert!(should_retry_decommission_cancel_reload(true, false));
29682 assert!(should_retry_decommission_cancel_reload(false, true));
29683 assert!(!should_retry_decommission_cancel_reload(false, false));
29684 }
29685
29686 #[test]
29687 fn test_ensure_decommission_cancel_allowed_rejects_not_started() {
29688 let err =
29689 ensure_decommission_cancel_allowed(true, false, false).expect_err("not-started decommission should be rejected");
29690 assert!(matches!(err, Error::DecommissionNotStarted));
29691 }
29692
29693 #[test]
29694 fn test_ensure_decommission_cancel_allowed_rejects_terminal() {
29695 let err = ensure_decommission_cancel_allowed(true, true, true).expect_err("terminal decommission should be rejected");
29696 assert!(matches!(err, Error::DecommissionNotStarted));
29697 }
29698
29699 #[test]
29700 fn test_ensure_decommission_cancel_allowed_allows_active() {
29701 assert!(ensure_decommission_cancel_allowed(true, true, false).is_ok());
29702 }
29703
29704 #[test]
29705 fn test_ensure_decommission_clear_allowed_allows_failed_or_canceled() {
29706 assert!(ensure_decommission_clear_allowed(true, true, false, true, false, 0).is_ok());
29707 assert!(ensure_decommission_clear_allowed(true, true, false, false, true, 0).is_ok());
29708 }
29709
29710 #[test]
29711 fn test_ensure_decommission_clear_allowed_rejects_active_or_completed() {
29712 let active = ensure_decommission_clear_allowed(true, true, false, false, false, 0)
29713 .expect_err("active decommission should not be clearable");
29714 assert!(matches!(active, Error::DecommissionAlreadyRunning));
29715
29716 let complete = ensure_decommission_clear_allowed(true, true, true, false, false, 0)
29717 .expect_err("completed decommission should not be clearable");
29718 assert!(matches!(complete, Error::DecommissionNotStarted));
29719 }
29720
29721 #[test]
29722 fn test_ensure_decommission_clear_allowed_rejects_unresolved_entries() {
29723 let err = ensure_decommission_clear_allowed(true, true, false, true, false, 1)
29724 .expect_err("unresolved entries must survive until a retry reconciles them");
29725
29726 assert!(err.to_string().contains("must be reconciled by retrying decommission"));
29727 }
29728
29729 #[test]
29730 fn test_pool_meta_clear_decommission_restores_failed_or_canceled_pool() {
29731 for decommission in [
29732 PoolDecommissionInfo {
29733 failed: true,
29734 ..Default::default()
29735 },
29736 PoolDecommissionInfo {
29737 canceled: true,
29738 ..Default::default()
29739 },
29740 ] {
29741 let mut meta = PoolMeta {
29742 pools: vec![PoolStatus {
29743 id: 0,
29744 cmd_line: "pool-0".to_string(),
29745 last_update: OffsetDateTime::UNIX_EPOCH,
29746 decommission: Some(decommission),
29747 }],
29748 ..Default::default()
29749 };
29750
29751 assert!(meta.is_suspended(0));
29752 assert!(meta.clear_decommission(0).expect("terminal decommission should clear"));
29753 assert!(
29754 meta.pools[0]
29755 .decommission
29756 .as_ref()
29757 .is_some_and(|info| !info.has_decommission_state())
29758 );
29759 assert!(!meta.is_suspended(0));
29760 }
29761 }
29762
29763 #[test]
29764 fn test_pool_meta_clear_decommission_preserves_unresolved_entries_for_retry() {
29765 let generation = OffsetDateTime::UNIX_EPOCH;
29766 let unresolved_entry = DecommissionUnresolvedEntry {
29767 bucket: "bucket-a".to_string(),
29768 object: "object-a".to_string(),
29769 pool_index: 0,
29770 set_index: 0,
29771 source_generation: generation,
29772 candidate_count: 1,
29773 disk_error_count: 1,
29774 observed_at: generation,
29775 reason: "metadata_resolution_failed".to_string(),
29776 };
29777 let mut meta = PoolMeta {
29778 pools: vec![PoolStatus {
29779 id: 0,
29780 cmd_line: "pool-0".to_string(),
29781 last_update: generation,
29782 decommission: Some(PoolDecommissionInfo {
29783 failed: true,
29784 unresolved_entries: vec![unresolved_entry.clone()],
29785 ..Default::default()
29786 }),
29787 }],
29788 ..Default::default()
29789 };
29790
29791 let err = meta
29792 .clear_decommission(0)
29793 .expect_err("clear must not discard the unresolved-entry recovery ledger");
29794
29795 assert!(err.to_string().contains("must be reconciled by retrying decommission"));
29796 assert_eq!(
29797 meta.pools[0]
29798 .decommission
29799 .as_ref()
29800 .expect("failed state should remain retryable")
29801 .unresolved_entries,
29802 vec![unresolved_entry]
29803 );
29804 }
29805
29806 #[test]
29807 fn test_pool_meta_clear_decommission_rejects_active_or_completed_pool() {
29808 for decommission in [
29809 PoolDecommissionInfo::default(),
29810 PoolDecommissionInfo {
29811 complete: true,
29812 ..Default::default()
29813 },
29814 ] {
29815 let mut meta = PoolMeta {
29816 pools: vec![PoolStatus {
29817 id: 0,
29818 cmd_line: "pool-0".to_string(),
29819 last_update: OffsetDateTime::UNIX_EPOCH,
29820 decommission: Some(decommission),
29821 }],
29822 ..Default::default()
29823 };
29824
29825 assert!(meta.clear_decommission(0).is_err());
29826 assert!(meta.pools[0].decommission.is_some());
29827 }
29828 }
29829
29830 #[test]
29831 fn test_contextualized_decommission_terminal_operation_supported_rejects_single_pool() {
29832 let err = ensure_decommission_terminal_operation_supported(true, "complete decommission")
29833 .expect_err("single-pool decommission terminal operations should be rejected");
29834 assert!(
29835 err.to_string()
29836 .contains("failed to complete decommission: single pool deployments do not support decommission")
29837 );
29838 }
29839
29840 #[test]
29841 fn test_contextualized_decommission_terminal_operation_supported_allows_multi_pool() {
29842 assert!(ensure_decommission_terminal_operation_supported(false, "mark decommission failed").is_ok());
29843 }
29844
29845 #[test]
29846 fn test_contextualized_decommission_start_request_rejects_empty_indices() {
29847 let err = validate_start_decommission_request(&[], false).expect_err("empty decommission target list should be rejected");
29848 assert!(
29849 err.to_string()
29850 .contains("failed to start decommission: no target pools were provided")
29851 );
29852 }
29853
29854 #[test]
29855 fn test_contextualized_decommission_start_request_rejects_single_pool() {
29856 let err = validate_start_decommission_request(&[0], true)
29857 .expect_err("single-pool deployments should reject decommission start");
29858 assert!(
29859 err.to_string()
29860 .contains("failed to start decommission: single pool deployments do not support decommission")
29861 );
29862 }
29863
29864 #[test]
29865 fn test_contextualized_decommission_start_request_allows_multiple_target_pools() {
29866 assert!(validate_start_decommission_request(&[0, 1], false).is_ok());
29867 }
29868
29869 #[test]
29870 fn test_contextualized_decommission_start_request_allows_one_target_pool() {
29871 assert!(validate_start_decommission_request(&[0], false).is_ok());
29872 }
29873
29874 #[test]
29875 fn test_decommission_retry_preserves_and_rebinds_unresolved_entries() {
29876 let previous_generation = OffsetDateTime::UNIX_EPOCH;
29877 let unresolved_entry = DecommissionUnresolvedEntry {
29878 bucket: "bucket-a".to_string(),
29879 object: "object-a".to_string(),
29880 pool_index: 0,
29881 set_index: 0,
29882 source_generation: previous_generation,
29883 candidate_count: 1,
29884 disk_error_count: 1,
29885 observed_at: previous_generation,
29886 reason: "metadata_resolution_failed".to_string(),
29887 };
29888 let mut meta = PoolMeta {
29889 pools: vec![PoolStatus {
29890 id: 0,
29891 cmd_line: "pool-0".to_string(),
29892 last_update: previous_generation,
29893 decommission: Some(PoolDecommissionInfo {
29894 failed: true,
29895 decommissioned_buckets: vec!["bucket-a".to_string()],
29896 items_decommissioned: 7,
29897 bytes_done: 1024,
29898 unresolved_entries: vec![unresolved_entry],
29899 ..Default::default()
29900 }),
29901 }],
29902 ..Default::default()
29903 };
29904
29905 meta.decommission(
29906 0,
29907 PoolSpaceInfo {
29908 total: 200,
29909 free: 50,
29910 used: 150,
29911 },
29912 )
29913 .expect("failed decommission with unresolved entries should retry atomically");
29914
29915 let info = meta.pools[0]
29916 .decommission
29917 .as_ref()
29918 .expect("retried decommission metadata should exist");
29919 let next_generation = info.start_time.expect("retry should assign a new generation");
29920 assert!(next_generation > previous_generation);
29921 assert_eq!(info.decommissioned_buckets, vec!["bucket-a".to_string()]);
29922 assert_eq!(info.items_decommissioned, 7);
29923 assert_eq!(info.bytes_done, 1024);
29924 assert_eq!(info.unresolved_entries.len(), 1);
29925 assert_eq!(info.unresolved_entries[0].source_generation, next_generation);
29926 }
29927
29928 #[test]
29929 fn test_queued_decommission_retry_rebinds_unresolved_entries_when_promoted() {
29930 let previous_generation = OffsetDateTime::UNIX_EPOCH;
29931 let mut meta = PoolMeta {
29932 pools: vec![PoolStatus {
29933 id: 0,
29934 cmd_line: "pool-0".to_string(),
29935 last_update: previous_generation,
29936 decommission: Some(PoolDecommissionInfo {
29937 canceled: true,
29938 unresolved_entries: vec![DecommissionUnresolvedEntry {
29939 bucket: "bucket-a".to_string(),
29940 object: "object-a".to_string(),
29941 pool_index: 0,
29942 set_index: 0,
29943 source_generation: previous_generation,
29944 candidate_count: 1,
29945 disk_error_count: 1,
29946 observed_at: previous_generation,
29947 reason: "metadata_resolution_failed".to_string(),
29948 }],
29949 ..Default::default()
29950 }),
29951 }],
29952 ..Default::default()
29953 };
29954
29955 meta.queue_decommission(
29956 0,
29957 PoolSpaceInfo {
29958 total: 100,
29959 free: 25,
29960 used: 75,
29961 },
29962 )
29963 .expect("canceled decommission with unresolved entries should queue an atomic retry");
29964
29965 assert!(meta.is_suspended(0));
29966 assert!(meta.promote_queued_decommission(0));
29967 let promoted = meta.pools[0]
29968 .decommission
29969 .as_ref()
29970 .expect("promoted decommission metadata should exist");
29971 let generation = promoted.start_time.expect("promotion should assign a generation");
29972 assert_eq!(promoted.unresolved_entries.len(), 1);
29973 assert_eq!(promoted.unresolved_entries[0].source_generation, generation);
29974 }
29975
29976 #[test]
29977 fn test_queued_decommission_promotion_advances_generation_after_clock_rollback() {
29978 let queued_at = OffsetDateTime::from_unix_timestamp(1_260).expect("fixed timestamp should be valid");
29979 let earlier_tick = queued_at - Duration::nanoseconds(10);
29980 let rebalance_floor = queued_at + Duration::nanoseconds(5);
29981 let rebalance = RebalanceMeta {
29982 stopped_at: Some(rebalance_floor),
29983 id: "completed-rebalance".to_string(),
29984 ..Default::default()
29985 };
29986 let mut meta = PoolMeta {
29987 pools: vec![PoolStatus {
29988 id: 0,
29989 cmd_line: "pool-0".to_string(),
29990 last_update: queued_at,
29991 decommission: Some(PoolDecommissionInfo {
29992 queued: true,
29993 ..Default::default()
29994 }),
29995 }],
29996 ..Default::default()
29997 };
29998
29999 assert!(meta.promote_queued_decommission_at_for_test(0, earlier_tick, Some(&rebalance)));
30000 let expected_generation = rebalance_floor + Duration::nanoseconds(1);
30001 let promoted = meta.pools[0]
30002 .decommission
30003 .as_ref()
30004 .expect("promoted decommission metadata should exist");
30005 assert_eq!(meta.pools[0].last_update, expected_generation);
30006 assert_eq!(promoted.start_time, Some(expected_generation));
30007 assert!(!promoted.queued);
30008 }
30009
30010 #[test]
30011 fn test_pool_meta_queued_decommission_is_suspended_to_preserve_reserved_capacity() {
30012 let mut meta = PoolMeta {
30013 pools: vec![PoolStatus {
30014 id: 0,
30015 cmd_line: "pool-0".to_string(),
30016 last_update: OffsetDateTime::UNIX_EPOCH,
30017 decommission: None,
30018 }],
30019 ..Default::default()
30020 };
30021
30022 meta.queue_decommission(
30023 0,
30024 PoolSpaceInfo {
30025 total: 100,
30026 free: 10,
30027 used: 90,
30028 },
30029 )
30030 .expect("queued decommission should be stored");
30031
30032 assert!(meta.is_suspended(0));
30033 assert!(meta.promote_queued_decommission(0));
30034 assert!(meta.is_suspended(0));
30035 }
30036
30037 #[test]
30038 fn test_pool_meta_promoted_queued_decommission_can_be_canceled() {
30039 let mut meta = PoolMeta {
30040 pools: vec![PoolStatus {
30041 id: 0,
30042 cmd_line: "pool-0".to_string(),
30043 last_update: OffsetDateTime::UNIX_EPOCH,
30044 decommission: None,
30045 }],
30046 ..Default::default()
30047 };
30048
30049 meta.queue_decommission(
30050 0,
30051 PoolSpaceInfo {
30052 total: 100,
30053 free: 10,
30054 used: 90,
30055 },
30056 )
30057 .expect("queued decommission should be stored");
30058
30059 assert!(pool_meta_has_active_decommission(&meta));
30060 assert!(meta.promote_queued_decommission(0));
30061 assert!(meta.decommission_cancel(0));
30062
30063 let info = meta.pools[0]
30064 .decommission
30065 .as_ref()
30066 .expect("canceled decommission state should be kept for clear");
30067 assert!(info.canceled);
30068 assert!(!info.queued);
30069 assert!(!info.failed);
30070 assert!(!info.complete);
30071 assert!(!pool_meta_has_active_decommission(&meta));
30072 }
30073
30074 #[test]
30075 fn test_pool_meta_failed_decommission_requires_clear_before_restart() {
30076 let mut meta = PoolMeta {
30077 pools: vec![PoolStatus {
30078 id: 0,
30079 cmd_line: "pool-0".to_string(),
30080 last_update: OffsetDateTime::UNIX_EPOCH,
30081 decommission: Some(PoolDecommissionInfo {
30082 failed: true,
30083 decommissioned_buckets: vec!["bucket-done".to_string()],
30084 queued_buckets: vec!["bucket-pending".to_string()],
30085 bucket: "bucket-pending".to_string(),
30086 prefix: "prefix".to_string(),
30087 object: "object.txt".to_string(),
30088 items_decommissioned: 7,
30089 items_decommission_failed: 3,
30090 bytes_done: 1024,
30091 bytes_failed: 256,
30092 progress_save_item_baseline: 10,
30093 ..Default::default()
30094 }),
30095 }],
30096 ..Default::default()
30097 };
30098
30099 let err = meta
30100 .decommission(
30101 0,
30102 PoolSpaceInfo {
30103 total: 200,
30104 free: 50,
30105 used: 150,
30106 },
30107 )
30108 .expect_err("failed decommission should be blocked until cleared");
30109 assert!(err.to_string().contains("target pool decommission is blocked"));
30110 let blocked = meta.pools[0]
30111 .decommission
30112 .as_ref()
30113 .expect("blocked metadata should remain until clear");
30114 assert!(blocked.failed);
30115 assert_eq!(blocked.decommissioned_buckets, vec!["bucket-done".to_string()]);
30116 assert_eq!(blocked.items_decommissioned, 7);
30117 assert_eq!(blocked.bytes_done, 1024);
30118
30119 assert!(meta.clear_decommission(0).expect("failed decommission should clear"));
30120 assert!(
30121 meta.pools[0]
30122 .decommission
30123 .as_ref()
30124 .is_some_and(|info| !info.has_decommission_state())
30125 );
30126
30127 meta.decommission(
30128 0,
30129 PoolSpaceInfo {
30130 total: 200,
30131 free: 50,
30132 used: 150,
30133 },
30134 )
30135 .expect("cleared decommission should be restartable");
30136 meta.queue_buckets(
30137 0,
30138 vec![
30139 DecomBucketInfo {
30140 name: "bucket-done".to_string(),
30141 prefix: String::new(),
30142 },
30143 DecomBucketInfo {
30144 name: "bucket-pending".to_string(),
30145 prefix: String::new(),
30146 },
30147 ],
30148 );
30149
30150 let info = meta.pools[0]
30151 .decommission
30152 .as_ref()
30153 .expect("decommission info should be rebuilt");
30154 assert!(!info.failed);
30155 assert!(!info.canceled);
30156 assert!(!info.complete);
30157 assert!(info.decommissioned_buckets.is_empty());
30158 assert_eq!(info.queued_buckets, vec!["bucket-done".to_string(), "bucket-pending".to_string()]);
30159 assert_eq!(info.items_decommissioned, 0);
30160 assert_eq!(info.items_decommission_failed, 0);
30161 assert_eq!(info.bytes_done, 0);
30162 assert_eq!(info.bytes_failed, 0);
30163 assert_eq!(info.items_since_last_progress_save(), 0);
30164 assert_eq!(info.start_size, 50);
30165 assert_eq!(info.total_size, 200);
30166 assert_eq!(info.current_size, 50);
30167 assert_eq!(info.bucket, "bucket-pending");
30168 assert!(info.prefix.is_empty());
30169 assert!(info.object.is_empty());
30170 assert!(info.start_time.is_some());
30171 }
30172
30173 #[test]
30174 fn test_pool_meta_canceled_queued_decommission_requires_clear_before_restart() {
30175 let mut meta = PoolMeta {
30176 pools: vec![PoolStatus {
30177 id: 0,
30178 cmd_line: "pool-0".to_string(),
30179 last_update: OffsetDateTime::UNIX_EPOCH,
30180 decommission: Some(PoolDecommissionInfo {
30181 canceled: true,
30182 decommissioned_buckets: vec!["bucket-done".to_string()],
30183 items_decommissioned: 5,
30184 bytes_done: 512,
30185 ..Default::default()
30186 }),
30187 }],
30188 ..Default::default()
30189 };
30190
30191 let err = meta
30192 .queue_decommission(
30193 0,
30194 PoolSpaceInfo {
30195 total: 100,
30196 free: 25,
30197 used: 75,
30198 },
30199 )
30200 .expect_err("canceled queued decommission should be blocked until cleared");
30201 assert!(err.to_string().contains("target pool decommission is blocked"));
30202 let blocked = meta.pools[0]
30203 .decommission
30204 .as_ref()
30205 .expect("blocked metadata should remain until clear");
30206 assert!(blocked.canceled);
30207 assert_eq!(blocked.decommissioned_buckets, vec!["bucket-done".to_string()]);
30208 assert_eq!(blocked.items_decommissioned, 5);
30209 assert_eq!(blocked.bytes_done, 512);
30210
30211 assert!(meta.clear_decommission(0).expect("canceled decommission should clear"));
30212 assert!(
30213 meta.pools[0]
30214 .decommission
30215 .as_ref()
30216 .is_some_and(|info| !info.has_decommission_state())
30217 );
30218
30219 meta.queue_decommission(
30220 0,
30221 PoolSpaceInfo {
30222 total: 100,
30223 free: 25,
30224 used: 75,
30225 },
30226 )
30227 .expect("cleared queued decommission should be restartable");
30228
30229 let info = meta.pools[0]
30230 .decommission
30231 .as_ref()
30232 .expect("decommission info should be rebuilt");
30233 assert!(info.queued);
30234 assert!(info.start_time.is_none());
30235 assert!(info.decommissioned_buckets.is_empty());
30236 assert_eq!(info.items_decommissioned, 0);
30237 assert_eq!(info.bytes_done, 0);
30238 }
30239
30240 #[test]
30241 fn test_contextualized_decommission_listing_disks_available_rejects_empty_set() {
30242 let err = ensure_decommission_listing_disks_available(false, "bucket-a")
30243 .expect_err("missing online disks should be reported with bucket context");
30244 assert!(
30245 err.to_string()
30246 .contains("failed to list objects to decommission for bucket bucket-a: no disks available")
30247 );
30248 }
30249
30250 #[test]
30251 fn test_contextualized_decommission_listing_disks_available_allows_online_disks() {
30252 assert!(ensure_decommission_listing_disks_available(true, "bucket-a").is_ok());
30253 }
30254
30255 #[test]
30256 fn test_require_decommission_store_returns_value_when_present() {
30257 let store = require_decommission_store(Some(7_u8), "start decommission").expect("present store should be returned");
30258 assert_eq!(store, 7);
30259 }
30260
30261 #[test]
30262 fn test_require_decommission_store_returns_error_when_missing() {
30263 let err = require_decommission_store::<u8>(None, "start decommission").expect_err("missing store should return error");
30264 assert!(
30265 err.to_string()
30266 .contains("failed to start decommission: store not initialized")
30267 );
30268 }
30269
30270 #[test]
30271 fn test_bind_decommission_cancelers_binds_existing_slots_only() {
30272 let parent = CancellationToken::new();
30273 let mut cancelers = vec![None, None];
30274
30275 let bound = bind_decommission_cancelers(&[0, 3, 1], &parent, cancelers.as_mut_slice());
30276
30277 assert_eq!(bound.len(), 2);
30278 assert_eq!(bound[0].0, 0);
30279 assert_eq!(bound[1].0, 1);
30280 assert!(cancelers[0].is_some());
30281 assert!(cancelers[1].is_some());
30282 }
30283
30284 #[test]
30285 fn test_bind_decommission_cancelers_child_tokens_follow_parent_cancel() {
30286 let parent = CancellationToken::new();
30287 let mut cancelers = vec![None];
30288
30289 let bound = bind_decommission_cancelers(&[0], &parent, cancelers.as_mut_slice());
30290 assert_eq!(bound.len(), 1);
30291 assert!(!bound[0].1.is_cancelled());
30292
30293 parent.cancel();
30294 assert!(bound[0].1.is_cancelled());
30295 }
30296
30297 #[test]
30298 fn test_bind_decommission_cancelers_replaces_existing_slot() {
30299 let parent = CancellationToken::new();
30300 let existing = DecommissionCanceler::new(CancellationToken::new());
30301 let mut cancelers = vec![Some(existing.clone())];
30302
30303 let bound = bind_decommission_cancelers(&[0], &parent, cancelers.as_mut_slice());
30304
30305 assert_eq!(bound.len(), 1);
30306 assert_eq!(bound[0].0, 0);
30307 assert!(existing.is_cancelled());
30308 let replacement = cancelers[0].as_ref().expect("replacement token should be stored");
30309 assert!(!replacement.is_cancelled());
30310 parent.cancel();
30311 assert!(replacement.is_cancelled());
30312 }
30313
30314 #[test]
30315 fn test_bind_missing_decommission_cancelers_stops_at_existing_slot() {
30316 let parent = CancellationToken::new();
30317 let existing = DecommissionCanceler::new(CancellationToken::new());
30318 let mut cancelers = vec![None, Some(existing.clone()), None];
30319
30320 let bound = bind_missing_decommission_cancelers(&[0, 1, 2], &parent, cancelers.as_mut_slice());
30321
30322 assert_eq!(bound.len(), 1);
30323 assert_eq!(bound[0].0, 0);
30324 assert!(cancelers[0].is_some());
30325 assert!(cancelers[1].is_some());
30326 assert!(cancelers[2].is_none());
30327 assert!(!existing.is_cancelled());
30328 }
30329
30330 #[test]
30331 fn test_serialized_decommission_double_start_preserves_first_operation() {
30332 let mut pool_meta = PoolMeta {
30333 pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
30334 ..Default::default()
30335 };
30336 let first_parent = CancellationToken::new();
30337 let second_parent = CancellationToken::new();
30338 let mut cancelers = vec![None, None];
30339
30340 let first = reserve_decommission_start_cancelers(&pool_meta, &[0], &[0], &first_parent, cancelers.as_mut_slice())
30341 .expect("first start should reserve its worker");
30342 pool_meta
30343 .decommission(
30344 0,
30345 PoolSpaceInfo {
30346 total: 100,
30347 free: 40,
30348 used: 60,
30349 },
30350 )
30351 .expect("first start should install active metadata");
30352
30353 let second = reserve_decommission_start_cancelers(&pool_meta, &[0], &[0], &second_parent, cancelers.as_mut_slice());
30354
30355 assert!(matches!(second, Err(Error::DecommissionAlreadyRunning)));
30356 let current = cancelers[0].as_ref().expect("first operation should retain the slot");
30357 assert!(current.owns_same_operation(first[0].1.canceler()));
30358 assert!(current.is_active());
30359 assert!(!first_parent.is_cancelled());
30360 }
30361
30362 #[test]
30363 fn test_local_decommission_queue_prefix_stops_at_remote_leader() {
30364 let endpoints = EndpointServerPools::from(vec![
30365 decommission_test_pool_endpoint(0, true),
30366 decommission_test_pool_endpoint(1, true),
30367 decommission_test_pool_endpoint(2, false),
30368 decommission_test_pool_endpoint(3, true),
30369 ]);
30370
30371 let local = local_decommission_queue_prefix(&endpoints, &[0, 1, 2, 3]).expect("prefix should resolve");
30372
30373 assert_eq!(local, vec![0, 1]);
30374 }
30375
30376 #[test]
30377 fn test_local_decommission_queue_prefix_empty_when_first_leader_remote() {
30378 let endpoints = EndpointServerPools::from(vec![
30379 decommission_test_pool_endpoint(0, false),
30380 decommission_test_pool_endpoint(1, true),
30381 ]);
30382
30383 let local = local_decommission_queue_prefix(&endpoints, &[0, 1]).expect("prefix should resolve");
30384
30385 assert!(local.is_empty());
30386 }
30387
30388 #[test]
30389 fn test_decommission_start_local_leader_allows_remote_queued_pool() {
30390 let endpoints = EndpointServerPools::from(vec![
30391 decommission_test_pool_endpoint(0, true),
30392 decommission_test_pool_endpoint(1, false),
30393 ]);
30394
30395 assert!(ensure_decommission_start_local_leader(&endpoints, &[0, 1]).is_ok());
30396 }
30397
30398 #[test]
30399 fn test_decommission_start_local_leader_rejects_remote_active_pool() {
30400 let endpoints = EndpointServerPools::from(vec![decommission_test_pool_endpoint(0, false)]);
30401
30402 let err = ensure_decommission_start_local_leader(&endpoints, &[0]).expect_err("remote active pool should be rejected");
30403
30404 assert!(
30405 err.to_string()
30406 .contains("decommission for pool 0 must run on the pool first endpoint")
30407 );
30408 }
30409
30410 #[test]
30411 fn test_missing_decommission_worker_prefix_stops_at_active_worker() {
30412 let cancelers = vec![None, Some(DecommissionCanceler::new(CancellationToken::new())), None];
30413
30414 let missing = missing_decommission_worker_prefix(&[0, 1, 2], cancelers.as_slice());
30415
30416 assert_eq!(missing, vec![0]);
30417 }
30418
30419 #[test]
30420 fn test_resumable_decommission_queue_indices_skip_terminal_predecessors() {
30421 let meta = PoolMeta {
30422 pools: vec![
30423 decommission_test_pool_status(
30424 0,
30425 Some(PoolDecommissionInfo {
30426 complete: true,
30427 ..Default::default()
30428 }),
30429 ),
30430 decommission_test_pool_status(
30431 1,
30432 Some(PoolDecommissionInfo {
30433 canceled: true,
30434 ..Default::default()
30435 }),
30436 ),
30437 decommission_test_pool_status(
30438 2,
30439 Some(PoolDecommissionInfo {
30440 failed: true,
30441 ..Default::default()
30442 }),
30443 ),
30444 decommission_test_pool_status(
30445 3,
30446 Some(PoolDecommissionInfo {
30447 queued: true,
30448 ..Default::default()
30449 }),
30450 ),
30451 decommission_test_pool_status(4, None),
30452 ],
30453 ..Default::default()
30454 };
30455
30456 assert_eq!(resumable_decommission_queue_indices(&meta), vec![3]);
30457 }
30458
30459 #[test]
30460 fn test_resumable_decommission_queue_indices_preserve_active_predecessor_order() {
30461 let meta = PoolMeta {
30462 pools: vec![
30463 decommission_test_pool_status(
30464 0,
30465 Some(PoolDecommissionInfo {
30466 complete: true,
30467 ..Default::default()
30468 }),
30469 ),
30470 decommission_test_pool_status(
30471 1,
30472 Some(PoolDecommissionInfo {
30473 start_time: Some(OffsetDateTime::UNIX_EPOCH),
30474 ..Default::default()
30475 }),
30476 ),
30477 decommission_test_pool_status(
30478 2,
30479 Some(PoolDecommissionInfo {
30480 queued: true,
30481 ..Default::default()
30482 }),
30483 ),
30484 ],
30485 ..Default::default()
30486 };
30487
30488 assert_eq!(resumable_decommission_queue_indices(&meta), vec![1, 2]);
30489 }
30490
30491 #[tokio::test]
30492 async fn test_runtime_recovery_reserves_the_startup_resumable_queue() {
30493 let meta = PoolMeta {
30494 version: super::POOL_META_VERSION,
30495 pools: vec![
30496 decommission_test_pool_status(
30497 0,
30498 Some(PoolDecommissionInfo {
30499 failed: true,
30500 ..Default::default()
30501 }),
30502 ),
30503 decommission_test_pool_status(
30504 1,
30505 Some(PoolDecommissionInfo {
30506 canceled: true,
30507 ..Default::default()
30508 }),
30509 ),
30510 decommission_test_pool_status(
30511 2,
30512 Some(PoolDecommissionInfo {
30513 start_time: Some(OffsetDateTime::UNIX_EPOCH),
30514 ..Default::default()
30515 }),
30516 ),
30517 decommission_test_pool_status(
30518 3,
30519 Some(PoolDecommissionInfo {
30520 queued: true,
30521 ..Default::default()
30522 }),
30523 ),
30524 ],
30525 ..Default::default()
30526 };
30527 let startup_ids = meta
30528 .return_resumable_pools()
30529 .into_iter()
30530 .map(|pool| pool.id)
30531 .collect::<Vec<_>>();
30532 let store = decommission_worker_test_store(meta, vec![None, None, None, None]);
30533 let endpoints = EndpointServerPools::from(vec![
30534 decommission_test_pool_endpoint(0, true),
30535 decommission_test_pool_endpoint(1, true),
30536 decommission_test_pool_endpoint(2, true),
30537 decommission_test_pool_endpoint(3, true),
30538 ]);
30539 let reserved = store
30540 .reserve_missing_local_decommission_routines(&CancellationToken::new(), &endpoints)
30541 .await
30542 .expect("runtime recovery reservation should succeed");
30543 let runtime_indices = reserved.iter().map(|(idx, _)| *idx).collect::<Vec<_>>();
30544
30545 assert_eq!(runtime_indices, vec![2, 3]);
30546 assert_eq!(startup_ids, vec![2, 3]);
30547 }
30548
30549 #[tokio::test]
30550 async fn test_runtime_recovery_does_not_reserve_behind_active_predecessor() {
30551 let meta = PoolMeta {
30552 version: super::POOL_META_VERSION,
30553 pools: vec![
30554 decommission_test_pool_status(
30555 0,
30556 Some(PoolDecommissionInfo {
30557 failed: true,
30558 ..Default::default()
30559 }),
30560 ),
30561 decommission_test_pool_status(
30562 1,
30563 Some(PoolDecommissionInfo {
30564 canceled: true,
30565 ..Default::default()
30566 }),
30567 ),
30568 decommission_test_pool_status(
30569 2,
30570 Some(PoolDecommissionInfo {
30571 start_time: Some(OffsetDateTime::UNIX_EPOCH),
30572 ..Default::default()
30573 }),
30574 ),
30575 decommission_test_pool_status(
30576 3,
30577 Some(PoolDecommissionInfo {
30578 queued: true,
30579 ..Default::default()
30580 }),
30581 ),
30582 ],
30583 ..Default::default()
30584 };
30585 let active = DecommissionCanceler::new(CancellationToken::new());
30586 let store = decommission_worker_test_store(meta, vec![None, None, Some(active.clone()), None]);
30587 let endpoints = EndpointServerPools::from(vec![
30588 decommission_test_pool_endpoint(0, true),
30589 decommission_test_pool_endpoint(1, true),
30590 decommission_test_pool_endpoint(2, true),
30591 decommission_test_pool_endpoint(3, true),
30592 ]);
30593
30594 let reserved = store
30595 .reserve_missing_local_decommission_routines(&CancellationToken::new(), &endpoints)
30596 .await
30597 .expect("runtime recovery reservation should succeed");
30598
30599 assert!(reserved.is_empty());
30600 assert!(active.is_active());
30601 }
30602
30603 #[test]
30604 fn test_take_decommission_canceler_takes_and_clears_slot() {
30605 let canceler = DecommissionCanceler::new(CancellationToken::new());
30606 let mut cancelers = vec![Some(canceler)];
30607
30608 let taken = take_decommission_canceler(cancelers.as_mut_slice(), 0);
30609 assert!(taken.is_some());
30610 assert!(cancelers[0].is_none());
30611 }
30612
30613 #[test]
30614 fn test_take_decommission_canceler_returns_none_for_missing_slot() {
30615 let mut cancelers: Vec<Option<DecommissionCanceler>> = Vec::new();
30616 assert!(take_decommission_canceler(cancelers.as_mut_slice(), 0).is_none());
30617 }
30618
30619 #[test]
30620 fn test_has_active_decommission_canceler_true_when_any_slot_present() {
30621 let cancelers = vec![None, Some(DecommissionCanceler::new(CancellationToken::new()))];
30622 assert!(has_active_decommission_canceler(cancelers.as_slice()));
30623 }
30624
30625 #[test]
30626 fn test_has_active_decommission_canceler_false_when_all_empty() {
30627 let cancelers = vec![None, None];
30628 assert!(!has_active_decommission_canceler(cancelers.as_slice()));
30629 }
30630
30631 #[test]
30632 fn test_cancel_decommission_canceler_cancels_when_present() {
30633 let canceler = DecommissionCanceler::new(CancellationToken::new());
30634 let canceled = cancel_decommission_canceler(Some(canceler.clone()));
30635
30636 assert!(canceled);
30637 assert!(canceler.is_cancelled());
30638 assert!(!canceler.is_active());
30639 }
30640
30641 #[test]
30642 fn test_cancel_decommission_canceler_returns_false_when_missing() {
30643 assert!(!cancel_decommission_canceler(None));
30644 }
30645
30646 #[test]
30647 fn test_take_and_cancel_decommission_canceler_clears_slot() {
30648 let canceler = DecommissionCanceler::new(CancellationToken::new());
30649 let mut cancelers = vec![Some(canceler.clone())];
30650
30651 assert!(take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), 0));
30652 assert!(cancelers[0].is_none());
30653 assert!(canceler.is_cancelled());
30654 assert!(!canceler.is_active());
30655 }
30656
30657 #[test]
30658 fn test_take_and_cancel_decommission_canceler_missing_slot_is_false() {
30659 let mut cancelers = vec![None];
30660
30661 assert!(!take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), 0));
30662 assert!(cancelers[0].is_none());
30663 }
30664
30665 #[test]
30666 fn test_guarded_decommission_future_releases_without_first_poll() {
30667 let canceler = DecommissionCanceler::new(CancellationToken::new());
30668 let cancelers = vec![Some(canceler.clone())];
30669 let guards = guard_decommission_cancelers(vec![(0, canceler.clone())]);
30670 let unpolled = async move {
30671 let _guards = guards;
30672 std::future::pending::<()>().await;
30673 };
30674
30675 drop(unpolled);
30676
30677 assert!(canceler.is_cancelled());
30678 assert!(!has_active_decommission_canceler(cancelers.as_slice()));
30679 }
30680
30681 #[test]
30682 fn test_partial_decommission_spawn_reservation_releases_bound_slot() {
30683 let parent = CancellationToken::new();
30684 let mut cancelers = vec![None];
30685 let bound = bind_decommission_cancelers(&[0, 1], &parent, cancelers.as_mut_slice());
30686 let guards = guard_decommission_cancelers(bound);
30687
30688 let result = super::ensure_decommission_routines_scheduled(guards.len(), 2);
30689 drop(guards);
30690
30691 assert!(result.is_err());
30692 assert!(!has_active_decommission_canceler(cancelers.as_slice()));
30693 }
30694
30695 #[tokio::test]
30696 async fn test_decommission_supervisor_observes_worker_abort() {
30697 let (started_tx, started_rx) = tokio::sync::oneshot::channel();
30698 let worker = tokio::spawn(async move {
30699 started_tx.send(()).expect("worker start should be observed");
30700 std::future::pending::<()>().await;
30701 #[allow(unreachable_code)]
30702 Ok(())
30703 });
30704
30705 started_rx.await.expect("worker start should be observed");
30706 worker.abort();
30707 let err = await_decommission_worker(3, worker)
30708 .await
30709 .expect_err("supervisor should observe aborted worker");
30710
30711 assert!(err.to_string().contains("decommission worker 3 task join error"));
30712 }
30713
30714 #[tokio::test]
30715 async fn test_decommission_supervisor_observes_worker_panic() {
30716 let worker = tokio::spawn(async move {
30717 panic!("injected decommission worker panic");
30718 });
30719
30720 let err = await_decommission_worker(4, worker)
30721 .await
30722 .expect_err("supervisor should observe panicked worker");
30723
30724 assert!(err.to_string().contains("decommission worker 4 task join error"));
30725 }
30726
30727 #[tokio::test]
30728 async fn test_decommission_worker_metadata_missing_releases_owned_slot() {
30729 let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
30730 let canceler = DecommissionCanceler::new(CancellationToken::new());
30731 *store.pool_meta.write().await = PoolMeta::default();
30732 store.decommission_cancelers.write().await[0] = Some(canceler.clone());
30733
30734 let err = store
30735 .do_decommission_in_routine(canceler.clone(), 0, Arc::new(Semaphore::new(1)))
30736 .await
30737 .expect_err("missing worker metadata should fail the routine");
30738
30739 assert!(err.to_string().contains("target pool was not found"));
30740 assert!(!canceler.is_active());
30741 assert!(store.decommission_cancelers.read().await[0].is_none());
30742 }
30743
30744 #[tokio::test]
30745 async fn test_decommission_supervisor_failure_cancels_queued_successor() {
30746 let first = DecommissionCanceler::new(CancellationToken::new());
30747 let queued = DecommissionCanceler::new(CancellationToken::new());
30748 let store = decommission_worker_test_store(PoolMeta::default(), vec![Some(first.clone()), Some(queued.clone())]);
30749 let guards = guard_decommission_cancelers(vec![(0, first.clone()), (1, queued.clone())]);
30750
30751 spawn_decommission_index_cancelers(store.clone(), CancellationToken::new(), guards, Arc::new(Semaphore::new(1)))
30752 .await
30753 .expect("decommission supervisor should finish after queued cleanup");
30754
30755 assert!(!first.is_active());
30756 assert!(!queued.is_active());
30757 assert!(queued.is_cancelled());
30758 assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
30759 }
30760
30761 #[tokio::test]
30762 async fn test_decommission_supervisor_releases_slot_without_terminal_retry_when_pool_meta_is_blocked() {
30763 let generation = OffsetDateTime::UNIX_EPOCH;
30764 let canceler = DecommissionCanceler::new(CancellationToken::new());
30765 let pool_meta = PoolMeta {
30766 version: super::POOL_META_VERSION,
30767 pools: vec![decommission_test_pool_status(
30768 0,
30769 Some(PoolDecommissionInfo {
30770 start_time: Some(generation),
30771 ..Default::default()
30772 }),
30773 )],
30774 ..Default::default()
30775 };
30776 let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
30777 store
30778 .pool_meta_save_gate
30779 .lock()
30780 .await
30781 .observe_replicas(super::PoolMetaReplicaState {
30782 needs_repair: true,
30783 repair_write_safe: false,
30784 });
30785 let guards = guard_decommission_cancelers(vec![(0, canceler.clone())]);
30786
30787 tokio::time::timeout(
30788 StdDuration::from_secs(1),
30789 spawn_decommission_index_cancelers(store.clone(), CancellationToken::new(), guards, Arc::new(Semaphore::new(1))),
30790 )
30791 .await
30792 .expect("blocked supervisor must not enter terminal retry")
30793 .expect("blocked supervisor task should not panic");
30794
30795 assert!(store.decommission_cancelers.read().await[0].is_none());
30796 assert!(!canceler.is_active());
30797 let pool_meta = store.pool_meta.read().await;
30798 let info = pool_meta.pools[0]
30799 .decommission
30800 .as_ref()
30801 .expect("blocked decommission metadata should remain active");
30802 assert!(!info.failed);
30803 assert!(!info.canceled);
30804 assert!(!info.complete);
30805 assert_eq!(info.start_time, Some(generation));
30806 }
30807
30808 #[tokio::test]
30809 async fn test_decommission_promotion_rechecks_sticky_gate_after_start_wait() {
30810 let pool_meta = PoolMeta {
30811 pools: vec![decommission_test_pool_status(
30812 0,
30813 Some(PoolDecommissionInfo {
30814 queued: true,
30815 ..Default::default()
30816 }),
30817 )],
30818 ..Default::default()
30819 };
30820 let store = decommission_worker_test_store(pool_meta, vec![None]);
30821 let start_guard = store.start_gate.lock().await;
30822 let mut promotion = tokio::spawn({
30823 let store = store.clone();
30824 async move { store.promote_queued_decommission_for_test(0).await }
30825 });
30826 tokio::task::yield_now().await;
30827 assert!(!promotion.is_finished(), "promotion should be waiting for the start gate");
30828
30829 store
30830 .pool_meta_save_gate
30831 .lock()
30832 .await
30833 .observe_replicas(super::PoolMetaReplicaState {
30834 needs_repair: true,
30835 repair_write_safe: false,
30836 });
30837 drop(start_guard);
30838
30839 let err = tokio::time::timeout(StdDuration::from_secs(1), &mut promotion)
30840 .await
30841 .expect("blocked promotion should finish")
30842 .expect("promotion task should not panic")
30843 .expect_err("promotion must recheck the sticky gate after waiting");
30844 assert!(
30845 err.to_string()
30846 .contains("restart after all replicas are readable and consistent")
30847 );
30848
30849 let pool_meta = store.pool_meta.read().await;
30850 let info = pool_meta.pools[0]
30851 .decommission
30852 .as_ref()
30853 .expect("queued metadata should remain present");
30854 assert!(info.queued);
30855 assert!(info.start_time.is_none());
30856 }
30857
30858 #[tokio::test]
30859 #[serial_test::serial(pool_meta_version_env)]
30860 async fn test_decommission_promotion_persists_all_recovered_capacity_sources() {
30861 let (_temp_dirs, store, _other_store) =
30862 crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
30863 persist_v3_pool_meta_for_test(&store).await;
30864 let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(3);
30865 let layout = DecommissionErasureLayout { data: 1, parity: 0 };
30866 let mut queued_reservation = build_decommission_capacity_reservation(
30867 DecommissionPoolCapacityInfo::for_test(1, layout, 90, 100, 10),
30868 layout,
30869 uuid::Uuid::new_v4(),
30870 7,
30871 now,
30872 )
30873 .expect("queued source reservation should be valid");
30874 queued_reservation.targets.push(DecommissionCapacityTarget {
30875 pool_index: 2,
30876 layout,
30877 physical_total_at_reservation: 200,
30878 physical_free_at_reservation: 100,
30879 reserved_physical_bytes: queued_reservation.peak_physical_bytes,
30880 consumed_physical_bytes: 0,
30881 observed_physical_bytes: 0,
30882 inflight_physical_bytes: 0,
30883 pending_physical_bytes: 0,
30884 pending_mutation_id: None,
30885 temporary_mutations: Vec::new(),
30886 });
30887 {
30888 let mut pool_meta = store.pool_meta.write().await;
30889 pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
30890 start_time: Some(now),
30891 ..Default::default()
30892 });
30893 pool_meta.pools[1].decommission = Some(PoolDecommissionInfo {
30894 queued: true,
30895 start_time: Some(now),
30896 capacity_reservation: Some(queued_reservation),
30897 ..Default::default()
30898 });
30899 }
30900 let mut loaded_v3 = PoolMeta::default();
30901 loaded_v3
30902 .load_no_lock_from_replicas(store.pools.clone())
30903 .await
30904 .expect("the startup fixture must provide a durable V3 pool metadata replica");
30905 assert_eq!(loaded_v3.version, POOL_META_GENERATION_VERSION);
30906 store
30907 .save_current_pool_meta_for_test(&[0, 1, 2])
30908 .await
30909 .expect("active and queued source metadata should be persisted before promotion");
30910 set_decommission_capacity_info_overrides_for_test(
30911 store.id,
30912 vec![vec![
30913 DecommissionPoolCapacityInfo::for_test(0, layout, 90, 100, 10),
30914 DecommissionPoolCapacityInfo::for_test(1, layout, 90, 100, 10),
30915 DecommissionPoolCapacityInfo::for_test(2, layout, 100, 200, 100),
30916 ]],
30917 );
30918
30919 store
30920 .promote_queued_decommission_for_test(1)
30921 .await
30922 .expect("queued promotion should recover all active source reservations");
30923
30924 let mut persisted = PoolMeta::default();
30925 persisted
30926 .load_no_lock_from_replicas(store.pools.clone())
30927 .await
30928 .expect("promoted active and recovered source metadata should reload");
30929 assert_eq!(persisted.version, POOL_META_GENERATION_VERSION);
30930 assert!(
30931 persisted.pools[0]
30932 .decommission
30933 .as_ref()
30934 .and_then(|info| info.capacity_reservation.as_ref())
30935 .is_some_and(DecommissionCapacityReservation::active)
30936 );
30937 let promoted = persisted.pools[1]
30938 .decommission
30939 .as_ref()
30940 .expect("queued source should remain after promotion");
30941 assert!(!promoted.queued, "queued source should be promoted durably");
30942 assert!(
30943 promoted
30944 .capacity_reservation
30945 .as_ref()
30946 .is_some_and(DecommissionCapacityReservation::active)
30947 );
30948 }
30949
30950 #[tokio::test]
30951 async fn test_decommission_bucket_done_rechecks_sticky_gate_before_mutation() {
30952 let bucket = DecomBucketInfo {
30953 name: "bucket-a".to_string(),
30954 prefix: String::new(),
30955 };
30956 let pool_meta = PoolMeta {
30957 pools: vec![decommission_test_pool_status(
30958 0,
30959 Some(PoolDecommissionInfo {
30960 queued_buckets: vec![bucket.to_string()],
30961 ..Default::default()
30962 }),
30963 )],
30964 ..Default::default()
30965 };
30966 let store = decommission_worker_test_store(pool_meta, vec![None]);
30967 store
30968 .pool_meta_save_gate
30969 .lock()
30970 .await
30971 .observe_replicas(super::PoolMetaReplicaState {
30972 needs_repair: true,
30973 repair_write_safe: false,
30974 });
30975
30976 let err = store
30977 .mark_decommission_bucket_done_and_save(0, OffsetDateTime::UNIX_EPOCH, &bucket)
30978 .await
30979 .expect_err("bucket completion must stop before mutating sticky pool metadata");
30980 assert!(
30981 err.to_string()
30982 .contains("restart after all replicas are readable and consistent")
30983 );
30984
30985 let pool_meta = store.pool_meta.read().await;
30986 let info = pool_meta.pools[0]
30987 .decommission
30988 .as_ref()
30989 .expect("decommission metadata should remain present");
30990 assert_eq!(info.queued_buckets, vec![bucket.to_string()]);
30991 assert!(info.decommissioned_buckets.is_empty());
30992 }
30993
30994 #[tokio::test]
30995 async fn test_decommission_cancel_save_failure_preserves_generation_and_blocks_retry() {
30996 let generation = OffsetDateTime::UNIX_EPOCH;
30997 let canceler = DecommissionCanceler::new(CancellationToken::new());
30998 let pool_meta = PoolMeta {
30999 version: super::POOL_META_VERSION,
31000 pools: vec![decommission_test_pool_status(
31001 0,
31002 Some(PoolDecommissionInfo {
31003 start_time: Some(generation),
31004 ..Default::default()
31005 }),
31006 )],
31007 ..Default::default()
31008 };
31009 let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
31010
31011 let err = store
31012 .decommission_cancel_with_owner_and_save(0, Some(&canceler), |_, _| async { Err(Error::Timeout) })
31013 .await
31014 .expect_err("injected pool metadata timeout should fail cancel");
31015 assert!(matches!(err, Error::Timeout));
31016
31017 {
31018 let pool_meta = store.pool_meta.read().await;
31019 let info = pool_meta.pools[0]
31020 .decommission
31021 .as_ref()
31022 .expect("failed cancel must retain decommission metadata");
31023 assert_eq!(info.start_time, Some(generation));
31024 assert!(!info.canceled);
31025 assert!(!info.complete);
31026 assert!(!info.failed);
31027 assert!(ensure_decommission_generation(&pool_meta, 0, generation).is_ok());
31028 }
31029 {
31030 let cancelers = store.decommission_cancelers.read().await;
31031 let current = cancelers[0].as_ref().expect("failed cancel must retain the worker owner");
31032 assert!(current.owns_same_operation(&canceler));
31033 assert!(current.is_active());
31034 }
31035 assert!(!canceler.is_cancelled());
31036 assert!(store.decommission_terminal_retryable_for_operation(0, &canceler).await);
31037
31038 let retry_save_called = Arc::new(AtomicBool::new(false));
31039 let retry_save_called_by_closure = retry_save_called.clone();
31040 let retry_err = store
31041 .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |_, _| async move {
31042 retry_save_called_by_closure.store(true, Ordering::SeqCst);
31043 Ok(())
31044 })
31045 .await
31046 .expect_err("an ambiguous save failure must block same-process retry");
31047 assert!(
31048 retry_err
31049 .to_string()
31050 .contains("restart after all replicas are readable and consistent")
31051 );
31052 assert!(!retry_save_called.load(Ordering::SeqCst));
31053
31054 {
31055 let pool_meta = store.pool_meta.read().await;
31056 let info = pool_meta.pools[0]
31057 .decommission
31058 .as_ref()
31059 .expect("blocked retry must retain decommission metadata");
31060 assert_eq!(info.start_time, Some(generation));
31061 assert!(!info.canceled);
31062 assert!(!info.complete);
31063 assert!(!info.failed);
31064 }
31065 assert!(store.decommission_cancelers.read().await[0].is_some());
31066 assert!(canceler.is_active());
31067 assert!(!canceler.is_cancelled());
31068 }
31069
31070 #[tokio::test]
31071 async fn test_decommission_cancel_stays_blocked_after_unreadable_pool_meta_replica() {
31072 let generation = OffsetDateTime::UNIX_EPOCH;
31073 let canceler = DecommissionCanceler::new(CancellationToken::new());
31074 let pool_meta = PoolMeta {
31075 version: super::POOL_META_VERSION,
31076 pools: vec![decommission_test_pool_status(
31077 0,
31078 Some(PoolDecommissionInfo {
31079 start_time: Some(generation),
31080 ..Default::default()
31081 }),
31082 )],
31083 ..Default::default()
31084 };
31085 let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
31086 store
31087 .pool_meta_save_gate
31088 .lock()
31089 .await
31090 .observe_replicas(super::PoolMetaReplicaState {
31091 needs_repair: true,
31092 repair_write_safe: false,
31093 });
31094 let save_called = Arc::new(AtomicBool::new(false));
31095 let save_called_by_closure = save_called.clone();
31096
31097 let err = store
31098 .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |_, _| async move {
31099 save_called_by_closure.store(true, Ordering::SeqCst);
31100 Ok(())
31101 })
31102 .await
31103 .expect_err("cancel must remain blocked until restart after an unreadable replica");
31104
31105 assert!(
31106 err.to_string()
31107 .contains("restart after all replicas are readable and consistent")
31108 );
31109 assert!(!save_called.load(Ordering::SeqCst));
31110 let pool_meta = store.pool_meta.read().await;
31111 let info = pool_meta.pools[0]
31112 .decommission
31113 .as_ref()
31114 .expect("blocked cancel must preserve decommission metadata");
31115 assert_eq!(info.start_time, Some(generation));
31116 assert!(!info.canceled);
31117 drop(pool_meta);
31118 assert!(canceler.is_active());
31119 assert!(!canceler.is_cancelled());
31120 let cancelers = store.decommission_cancelers.read().await;
31121 assert!(
31122 cancelers[0]
31123 .as_ref()
31124 .is_some_and(|current| current.owns_same_operation(&canceler))
31125 );
31126 }
31127
31128 #[tokio::test]
31129 async fn test_decommission_cancel_serializes_reload_until_local_commit() {
31130 let generation = OffsetDateTime::UNIX_EPOCH;
31131 let canceler = DecommissionCanceler::new(CancellationToken::new());
31132 let pool_meta = PoolMeta {
31133 version: super::POOL_META_VERSION,
31134 pools: vec![decommission_test_pool_status(
31135 0,
31136 Some(PoolDecommissionInfo {
31137 start_time: Some(generation),
31138 stage: "migrate_object".to_string(),
31139 ..Default::default()
31140 }),
31141 )],
31142 ..Default::default()
31143 };
31144 let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
31145 let (persisted_tx, persisted_rx) = tokio::sync::oneshot::channel();
31146 let save_release = Arc::new(tokio::sync::Notify::new());
31147
31148 let cancel = tokio::spawn({
31149 let store = store.clone();
31150 let canceler = canceler.clone();
31151 let save_release = save_release.clone();
31152 async move {
31153 store
31154 .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move {
31155 persisted_tx
31156 .send(snapshot.encode_config_data()?)
31157 .map_err(|_| Error::other("failed to expose saved cancel snapshot"))?;
31158 save_release.notified().await;
31159 Ok(())
31160 })
31161 .await
31162 }
31163 });
31164 let persisted = persisted_rx.await.expect("save should expose the canceled snapshot");
31165 let reload_started = Arc::new(tokio::sync::Notify::new());
31166 let reload = tokio::spawn({
31167 let store = store.clone();
31168 let reload_started = reload_started.clone();
31169 async move {
31170 let mut reloaded = PoolMeta::default();
31171 reloaded.load_from_config_data(persisted)?;
31172 reload_started.notify_one();
31173 *store.pool_meta.write().await = reloaded;
31174 Ok::<(), Error>(())
31175 }
31176 });
31177 reload_started.notified().await;
31178 assert!(!reload.is_finished(), "peer reload must wait for cancel publication");
31179
31180 save_release.notify_one();
31181 cancel
31182 .await
31183 .expect("cancel task should not panic")
31184 .expect("cancel should commit before releasing the reload");
31185 reload
31186 .await
31187 .expect("reload task should not panic")
31188 .expect("reload should install the saved cancel after publication");
31189
31190 let pool_meta = store.pool_meta.read().await;
31191 let info = pool_meta.pools[0]
31192 .decommission
31193 .as_ref()
31194 .expect("reloaded cancel metadata should remain present");
31195 assert!(info.canceled);
31196 assert!(!info.complete);
31197 assert!(!info.failed);
31198 assert!(info.start_time.is_none());
31199 assert!(info.stage.is_empty(), "the persisted snapshot should have been decoded before commit");
31200 drop(pool_meta);
31201 assert!(store.decommission_cancelers.read().await[0].is_none());
31202 assert!(!canceler.is_active());
31203 assert!(canceler.is_cancelled());
31204 }
31205
31206 #[tokio::test]
31207 async fn test_decommission_cancel_transaction_survives_caller_abort_after_durable_save() {
31208 let generation = OffsetDateTime::UNIX_EPOCH;
31209 let canceler = DecommissionCanceler::new(CancellationToken::new());
31210 let pool_meta = PoolMeta {
31211 version: super::POOL_META_VERSION,
31212 pools: vec![decommission_test_pool_status(
31213 0,
31214 Some(PoolDecommissionInfo {
31215 start_time: Some(generation),
31216 ..Default::default()
31217 }),
31218 )],
31219 ..Default::default()
31220 };
31221 let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
31222 let persisted = Arc::new(std::sync::Mutex::new(None));
31223 let (durable_tx, durable_rx) = tokio::sync::oneshot::channel();
31224 let save_release = Arc::new(tokio::sync::Notify::new());
31225 let cancel = tokio::spawn({
31226 let store = store.clone();
31227 let canceler = canceler.clone();
31228 let persisted = persisted.clone();
31229 let save_release = save_release.clone();
31230 async move {
31231 store
31232 .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move {
31233 assert!(
31234 snapshot.pools[0]
31235 .decommission
31236 .as_ref()
31237 .is_some_and(|info| info.canceled && info.start_time.is_none())
31238 );
31239 *persisted.lock().expect("persisted cancel lock should not be poisoned") =
31240 Some(snapshot.encode_config_data()?);
31241 durable_tx
31242 .send(())
31243 .map_err(|_| Error::other("failed to report durable cancel snapshot"))?;
31244 save_release.notified().await;
31245 Ok(())
31246 })
31247 .await
31248 }
31249 });
31250 durable_rx.await.expect("save hook should report the durable cancel snapshot");
31251 cancel.abort();
31252 let join_err = cancel.await.expect_err("caller cancel future should be aborted");
31253 assert!(join_err.is_cancelled());
31254 assert!(
31255 store.pool_meta.try_read().is_err(),
31256 "the detached transaction must retain its state guard"
31257 );
31258 assert!(
31259 store.decommission_cancelers.try_read().is_err(),
31260 "the detached transaction must retain its owner guard"
31261 );
31262
31263 save_release.notify_one();
31264 tokio::time::timeout(StdDuration::from_secs(1), canceler.token().cancelled())
31265 .await
31266 .expect("detached cancel transaction should terminate the old token");
31267
31268 let persisted = persisted
31269 .lock()
31270 .expect("persisted cancel lock should not be poisoned")
31271 .take()
31272 .expect("save should capture the durable cancel snapshot");
31273 let mut durable = PoolMeta::default();
31274 durable
31275 .load_from_config_data(persisted)
31276 .expect("durable cancel snapshot should decode");
31277 assert!(
31278 durable.pools[0]
31279 .decommission
31280 .as_ref()
31281 .is_some_and(|info| info.canceled && info.start_time.is_none())
31282 );
31283 assert!(store.decommission_cancelers.read().await[0].is_none());
31284 assert!(!canceler.is_active());
31285 assert!(canceler.is_cancelled());
31286
31287 let side_effect_ran = Arc::new(AtomicBool::new(false));
31288 let operation_gate = store.ctx.data_movement_operation_gate();
31289 let result = run_decommission_side_effect(canceler.token(), &operation_gate, {
31290 let side_effect_ran = side_effect_ran.clone();
31291 move || async move {
31292 side_effect_ran.store(true, Ordering::SeqCst);
31293 Ok(())
31294 }
31295 })
31296 .await;
31297 assert!(matches!(result, Err(Error::OperationCanceled)));
31298 assert!(!side_effect_ran.load(Ordering::SeqCst));
31299 assert!(
31300 store.pool_meta.read().await.pools[0]
31301 .decommission
31302 .as_ref()
31303 .is_some_and(|info| info.canceled && !info.failed)
31304 );
31305 }
31306
31307 #[tokio::test]
31308 async fn test_decommission_cancel_persists_before_signaling_and_quiesces_before_return() {
31309 let generation = OffsetDateTime::UNIX_EPOCH;
31310 let canceler = DecommissionCanceler::new(CancellationToken::new());
31311 let pool_meta = PoolMeta {
31312 pools: vec![decommission_test_pool_status(
31313 0,
31314 Some(PoolDecommissionInfo {
31315 start_time: Some(generation),
31316 ..Default::default()
31317 }),
31318 )],
31319 ..Default::default()
31320 };
31321 let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
31322 let operation_gate = store.ctx.data_movement_operation_gate();
31323 let side_effect = operation_gate.read().await;
31324 let save_started = Arc::new(tokio::sync::Notify::new());
31325 let save_release = Arc::new(tokio::sync::Notify::new());
31326 let save_entered = Arc::new(AtomicBool::new(false));
31327 let persisted = Arc::new(std::sync::Mutex::new(None));
31328
31329 let mut cancel = tokio::spawn({
31330 let store = store.clone();
31331 let canceler = canceler.clone();
31332 let save_started = save_started.clone();
31333 let save_release = save_release.clone();
31334 let save_entered = save_entered.clone();
31335 let persisted = persisted.clone();
31336 async move {
31337 store
31338 .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move {
31339 *persisted.lock().expect("persisted cancel lock should not be poisoned") =
31340 Some(snapshot.encode_config_data()?);
31341 save_entered.store(true, Ordering::SeqCst);
31342 save_started.notify_one();
31343 save_release.notified().await;
31344 Ok(())
31345 })
31346 .await
31347 }
31348 });
31349
31350 tokio::time::timeout(StdDuration::from_secs(1), save_started.notified())
31351 .await
31352 .expect("cancel should persist while the side effect is still in flight");
31353 assert!(save_entered.load(Ordering::SeqCst));
31354 assert!(!cancel.is_finished(), "cancel must wait for the injected save");
31355 assert!(
31356 store.pool_meta_save_gate.try_lock().is_err(),
31357 "the cancel save must exclude stale full-document saves until publication"
31358 );
31359 assert!(
31360 store.pool_meta.try_read().is_err(),
31361 "the active generation must stay write-locked through persistence"
31362 );
31363 assert!(!canceler.is_cancelled(), "the token must remain live until persistence commits");
31364
31365 let mut fail = tokio::spawn({
31366 let store = store.clone();
31367 async move { store.decommission_failed(0).await }
31368 });
31369 tokio::task::yield_now().await;
31370 assert!(!fail.is_finished(), "fail must serialize behind the pending cancel");
31371
31372 save_release.notify_one();
31373 tokio::time::timeout(StdDuration::from_secs(1), canceler.token().cancelled())
31374 .await
31375 .expect("the durable cancel must signal the active worker");
31376 assert!(!cancel.is_finished(), "cancel must wait for in-flight movement after the durable signal");
31377 let canceled_at = {
31378 let pool_meta = store.pool_meta.read().await;
31379 let info = pool_meta.pools[0]
31380 .decommission
31381 .as_ref()
31382 .expect("the durable cancel must be published before quiescence");
31383 assert!(info.canceled);
31384 assert!(!info.complete);
31385 assert!(!info.failed);
31386 assert!(info.start_time.is_none());
31387 pool_meta.pools[0].last_update
31388 };
31389 let expected_generation = crate::store::scanner_data_movement_timestamp_generation(canceled_at);
31390 let scanner_status = store.scanner_data_movement_pause_snapshot_for_test().await;
31391 assert_eq!(scanner_status.movement_generation, expected_generation);
31392
31393 drop(side_effect);
31394 tokio::time::timeout(StdDuration::from_secs(1), &mut cancel)
31395 .await
31396 .expect("cancel should finish after in-flight movement quiesces")
31397 .expect("cancel task should not panic")
31398 .expect("cancel should commit");
31399 tokio::time::timeout(StdDuration::from_secs(1), &mut fail)
31400 .await
31401 .expect("fail should finish after cancel commits")
31402 .expect("fail task should not panic")
31403 .expect("stale fail should be a no-op");
31404
31405 let pool_meta = store.pool_meta.read().await;
31406 let info = pool_meta.pools[0]
31407 .decommission
31408 .as_ref()
31409 .expect("cancel metadata should remain present");
31410 assert!(info.canceled);
31411 assert!(!info.complete);
31412 assert!(!info.failed);
31413 assert!(info.start_time.is_none());
31414 drop(pool_meta);
31415 assert!(canceler.is_cancelled());
31416 assert!(!canceler.is_active());
31417 assert!(store.decommission_cancelers.read().await[0].is_none());
31418 assert_eq!(store.scanner_data_movement_generation(), expected_generation);
31419
31420 let persisted = persisted
31421 .lock()
31422 .expect("persisted cancel lock should not be poisoned")
31423 .take()
31424 .expect("cancel should persist a durable snapshot");
31425 let mut durable = PoolMeta::default();
31426 durable
31427 .load_from_config_data(persisted)
31428 .expect("durable cancel snapshot should decode after restart");
31429 let restarted = decommission_worker_test_store(durable.clone(), Vec::new());
31430 let restarted_status = restarted.scanner_data_movement_pause_status().await;
31431 assert_eq!(restarted_status.movement_generation, expected_generation);
31432
31433 assert!(
31434 durable
31435 .clear_decommission_at_for_test(0, canceled_at, None)
31436 .expect("same-tick clear should succeed")
31437 );
31438 assert!(durable.pools[0].last_update > canceled_at);
31439 }
31440
31441 #[test]
31442 fn test_decommission_cancel_commit_rejects_a_replaced_generation() {
31443 let old_generation = OffsetDateTime::UNIX_EPOCH;
31444 let new_generation = old_generation + Duration::seconds(1);
31445 let mut canceled = PoolMeta {
31446 pools: vec![decommission_test_pool_status(
31447 0,
31448 Some(PoolDecommissionInfo {
31449 start_time: Some(old_generation),
31450 ..Default::default()
31451 }),
31452 )],
31453 ..Default::default()
31454 };
31455 let previous_last_update = canceled.pools[0].last_update;
31456 assert!(canceled.decommission_cancel(0));
31457 let canceled_pool = canceled.pools.remove(0);
31458 let commit = super::DecommissionCancelCommit {
31459 previous_start_time: Some(old_generation),
31460 previous_queued: false,
31461 previous_last_update,
31462 canceled_pool,
31463 };
31464 let mut current = PoolMeta {
31465 pools: vec![decommission_test_pool_status(
31466 0,
31467 Some(PoolDecommissionInfo {
31468 start_time: Some(new_generation),
31469 ..Default::default()
31470 }),
31471 )],
31472 ..Default::default()
31473 };
31474
31475 let err = super::commit_decommission_cancel(&mut current, 0, commit)
31476 .expect_err("an old cancel must not publish over a replacement generation");
31477 assert!(err.to_string().contains("operation generation changed"));
31478 let info = current.pools[0]
31479 .decommission
31480 .as_ref()
31481 .expect("replacement generation should remain present");
31482 assert_eq!(info.start_time, Some(new_generation));
31483 assert!(!info.canceled);
31484 }
31485
31486 #[tokio::test]
31487 async fn test_decommission_cancel_rejects_stale_retry_after_queued_replacement() {
31488 let old_generation = OffsetDateTime::UNIX_EPOCH;
31489 let canceler = DecommissionCanceler::new(CancellationToken::new());
31490 let pool_meta = PoolMeta {
31491 pools: vec![decommission_test_pool_status(
31492 0,
31493 Some(PoolDecommissionInfo {
31494 start_time: Some(old_generation),
31495 ..Default::default()
31496 }),
31497 )],
31498 ..Default::default()
31499 };
31500 let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
31501 let queued_replacement = Arc::new(std::sync::Mutex::new(None));
31502 let queued_replacement_for_save = queued_replacement.clone();
31503
31504 store
31505 .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move {
31506 let saved_cancel = snapshot.pools[0]
31507 .decommission
31508 .as_ref()
31509 .expect("saved snapshot should contain decommission metadata");
31510 assert!(saved_cancel.canceled);
31511 assert!(saved_cancel.start_time.is_none());
31512 let mut queued_replacement = snapshot.clone();
31513 let replacement = queued_replacement
31514 .pools
31515 .get_mut(0)
31516 .expect("cancel snapshot should contain the pool");
31517 replacement.last_update += Duration::seconds(1);
31518 let info = replacement
31519 .decommission
31520 .as_mut()
31521 .expect("cancel snapshot should contain decommission metadata");
31522 info.canceled = false;
31523 info.queued = true;
31524 *queued_replacement_for_save
31525 .lock()
31526 .expect("queued replacement lock should not be poisoned") = Some(queued_replacement);
31527 Ok(())
31528 })
31529 .await
31530 .expect("cancel should commit before a queued replacement is installed");
31531 assert!(store.decommission_cancelers.read().await[0].is_none());
31532 assert!(!canceler.is_active());
31533 assert!(canceler.is_cancelled());
31534
31535 let queued_replacement = queued_replacement
31536 .lock()
31537 .expect("queued replacement lock should not be poisoned")
31538 .take()
31539 .expect("save should prepare the queued replacement");
31540 *store.pool_meta.write().await = queued_replacement;
31541
31542 let replacement_revision = {
31543 let pool_meta = store.pool_meta.read().await;
31544 let info = pool_meta.pools[0]
31545 .decommission
31546 .as_ref()
31547 .expect("queued replacement should remain present");
31548 assert!(info.queued);
31549 assert!(!info.canceled);
31550 assert!(info.start_time.is_none());
31551 pool_meta.pools[0].last_update
31552 };
31553
31554 store.retry_decommission_cancel_for_operation(0, &canceler).await;
31555
31556 let pool_meta = store.pool_meta.read().await;
31557 let info = pool_meta.pools[0]
31558 .decommission
31559 .as_ref()
31560 .expect("stale retry must preserve the queued replacement");
31561 assert_eq!(pool_meta.pools[0].last_update, replacement_revision);
31562 assert!(info.queued);
31563 assert!(!info.canceled);
31564 assert!(info.start_time.is_none());
31565 }
31566
31567 #[tokio::test]
31568 async fn test_decommission_failed_save_failure_preserves_owner_until_retry_succeeds() {
31569 let canceler = DecommissionCanceler::new(CancellationToken::new());
31570 let pool_meta = PoolMeta {
31571 pools: vec![decommission_test_pool_status(
31572 0,
31573 Some(PoolDecommissionInfo {
31574 start_time: Some(OffsetDateTime::UNIX_EPOCH),
31575 ..Default::default()
31576 }),
31577 )],
31578 ..Default::default()
31579 };
31580 let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
31581
31582 store
31583 .decommission_failed_with_owner_and_save(0, Some(&canceler), async { Err(Error::SlowDown) })
31584 .await
31585 .expect_err("injected terminal save failure should be returned");
31586
31587 {
31588 let cancelers = store.decommission_cancelers.read().await;
31589 let current = cancelers[0].as_ref().expect("failed save must retain the exact owner slot");
31590 assert!(current.owns_same_operation(&canceler));
31591 assert!(current.is_active());
31592 }
31593 {
31594 let pool_meta = store.pool_meta.read().await;
31595 let info = pool_meta.pools[0]
31596 .decommission
31597 .as_ref()
31598 .expect("rollback must retain active decommission metadata");
31599 assert!(info.has_decommission_state());
31600 assert!(!info.failed);
31601 assert!(!info.complete);
31602 assert!(!info.canceled);
31603 }
31604 assert!(store.decommission_terminal_retryable_for_operation(0, &canceler).await);
31605
31606 store
31607 .decommission_failed_with_owner_and_save(0, Some(&canceler), async { Ok(()) })
31608 .await
31609 .expect("terminal retry should commit");
31610
31611 let pool_meta = store.pool_meta.read().await;
31612 assert!(
31613 pool_meta.pools[0]
31614 .decommission
31615 .as_ref()
31616 .expect("terminal metadata should remain")
31617 .failed
31618 );
31619 drop(pool_meta);
31620 assert!(store.decommission_cancelers.read().await[0].is_none());
31621 assert!(!canceler.is_active());
31622 assert!(canceler.is_cancelled());
31623 assert_eq!(store.ctx.data_movement_operation_epoch(), 1);
31624 }
31625
31626 #[test]
31627 fn test_stale_decommission_operation_cannot_cancel_replacement() {
31628 let stale = DecommissionCanceler::new(CancellationToken::new());
31629 let replacement = DecommissionCanceler::new(CancellationToken::new());
31630 let cancelers = vec![Some(replacement.clone())];
31631 let mut pool_meta = PoolMeta {
31632 pools: vec![decommission_test_pool_status(
31633 0,
31634 Some(PoolDecommissionInfo {
31635 start_time: Some(OffsetDateTime::UNIX_EPOCH),
31636 ..Default::default()
31637 }),
31638 )],
31639 ..Default::default()
31640 };
31641
31642 let changed = update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, 0, Some(&stale), |pool_meta| {
31643 pool_meta.decommission_cancel(0)
31644 });
31645
31646 assert!(changed.is_none());
31647 assert!(
31648 !pool_meta.pools[0]
31649 .decommission
31650 .as_ref()
31651 .expect("replacement metadata should remain")
31652 .canceled
31653 );
31654 assert!(replacement.is_active());
31655 assert!(!replacement.is_cancelled());
31656 assert!(!stale.is_active());
31657 assert!(stale.is_cancelled());
31658 }
31659
31660 #[test]
31661 fn test_ensure_decommission_routines_scheduled_accepts_positive_bound_count() {
31662 assert!(super::ensure_decommission_routines_scheduled(2, 2).is_ok());
31663 }
31664
31665 #[test]
31666 fn test_ensure_decommission_routines_scheduled_rejects_zero_bound_count() {
31667 let err = super::ensure_decommission_routines_scheduled(0, 1).expect_err("zero bound count should be rejected");
31668 assert!(
31669 err.to_string()
31670 .contains("failed to start decommission routines: scheduled 0 of 1 expected workers")
31671 );
31672 }
31673
31674 #[test]
31675 fn test_ensure_decommission_routines_scheduled_rejects_partial_binding() {
31676 let err = super::ensure_decommission_routines_scheduled(1, 2).expect_err("partial binding should be rejected");
31677 assert!(
31678 err.to_string()
31679 .contains("failed to start decommission routines: scheduled 1 of 2 expected workers")
31680 );
31681 }
31682
31683 #[test]
31684 #[cfg(windows)]
31685 fn test_path2_bucket_object_with_base_path_supports_windows_separators() {
31686 let (bucket, object) = super::path2_bucket_object_with_base_path("C:\\data", "C:\\data\\my-bucket\\nested\\object.txt");
31687
31688 assert_eq!(bucket, "my-bucket");
31689 assert_eq!(object, "nested/object.txt");
31690 }
31691}