Skip to main content

rustfs_ecstore/core/
pools.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use crate::bucket::replication::replication_state_from_filemeta;
16#[cfg(all(test, feature = "test-util"))]
17use crate::bucket::utils::is_meta_bucketname;
18use crate::bucket::versioning::VersioningApi as _;
19use crate::bucket::versioning_sys::BucketVersioningSys;
20use crate::bucket::{
21    lifecycle::{
22        DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
23        ValidatedDurableIlmRecord,
24        bucket_lifecycle_audit::LcEventSrc,
25        bucket_lifecycle_ops::{
26            LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_for_data_movement, apply_expiry_rule_in,
27            eval_action_from_lifecycle, lifecycle_delete_all_versions_blocked_by_replication,
28        },
29        classify_durable_ilm_record, get_expiry_configs,
30        lifecycle::IlmAction,
31        tier_delete_journal::durable_ilm_v6_topology_generation,
32        validate_durable_ilm_record,
33    },
34    metadata_sys,
35};
36use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
37use crate::config::com::{
38    CONFIG_PREFIX, delete_config, read_config_limited_preserve_empty, read_config_limited_preserve_empty_with_metadata,
39    read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty, save_config_with_opts,
40    save_config_with_opts_and_metadata,
41};
42use crate::data_movement;
43use crate::data_movement::backpressure::{self, DataMovementOperation};
44use crate::data_usage::DATA_USAGE_CACHE_NAME;
45use crate::disk::error::DiskError;
46use crate::disk::{BUCKET_META_PREFIX, DiskAPI, RUSTFS_META_BUCKET};
47use crate::error::{Error, Result};
48use crate::error::{
49    StorageError, is_err_bucket_exists, is_err_bucket_not_found, is_err_object_not_found, is_err_operation_canceled,
50    is_err_version_not_found,
51};
52use crate::layout::endpoints::EndpointServerPools;
53use crate::object_api::{DecommissionCapacityOptions, GetObjectReader, ObjectInfo, ObjectOptions};
54use crate::runtime::sources as runtime_sources;
55use crate::services::notification_sys::{
56    acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches, tier_delete_journal_topology_generation,
57};
58use crate::services::rebalance::{REBAL_META_NAME, RebalanceMeta, is_rebalance_conflicting_with_decommission};
59use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
60use crate::storage_api_contracts::{
61    admin::StorageAdminApi,
62    bucket::{BucketOperations, BucketOptions, MakeBucketOptions},
63    heal::HealOperations as _,
64    list::ListOperations as _,
65    namespace::NamespaceLocking as _,
66    object::{EcstoreObjectIO, HTTPPreconditions, ObjectIO as _, ObjectOperations as _},
67};
68use crate::store::PoolMetaWriteGateStatus;
69use crate::{core::sets::Sets, store::ECStore};
70use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
71use futures::{
72    StreamExt,
73    future::{BoxFuture, join_all},
74    stream::FuturesUnordered,
75};
76use http::HeaderMap;
77#[cfg(test)]
78use rmp_serde::Deserializer;
79use rmp_serde::Serializer;
80use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
81use rustfs_heal_contracts::heal_channel::HealOpts;
82use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
83use rustfs_utils::path::{
84    decode_dir_object, encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path,
85};
86use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ReplicationConfiguration};
87use serde::{Deserialize, Serialize};
88use sha2::{Digest, Sha256};
89use std::collections::{HashMap, HashSet};
90use std::fmt::Display;
91#[cfg(test)]
92use std::io::Cursor;
93use std::io::Write;
94use std::path::PathBuf;
95use std::sync::{
96    Arc,
97    atomic::{AtomicBool, AtomicUsize, Ordering},
98};
99use time::{Duration, OffsetDateTime};
100use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc};
101use tokio_util::sync::CancellationToken;
102use tracing::{debug, error, info, warn};
103
104const LOG_COMPONENT_ECSTORE: &str = "ecstore";
105const LOG_SUBSYSTEM_POOLS: &str = "pools";
106const EVENT_DECOMMISSION_STATE: &str = "decommission_state";
107const EVENT_DECOMMISSION_BUCKET: &str = "decommission_bucket";
108const EVENT_DECOMMISSION_ENTRY: &str = "decommission_entry";
109const POOL_ACTIVATION_FLEET_PROOF_REQUIRED: &str = "pool activation requires a live fleet capability proof";
110const POOL_ACTIVATION_FLEET_PROOF_EXPIRED: &str = "pool activation fleet capability proof expired before commit";
111const DECOMMISSION_TARGET_FLEET_PROOF_EXPIRED: &str =
112    "decommission target fence fleet capability proof expired before reservation commit";
113const DECOMMISSION_STAGE_MIGRATE_OBJECT: &str = "migrate_object";
114const DECOMMISSION_STAGE_CLEANUP_PREFLIGHT: &str = "cleanup_preflight";
115const DECOMMISSION_STAGE_SOURCE_CLEANUP: &str = "source_cleanup";
116const DECOMMISSION_STAGE_ENTRY_FINISHED: &str = "entry_finished";
117const DECOMMISSION_PROGRESS_SAVE_INTERVAL: Duration = Duration::seconds(30);
118const DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD: usize = 1000;
119const DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF: Duration = Duration::seconds(1);
120const DECOMMISSION_BUCKET_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_BUCKET_CONCURRENCY";
121const DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP: usize = 4;
122const DECOMMISSION_ENTRY_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_ENTRY_CONCURRENCY";
123const DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP: usize = 8;
124const DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP: usize = 64;
125const DECOMMISSION_ENTRY_WORKERS_PER_SET: usize = 2;
126const DECOMMISSION_META_PREFIXES: [&str; 3] = [CONFIG_PREFIX, BUCKET_META_PREFIX, ILM_META_PREFIX];
127const DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION: u16 = 1;
128const DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION: u16 = 2;
129#[cfg(test)]
130const DECOMMISSION_CAPACITY_MODEL_VERSION: u16 = DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION;
131const DECOMMISSION_CAPACITY_TEMPORARY_COPIES: usize = 1;
132const DECOMMISSION_CAPACITY_RESERVATION_TTL: Duration = Duration::minutes(10);
133const DECOMMISSION_CAPACITY_RELEASE_CANCELED: &str = "canceled";
134const DECOMMISSION_CAPACITY_RELEASE_FAILED: &str = "failed";
135const DECOMMISSION_CAPACITY_RELEASE_COMPLETED: &str = "completed";
136pub(crate) const DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX: &str = "decommission/capacity-target";
137const DECOMMISSION_CAPACITY_TARGET_LOCK_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(250);
138const DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX: &str = "target pool ";
139const DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX: &str = " target capacity mutation gate is busy";
140const METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL: &str = "rustfs_decommission_capacity_conflicts_total";
141const METRIC_DECOMMISSION_CAPACITY_PREDICTED_BYTES: &str = "rustfs_decommission_capacity_predicted_physical_bytes";
142const METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES: &str = "rustfs_decommission_capacity_reserved_physical_bytes";
143const METRIC_DECOMMISSION_CAPACITY_PREDICTION_ERROR_BYTES: &str = "rustfs_decommission_capacity_prediction_error_bytes";
144const METRIC_DECOMMISSION_CAPACITY_PREDICTION_ABSOLUTE_ERROR_BYTES: &str =
145    "rustfs_decommission_capacity_prediction_absolute_error_bytes";
146const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3;
147const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
148pub(crate) const DECOMMISSION_ENTRY_MAX_ATTEMPTS: usize = 3;
149const DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS: usize = 12;
150const DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(100);
151pub(crate) const DECOMMISSION_VERSION_COPY_ATTEMPTS: usize = 3;
152const DECOMMISSION_COPY_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(50);
153const DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT: usize = 100;
154const DECOMMISSION_TERMINAL_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(1);
155const DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS: usize = 3;
156const DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT: &str = "decommission/ilm-receipts";
157const DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT: &str = "decommission/ilm-manifests";
158const DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA: &str = "v2";
159const DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA: &str = "v1";
160const DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE: usize = 16 * 1024;
161const DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE: usize = 4 * 1024;
162const DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS: usize = 3;
163/// Background decommission walks must tolerate slow object migrations; the
164/// stall timeout is the drive-health bound, not the total listing duration.
165const DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
166
167pub const POOL_META_NAME: &str = "pool.bin";
168pub(crate) const POOL_META_IDENTITY_NAME: &str = "pool.bin.identity";
169pub const POOL_META_FORMAT: u16 = 1;
170const POOL_META_V1_VERSION: u16 = 1;
171pub const POOL_META_VERSION: u16 = 2;
172const POOL_META_GENERATION_VERSION: u16 = 3;
173const POOL_META_IDENTITY_FORMAT: u16 = 1;
174const POOL_META_IDENTITY_VERSION: u16 = 1;
175const POOL_META_INITIAL_EPOCH: u64 = 1;
176const POOL_META_CAS_MAX_ATTEMPTS: usize = 3;
177const METRIC_POOL_META_STALE_WRITE_REJECTIONS_TOTAL: &str = "rustfs_pool_meta_stale_write_rejections_total";
178
179fn record_pool_meta_stale_write_rejection(reason: &'static str) {
180    metrics::counter!(METRIC_POOL_META_STALE_WRITE_REJECTIONS_TOTAL, "reason" => reason).increment(1);
181}
182
183fn pool_meta_v2_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool {
184    requested && fleet_confirmed
185}
186
187fn pool_meta_v2_writer_enabled() -> bool {
188    pool_meta_v2_writer_enabled_for(
189        rustfs_utils::get_env_bool(rustfs_config::ENV_POOL_META_V2_WRITE, rustfs_config::DEFAULT_POOL_META_V2_WRITE),
190        rustfs_utils::get_env_bool(
191            rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
192            rustfs_config::DEFAULT_POOL_META_V2_FLEET_CONFIRMED,
193        ),
194    )
195}
196
197fn pool_meta_v3_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool {
198    requested && fleet_confirmed
199}
200
201fn pool_meta_v3_writer_enabled() -> bool {
202    pool_meta_v3_writer_enabled_for(
203        rustfs_utils::get_env_bool(rustfs_config::ENV_POOL_META_V3_WRITE, rustfs_config::DEFAULT_POOL_META_V3_WRITE),
204        rustfs_utils::get_env_bool(
205            rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED,
206            rustfs_config::DEFAULT_POOL_META_V3_FLEET_CONFIRMED,
207        ),
208    )
209}
210
211fn decommission_capacity_writer_supported_for(version: u16, v2_writer_enabled: bool, v3_writer_enabled: bool) -> bool {
212    matches!(version, POOL_META_VERSION | POOL_META_GENERATION_VERSION) || v2_writer_enabled || v3_writer_enabled
213}
214
215fn ensure_decommission_ledger_persistence_supported_for(
216    version: u16,
217    v2_writer_enabled: bool,
218    v3_writer_enabled: bool,
219) -> Result<()> {
220    if decommission_capacity_writer_supported_for(version, v2_writer_enabled, v3_writer_enabled) {
221        return Ok(());
222    }
223
224    Err(Error::InvalidArgument(
225        "decommission".to_string(),
226        "pool-metadata-version".to_string(),
227        format!(
228            "durable unresolved-entry recovery requires pool metadata V2 or V3; enable either the {} + {} V2 gate or the {} + {} V3 gate only after every reader and writer supports that format",
229            rustfs_config::ENV_POOL_META_V2_WRITE,
230            rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
231            rustfs_config::ENV_POOL_META_V3_WRITE,
232            rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED,
233        ),
234    ))
235}
236
237fn ensure_decommission_ledger_persistence_supported(pool_meta: &PoolMeta) -> Result<()> {
238    ensure_decommission_ledger_persistence_supported_for(
239        pool_meta.version,
240        pool_meta_v2_writer_enabled(),
241        pool_meta_v3_writer_enabled(),
242    )
243}
244
245#[derive(Clone, Debug)]
246pub struct DecommissionCanceler {
247    operation: Arc<DecommissionOperation>,
248}
249
250#[derive(Debug)]
251struct DecommissionOperation {
252    token: CancellationToken,
253    active: AtomicBool,
254}
255
256impl DecommissionCanceler {
257    pub(crate) fn new(token: CancellationToken) -> Self {
258        Self {
259            operation: Arc::new(DecommissionOperation {
260                token,
261                active: AtomicBool::new(true),
262            }),
263        }
264    }
265
266    #[cfg(test)]
267    pub(crate) fn new_for_test(token: CancellationToken) -> Self {
268        Self::new(token)
269    }
270
271    fn token(&self) -> &CancellationToken {
272        &self.operation.token
273    }
274
275    pub(crate) fn is_active(&self) -> bool {
276        self.operation.active.load(Ordering::Acquire)
277    }
278
279    #[cfg(test)]
280    fn is_cancelled(&self) -> bool {
281        self.token().is_cancelled()
282    }
283
284    fn cancel(&self) {
285        self.token().cancel();
286    }
287
288    fn release(&self) {
289        self.cancel();
290        self.operation.active.store(false, Ordering::Release);
291    }
292
293    fn owns_same_operation(&self, other: &Self) -> bool {
294        Arc::ptr_eq(&self.operation, &other.operation)
295    }
296}
297
298struct DecommissionCancelerGuard {
299    canceler: DecommissionCanceler,
300}
301
302impl DecommissionCancelerGuard {
303    fn new(canceler: DecommissionCanceler) -> Self {
304        Self { canceler }
305    }
306
307    fn canceler(&self) -> &DecommissionCanceler {
308        &self.canceler
309    }
310}
311
312impl Drop for DecommissionCancelerGuard {
313    fn drop(&mut self) {
314        self.canceler.release();
315    }
316}
317
318fn dedup_indices(indices: &[usize]) -> Vec<usize> {
319    let mut seen = HashSet::with_capacity(indices.len());
320    let mut output = Vec::with_capacity(indices.len());
321    for idx in indices {
322        if seen.insert(*idx) {
323            output.push(*idx);
324        }
325    }
326
327    output
328}
329
330fn bind_decommission_cancelers(
331    indices: &[usize],
332    parent: &CancellationToken,
333    cancelers: &mut [Option<DecommissionCanceler>],
334) -> Vec<(usize, DecommissionCanceler)> {
335    let mut bound = Vec::with_capacity(indices.len());
336
337    for idx in indices {
338        if let Some(slot) = cancelers.get_mut(*idx) {
339            if let Some(existing) = slot.take() {
340                existing.release();
341            }
342            let canceler = DecommissionCanceler::new(parent.child_token());
343            *slot = Some(canceler.clone());
344            bound.push((*idx, canceler));
345        }
346    }
347
348    bound
349}
350
351fn bind_missing_decommission_cancelers(
352    indices: &[usize],
353    parent: &CancellationToken,
354    cancelers: &mut [Option<DecommissionCanceler>],
355) -> Vec<(usize, DecommissionCanceler)> {
356    let mut bound = Vec::with_capacity(indices.len());
357
358    for idx in indices {
359        let Some(slot) = cancelers.get_mut(*idx) else {
360            continue;
361        };
362        if slot.as_ref().is_some_and(DecommissionCanceler::is_active) {
363            break;
364        }
365        if let Some(stale) = slot.take() {
366            stale.release();
367        }
368        let canceler = DecommissionCanceler::new(parent.child_token());
369        *slot = Some(canceler.clone());
370        bound.push((*idx, canceler));
371    }
372
373    bound
374}
375
376fn take_decommission_canceler(cancelers: &mut [Option<DecommissionCanceler>], idx: usize) -> Option<DecommissionCanceler> {
377    cancelers.get_mut(idx).and_then(Option::take)
378}
379
380fn take_decommission_canceler_for_operation(
381    cancelers: &mut [Option<DecommissionCanceler>],
382    idx: usize,
383    owner: &DecommissionCanceler,
384) -> Option<DecommissionCanceler> {
385    let slot = cancelers.get_mut(idx)?;
386    if slot.as_ref().is_some_and(|canceler| canceler.owns_same_operation(owner)) {
387        slot.take()
388    } else {
389        None
390    }
391}
392
393fn decommission_canceler_is_owned_by(
394    cancelers: &[Option<DecommissionCanceler>],
395    idx: usize,
396    owner: &DecommissionCanceler,
397) -> bool {
398    cancelers
399        .get(idx)
400        .and_then(Option::as_ref)
401        .is_some_and(|canceler| canceler.owns_same_operation(owner))
402}
403
404fn update_decommission_for_operation<T>(
405    cancelers: &[Option<DecommissionCanceler>],
406    pool_meta: &mut PoolMeta,
407    idx: usize,
408    owner: Option<&DecommissionCanceler>,
409    update: impl FnOnce(&mut PoolMeta) -> T,
410) -> Option<T> {
411    if let Some(owner) = owner
412        && !decommission_canceler_is_owned_by(cancelers, idx, owner)
413    {
414        owner.release();
415        return None;
416    }
417
418    Some(update(pool_meta))
419}
420
421fn has_active_decommission_canceler(cancelers: &[Option<DecommissionCanceler>]) -> bool {
422    cancelers.iter().flatten().any(DecommissionCanceler::is_active)
423}
424
425fn cancel_decommission_canceler(canceler: Option<DecommissionCanceler>) -> bool {
426    if let Some(canceler) = canceler {
427        canceler.release();
428        true
429    } else {
430        false
431    }
432}
433
434fn take_and_cancel_decommission_canceler(cancelers: &mut [Option<DecommissionCanceler>], idx: usize) -> bool {
435    let canceler = take_decommission_canceler(cancelers, idx);
436    cancel_decommission_canceler(canceler)
437}
438
439fn take_and_cancel_decommission_canceler_for_operation(
440    cancelers: &mut [Option<DecommissionCanceler>],
441    idx: usize,
442    owner: &DecommissionCanceler,
443) -> bool {
444    let canceler = take_decommission_canceler_for_operation(cancelers, idx, owner);
445    if canceler.is_none() {
446        owner.release();
447        return false;
448    }
449    cancel_decommission_canceler(canceler)
450}
451
452fn ensure_decommission_routines_scheduled(bound_count: usize, expected_count: usize) -> Result<()> {
453    if bound_count == 0 || bound_count != expected_count {
454        return Err(Error::other(format!(
455            "failed to start decommission routines: scheduled {bound_count} of {expected_count} expected workers"
456        )));
457    }
458
459    Ok(())
460}
461
462fn guard_decommission_cancelers(index_cancelers: Vec<(usize, DecommissionCanceler)>) -> Vec<(usize, DecommissionCancelerGuard)> {
463    index_cancelers
464        .into_iter()
465        .map(|(idx, canceler)| (idx, DecommissionCancelerGuard::new(canceler)))
466        .collect()
467}
468
469async fn await_decommission_worker(idx: usize, worker: tokio::task::JoinHandle<Result<()>>) -> Result<()> {
470    worker
471        .await
472        .map_err(|err| Error::other(format!("decommission worker {idx} task join error: {err}")))?
473}
474
475fn reserve_decommission_start_cancelers(
476    pool_meta: &PoolMeta,
477    indices: &[usize],
478    local_indices: &[usize],
479    parent: &CancellationToken,
480    cancelers: &mut [Option<DecommissionCanceler>],
481) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
482    ensure_decommission_start_pool_states(pool_meta, indices)?;
483    if local_indices.is_empty() {
484        return Ok(Vec::new());
485    }
486    let bound = bind_decommission_cancelers(local_indices, parent, cancelers);
487    let guards = guard_decommission_cancelers(bound);
488    ensure_decommission_routines_scheduled(guards.len(), local_indices.len())?;
489    Ok(guards)
490}
491
492fn default_decommission_bucket_concurrency(cpu_count: usize) -> usize {
493    cpu_count.clamp(1, DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP)
494}
495
496fn decommission_bucket_concurrency_limit() -> usize {
497    let default_limit = default_decommission_bucket_concurrency(num_cpus::get());
498    rustfs_utils::get_env_usize(DECOMMISSION_BUCKET_CONCURRENCY_ENV, default_limit).max(1)
499}
500
501fn default_decommission_entry_concurrency(cpu_count: usize) -> usize {
502    cpu_count.clamp(1, DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP)
503}
504
505fn clamp_decommission_entry_concurrency(limit: usize) -> usize {
506    limit.clamp(1, DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP)
507}
508
509fn decommission_entry_concurrency_limit() -> usize {
510    let default_limit = default_decommission_entry_concurrency(num_cpus::get());
511    clamp_decommission_entry_concurrency(rustfs_utils::get_env_usize(DECOMMISSION_ENTRY_CONCURRENCY_ENV, default_limit))
512}
513
514fn is_decommission_meta_bucket(bucket: &DecomBucketInfo) -> bool {
515    bucket.name == RUSTFS_META_BUCKET
516}
517
518fn decommission_meta_buckets() -> [DecomBucketInfo; DECOMMISSION_META_PREFIXES.len()] {
519    DECOMMISSION_META_PREFIXES.map(|prefix| DecomBucketInfo {
520        name: RUSTFS_META_BUCKET.to_owned(),
521        prefix: prefix.to_owned(),
522    })
523}
524
525fn reconcile_decommission_meta_buckets(meta: &mut PoolMeta, idx: usize) -> bool {
526    let before = meta.pending_buckets(idx).len();
527    meta.queue_buckets(idx, decommission_meta_buckets().into());
528    meta.pending_buckets(idx).len() != before
529}
530
531fn split_decommission_buckets(buckets: Vec<DecomBucketInfo>) -> (Vec<DecomBucketInfo>, Vec<DecomBucketInfo>) {
532    let mut regular = Vec::with_capacity(buckets.len());
533    let mut meta = Vec::new();
534
535    for bucket in buckets {
536        if is_decommission_meta_bucket(&bucket) {
537            meta.push(bucket);
538        } else {
539            regular.push(bucket);
540        }
541    }
542
543    regular.shrink_to_fit();
544    (regular, meta)
545}
546
547fn ensure_decommission_not_rebalancing(rebalance_running: bool) -> Result<()> {
548    if rebalance_running {
549        return Err(Error::RebalanceAlreadyRunning);
550    }
551
552    Ok(())
553}
554
555fn ensure_decommission_start_rebalance_meta_allowed(meta: Option<&RebalanceMeta>) -> Result<()> {
556    ensure_decommission_not_rebalancing(meta.is_some_and(is_rebalance_conflicting_with_decommission))
557}
558
559#[allow(dead_code, reason = "leader precondition asserted by this file's tests (backlog#1823)")]
560fn ensure_local_decommission_pool_leaders(endpoints: &EndpointServerPools, indices: &[usize]) -> Result<()> {
561    for idx in indices {
562        ensure_local_decommission_pool_leader(endpoints, *idx)?;
563    }
564
565    Ok(())
566}
567
568fn ensure_local_decommission_pool_leader(endpoints: &EndpointServerPools, idx: usize) -> Result<()> {
569    let pool = endpoints
570        .as_ref()
571        .get(idx)
572        .ok_or_else(|| invalid_decommission_pool_index_error(endpoints.as_ref().len(), idx))?;
573    let endpoint = pool
574        .endpoints
575        .as_ref()
576        .first()
577        .ok_or_else(|| Error::other(format!("decommission pool {idx} has no configured endpoints")))?;
578
579    if !endpoint.is_local {
580        return Err(Error::other(format!(
581            "decommission for pool {idx} must run on the pool first endpoint {endpoint}"
582        )));
583    }
584
585    Ok(())
586}
587
588fn decommission_pool_first_endpoint_is_local(endpoints: &EndpointServerPools, idx: usize) -> Result<bool> {
589    let pool = endpoints
590        .as_ref()
591        .get(idx)
592        .ok_or_else(|| invalid_decommission_pool_index_error(endpoints.as_ref().len(), idx))?;
593    let endpoint = pool
594        .endpoints
595        .as_ref()
596        .first()
597        .ok_or_else(|| Error::other(format!("decommission pool {idx} has no configured endpoints")))?;
598
599    Ok(endpoint.is_local)
600}
601
602pub(crate) fn local_decommission_queue_prefix(endpoints: &EndpointServerPools, indices: &[usize]) -> Result<Vec<usize>> {
603    let mut local = Vec::with_capacity(indices.len());
604
605    for idx in indices {
606        if decommission_pool_first_endpoint_is_local(endpoints, *idx)? {
607            local.push(*idx);
608        } else {
609            break;
610        }
611    }
612
613    Ok(local)
614}
615
616fn resumable_decommission_queue_indices(meta: &PoolMeta) -> Vec<usize> {
617    let mut indices = Vec::new();
618    for (idx, pool) in meta.pools.iter().enumerate() {
619        if let Some(decommission) = &pool.decommission {
620            if !decommission.has_decommission_state() {
621                continue;
622            }
623            if decommission.complete || decommission.failed || decommission.canceled {
624                continue;
625            }
626            indices.push(idx);
627        }
628    }
629
630    indices
631}
632
633fn missing_decommission_worker_prefix(indices: &[usize], cancelers: &[Option<DecommissionCanceler>]) -> Vec<usize> {
634    let mut missing = Vec::with_capacity(indices.len());
635
636    for idx in indices {
637        if cancelers
638            .get(*idx)
639            .and_then(Option::as_ref)
640            .is_some_and(DecommissionCanceler::is_active)
641        {
642            break;
643        }
644        missing.push(*idx);
645    }
646
647    missing
648}
649
650fn ensure_decommission_start_local_leader(endpoints: &EndpointServerPools, indices: &[usize]) -> Result<()> {
651    if let Some(first) = indices.first() {
652        ensure_local_decommission_pool_leader(endpoints, *first)?;
653    }
654
655    Ok(())
656}
657
658fn build_decommission_start_state(
659    pi: PoolSpaceInfo,
660    queued: bool,
661    now: OffsetDateTime,
662    previous: Option<&PoolDecommissionInfo>,
663) -> PoolDecommissionInfo {
664    let mut info = PoolDecommissionInfo {
665        start_time: if queued { None } else { Some(now) },
666        start_size: pi.free,
667        total_size: pi.total,
668        current_size: pi.free,
669        queued,
670        ..Default::default()
671    };
672
673    if let Some(previous) = previous
674        && (previous.failed || previous.canceled)
675    {
676        info.decommissioned_buckets = previous.decommissioned_buckets.clone();
677        info.items_decommissioned = previous.items_decommissioned;
678        info.bytes_done = previous.bytes_done;
679        info.unresolved_entries = previous.unresolved_entries.clone();
680        if let Some(generation) = info.start_time {
681            for entry in &mut info.unresolved_entries {
682                entry.source_generation = generation;
683            }
684        }
685        info.mark_progress_saved();
686    }
687
688    info
689}
690
691fn spawn_decommission_index_cancelers(
692    store: Arc<ECStore>,
693    rx: CancellationToken,
694    index_cancelers: Vec<(usize, DecommissionCancelerGuard)>,
695    entry_budget: Arc<Semaphore>,
696) -> tokio::task::JoinHandle<()> {
697    tokio::spawn(async move {
698        let mut stop_queue = false;
699
700        for (idx, canceler_guard) in index_cancelers {
701            let canceler = canceler_guard.canceler().clone();
702            if stop_queue || rx.is_cancelled() {
703                canceler.cancel();
704                store.retry_decommission_cancel_for_operation(idx, &canceler).await;
705                continue;
706            }
707
708            let worker = tokio::spawn({
709                let store = store.clone();
710                let canceler = canceler.clone();
711                let entry_budget = entry_budget.clone();
712                async move { store.do_decommission_in_routine(canceler, idx, entry_budget).await }
713            });
714            if let Err(err) = await_decommission_worker(idx, worker).await {
715                if is_decommission_capacity_blocked_error(&err) || is_decommission_target_capacity_error(&err) {
716                    warn!(
717                        event = EVENT_DECOMMISSION_STATE,
718                        component = LOG_COMPONENT_ECSTORE,
719                        subsystem = LOG_SUBSYSTEM_POOLS,
720                        pool_index = idx,
721                        state = "capacity_blocked",
722                        error = %err,
723                        "Decommission routine paused without a terminal transition"
724                    );
725                    store.release_decommission_canceler_slot(idx, &canceler).await;
726                    stop_queue = true;
727                    continue;
728                }
729                if let Err(blocked) = store
730                    .ensure_pool_meta_side_effects_safe("decommission paused because pool metadata requires recovery")
731                    .await
732                {
733                    warn!(
734                        event = EVENT_DECOMMISSION_STATE,
735                        component = LOG_COMPONENT_ECSTORE,
736                        subsystem = LOG_SUBSYSTEM_POOLS,
737                        pool_index = idx,
738                        state = "routine_blocked",
739                        error = %blocked,
740                        "Decommission routine paused without changing terminal state"
741                    );
742                    store.release_decommission_canceler_slot(idx, &canceler).await;
743                    stop_queue = true;
744                    continue;
745                }
746                error!(
747                    event = EVENT_DECOMMISSION_STATE,
748                    component = LOG_COMPONENT_ECSTORE,
749                    subsystem = LOG_SUBSYSTEM_POOLS,
750                    pool_index = idx,
751                    state = "routine_failed",
752                    error = %err,
753                    "Decommission routine failed"
754                );
755                store.quiesce_decommission_worker_after_join_error(&canceler).await;
756                store.retry_decommission_failed_for_operation(idx, &canceler).await;
757                stop_queue = true;
758                continue;
759            }
760
761            stop_queue = {
762                let pool_meta = store.pool_meta.read().await;
763                !should_continue_decommission_queue(&pool_meta, idx)
764            };
765        }
766    })
767}
768
769fn decommission_meta_bucket_options() -> MakeBucketOptions {
770    MakeBucketOptions {
771        force_create: true,
772        ..Default::default()
773    }
774}
775
776fn is_decommission_active(complete: bool, failed: bool, canceled: bool) -> bool {
777    !complete && !failed && !canceled
778}
779
780pub(crate) fn pool_meta_has_active_decommission(meta: &PoolMeta) -> bool {
781    meta.pools.iter().any(|pool| {
782        pool.decommission.as_ref().is_some_and(|info| {
783            info.has_decommission_state() && is_decommission_active(info.complete, info.failed, info.canceled)
784        })
785    })
786}
787
788fn is_decommission_suspended(info: &PoolDecommissionInfo) -> bool {
789    info.has_decommission_state()
790}
791
792fn validate_decommission_terminal_state(complete: bool, failed: bool, canceled: bool) -> Result<()> {
793    let terminal_count = [complete, failed, canceled].into_iter().filter(|terminal| *terminal).count();
794    if terminal_count > 1 {
795        return Err(Error::other(format!(
796            "pool metadata load failed: invalid decommission terminal state complete={complete} failed={failed} canceled={canceled}"
797        )));
798    }
799    Ok(())
800}
801
802fn invalid_decommission_pool_index_error(pool_count: usize, idx: usize) -> Error {
803    Error::other(format!("invalid decommission pool index {idx} for {pool_count} pools"))
804}
805
806#[derive(Debug, Clone, Copy, PartialEq, Eq)]
807enum DecommissionStartPoolState {
808    Missing,
809    Active,
810    Retryable,
811    Decommissioning,
812    Decommissioned,
813    Blocked,
814}
815
816fn decommission_start_pool_state(pool: Option<&PoolStatus>) -> DecommissionStartPoolState {
817    let Some(pool) = pool else {
818        return DecommissionStartPoolState::Missing;
819    };
820    let Some(info) = pool.decommission.as_ref() else {
821        return DecommissionStartPoolState::Active;
822    };
823    if !info.has_decommission_state() {
824        return DecommissionStartPoolState::Active;
825    }
826
827    if info.complete {
828        DecommissionStartPoolState::Decommissioned
829    } else if info.failed || info.canceled {
830        if info.unresolved_entries.is_empty() {
831            DecommissionStartPoolState::Blocked
832        } else {
833            DecommissionStartPoolState::Retryable
834        }
835    } else {
836        DecommissionStartPoolState::Decommissioning
837    }
838}
839
840fn is_decommission_start_active_pool(pool: &PoolStatus) -> bool {
841    decommission_start_pool_state(Some(pool)) == DecommissionStartPoolState::Active
842}
843
844fn invalid_decommission_request(reason: impl Into<String>) -> Error {
845    Error::InvalidArgument("decommission".to_string(), "pool-state".to_string(), reason.into())
846}
847
848fn ensure_decommission_start_allowed(state: DecommissionStartPoolState) -> Result<()> {
849    match state {
850        DecommissionStartPoolState::Missing => {
851            Err(invalid_decommission_request("failed to start decommission: target pool was not found"))
852        }
853        DecommissionStartPoolState::Active | DecommissionStartPoolState::Retryable => Ok(()),
854        DecommissionStartPoolState::Decommissioning => Err(StorageError::DecommissionAlreadyRunning),
855        DecommissionStartPoolState::Decommissioned => Err(invalid_decommission_request(
856            "failed to start decommission: target pool is already decommissioned",
857        )),
858        DecommissionStartPoolState::Blocked => Err(invalid_decommission_request(
859            "failed to start decommission: target pool decommission is blocked; clear failed or canceled metadata before starting again",
860        )),
861    }
862}
863
864fn ensure_decommission_start_keeps_active_pool(meta: &PoolMeta, indices: &[usize]) -> Result<()> {
865    let active_count = meta
866        .pools
867        .iter()
868        .filter(|pool| is_decommission_start_active_pool(pool))
869        .count();
870    let active_target_count = indices
871        .iter()
872        .filter(|idx| meta.pools.get(**idx).is_some_and(is_decommission_start_active_pool))
873        .count();
874    if active_count.saturating_sub(active_target_count) == 0 {
875        return Err(invalid_decommission_request(
876            "failed to start decommission: at least one active pool must remain after decommission start",
877        ));
878    }
879
880    Ok(())
881}
882
883fn ensure_decommission_start_pool_states(meta: &PoolMeta, indices: &[usize]) -> Result<()> {
884    for idx in indices.iter().copied() {
885        ensure_decommission_start_allowed(decommission_start_pool_state(meta.pools.get(idx)))?;
886    }
887    ensure_decommission_start_keeps_active_pool(meta, indices)
888}
889
890fn capacity_mul_div_ceil(value: usize, multiplier: usize, divisor: usize) -> usize {
891    if value == 0 || multiplier == 0 {
892        return 0;
893    }
894    if divisor == 0 {
895        return usize::MAX;
896    }
897    value
898        .checked_mul(multiplier)
899        .map(|product| product.div_ceil(divisor))
900        .unwrap_or(usize::MAX)
901}
902
903fn capacity_source_data_equivalent(physical_bytes: usize, layout: DecommissionErasureLayout) -> Result<usize> {
904    if !layout.is_valid() {
905        return Err(Error::DecommissionCapacity(format!(
906            "failed to model decommission capacity: invalid source layout data={} parity={}",
907            layout.data, layout.parity
908        )));
909    }
910    Ok(capacity_mul_div_ceil(physical_bytes, layout.data, layout.width()))
911}
912
913fn capacity_target_physical_bytes(data_bytes: usize, layout: DecommissionErasureLayout) -> Result<usize> {
914    if !layout.is_valid() {
915        return Err(Error::DecommissionCapacity(format!(
916            "failed to model decommission capacity: invalid target layout data={} parity={}",
917            layout.data, layout.parity
918        )));
919    }
920    Ok(capacity_mul_div_ceil(data_bytes, layout.width(), layout.data))
921}
922
923fn worst_decommission_target_layout(targets: &[DecommissionPoolCapacityInfo]) -> Result<DecommissionErasureLayout> {
924    targets
925        .iter()
926        .map(|target| target.layout)
927        .filter(|layout| layout.is_valid())
928        .max_by(|left, right| {
929            ((left.width() as u128) * (right.data as u128)).cmp(&((right.width() as u128) * (left.data as u128)))
930        })
931        .ok_or_else(|| Error::other("failed to start decommission: no valid target erasure layout is available"))
932}
933
934fn decommission_capacity_writer_supported(meta: &PoolMeta) -> bool {
935    decommission_capacity_writer_supported_for(meta.version, pool_meta_v2_writer_enabled(), pool_meta_v3_writer_enabled())
936}
937
938fn ensure_decommission_capacity_writer_supported(meta: &PoolMeta) -> Result<()> {
939    if decommission_capacity_writer_supported(meta) {
940        return Ok(());
941    }
942    Err(Error::DecommissionCapacity(format!(
943        "failed to start decommission: durable capacity reservations require pool metadata V2 or V3; enable either the {} + {} V2 gate or the {} + {} V3 gate only after every reader and writer supports that format",
944        rustfs_config::ENV_POOL_META_V2_WRITE,
945        rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
946        rustfs_config::ENV_POOL_META_V3_WRITE,
947        rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED,
948    )))
949}
950
951fn decommission_capacity_blocked_error(message: impl Display) -> Error {
952    Error::DecommissionCapacityBlocked {
953        message: message.to_string(),
954    }
955}
956
957fn is_decommission_capacity_blocked_error(err: &Error) -> bool {
958    if matches!(err, Error::DecommissionCapacityBlocked { .. }) {
959        return true;
960    }
961    data_movement::data_movement_stage_source(err).is_some_and(is_decommission_capacity_blocked_error)
962}
963
964fn is_decommission_capacity_intent_conflict(err: &Error) -> bool {
965    if let Error::DecommissionCapacityBlocked { message } = err {
966        return message.contains("unresolved target capacity intent")
967            || message.contains("pending capacity intent belongs to another mutation");
968    }
969    data_movement::data_movement_stage_source(err).is_some_and(is_decommission_capacity_intent_conflict)
970        || err.to_string().contains("unresolved target capacity intent")
971}
972
973fn decommission_capacity_target_gate_busy_index(err: &Error) -> Option<usize> {
974    if let Error::DecommissionCapacityBlocked { message } = err {
975        return message
976            .strip_prefix(DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX)?
977            .strip_suffix(DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX)?
978            .parse()
979            .ok();
980    }
981    data_movement::data_movement_stage_source(err).and_then(decommission_capacity_target_gate_busy_index)
982}
983
984fn is_decommission_capacity_target_gate_busy(err: &Error) -> bool {
985    decommission_capacity_target_gate_busy_index(err).is_some()
986}
987
988#[derive(Debug, Clone, Copy, PartialEq, Eq)]
989enum DecommissionCapacityRetryKind {
990    IntentConflict,
991}
992
993fn decommission_capacity_retry_kind(err: &Error, intent_conflict_attempt: usize) -> Option<DecommissionCapacityRetryKind> {
994    (intent_conflict_attempt < DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS
995        && is_decommission_capacity_intent_conflict(err))
996    .then_some(DecommissionCapacityRetryKind::IntentConflict)
997}
998
999fn ensure_decommission_capacity_target_fence(
1000    guard: &rustfs_lock::NamespaceLockGuard,
1001    target_pool_index: usize,
1002    phase: &str,
1003) -> Result<()> {
1004    if guard.is_lock_lost() {
1005        return Err(Error::DecommissionCapacity(format!(
1006            "target pool {target_pool_index} capacity mutation fence was lost during {phase}"
1007        )));
1008    }
1009    Ok(())
1010}
1011
1012fn ensure_decommission_capacity_mutation_intent_current(
1013    meta: &PoolMeta,
1014    owner: DecommissionCapacityOwner,
1015    target_pool_index: usize,
1016    expected_target_physical_bytes: usize,
1017    mutation_id: uuid::Uuid,
1018    temporary_release: bool,
1019    model_version: u16,
1020) -> Result<()> {
1021    let reservation = meta
1022        .pools
1023        .get(owner.source_pool_index)
1024        .and_then(|pool| pool.decommission.as_ref())
1025        .and_then(|info| info.capacity_reservation.as_ref())
1026        .filter(|reservation| {
1027            // Admission validated the owner nonce before persisting this
1028            // intent. The target lock is the mutation fence after that point,
1029            // so a concurrent lease renewal may rotate the nonce without
1030            // invalidating the already-durable intent.
1031            reservation.active()
1032                && reservation.source_pool_index == owner.source_pool_index
1033                && reservation.operation_id == owner.operation_id
1034                && reservation.generation == owner.generation
1035                && reservation.model_version == model_version
1036        })
1037        .ok_or_else(|| {
1038            decommission_capacity_blocked_error("decommission target mutation owner changed before capacity finalize")
1039        })?;
1040    let target = reservation
1041        .targets
1042        .iter()
1043        .find(|target| target.pool_index == target_pool_index)
1044        .ok_or_else(|| decommission_capacity_blocked_error("decommission target allocation changed before capacity finalize"))?;
1045
1046    if temporary_release {
1047        // Cleanup is authorized by the persisted owner and target identity.
1048        // Its exact temporary or pending record may already be absent after a
1049        // prior successful finalize, so attribution is decided from the
1050        // physical cleanup result below instead of rejecting the retry here.
1051        return Ok(());
1052    }
1053
1054    if expected_target_physical_bytes == 0
1055        || (target.pending_mutation_id == Some(mutation_id) && target.pending_physical_bytes >= expected_target_physical_bytes)
1056    {
1057        return Ok(());
1058    }
1059    Err(decommission_capacity_blocked_error(
1060        "pending capacity intent belongs to another mutation before capacity finalize",
1061    ))
1062}
1063
1064fn validate_decommission_capacity_reservation(reservation: Option<&DecommissionCapacityReservation>) -> Result<()> {
1065    let Some(reservation) = reservation else {
1066        return Ok(());
1067    };
1068    if !matches!(
1069        reservation.model_version,
1070        DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
1071    ) {
1072        return Err(Error::DecommissionCapacity(format!(
1073            "pool metadata load failed: unsupported decommission capacity model version {}",
1074            reservation.model_version
1075        )));
1076    }
1077    if reservation.operation_id.is_nil() || reservation.generation == 0 || reservation.owner_nonce.is_nil() {
1078        return Err(Error::other(
1079            "pool metadata load failed: decommission capacity reservation identity is invalid",
1080        ));
1081    }
1082    if !reservation.source_layout.is_valid() || reservation.targets.iter().any(|target| !target.layout.is_valid()) {
1083        return Err(Error::other(
1084            "pool metadata load failed: decommission capacity reservation contains an invalid erasure layout",
1085        ));
1086    }
1087    if reservation.temporary_copies != DECOMMISSION_CAPACITY_TEMPORARY_COPIES {
1088        return Err(Error::other(
1089            "pool metadata load failed: decommission capacity reservation temporary-copy model is invalid",
1090        ));
1091    }
1092    if reservation.source_data_equivalent_bytes
1093        != capacity_source_data_equivalent(reservation.source_physical_bytes, reservation.source_layout)?
1094    {
1095        return Err(Error::other(
1096            "pool metadata load failed: decommission capacity reservation source estimate is invalid",
1097        ));
1098    }
1099    if reservation.peak_physical_bytes
1100        != reservation
1101            .predicted_physical_bytes
1102            .saturating_add(reservation.temporary_physical_bytes)
1103        || reservation.temporary_physical_bytes
1104            != reservation
1105                .predicted_physical_bytes
1106                .saturating_mul(reservation.temporary_copies)
1107    {
1108        return Err(Error::other(
1109            "pool metadata load failed: decommission capacity reservation peak does not match its physical model",
1110        ));
1111    }
1112    if reservation.committed_data_bytes > reservation.source_data_equivalent_bytes {
1113        return Err(Error::other(
1114            "pool metadata load failed: decommission capacity committed bytes exceed the source estimate",
1115        ));
1116    }
1117    if reservation.consumed_target_physical_bytes > reservation.predicted_physical_bytes {
1118        return Err(Error::other(
1119            "pool metadata load failed: decommission capacity consumed target bytes exceed the prediction",
1120        ));
1121    }
1122    let mut target_indices = HashSet::with_capacity(reservation.targets.len());
1123    let mut reserved_physical_bytes = 0usize;
1124    let mut consumed_physical_bytes = 0usize;
1125    let mut observed_physical_bytes = 0usize;
1126    let mut inflight_physical_bytes = 0usize;
1127    let mut pending_physical_bytes = 0usize;
1128    for target in &reservation.targets {
1129        let mut temporary_mutation_ids = HashSet::with_capacity(target.temporary_mutations.len());
1130        let temporary_mutation_bytes = target.temporary_mutations.iter().try_fold(0usize, |total, mutation| {
1131            if mutation.mutation_id.is_nil()
1132                || (mutation.physical_bytes == 0 && reservation.model_version != DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION)
1133                || !temporary_mutation_ids.insert(mutation.mutation_id)
1134            {
1135                return Err(Error::other(
1136                    "pool metadata load failed: decommission capacity temporary mutation is invalid",
1137                ));
1138            }
1139            Ok(total.saturating_add(mutation.physical_bytes))
1140        })?;
1141        if target.pool_index == reservation.source_pool_index
1142            || !target_indices.insert(target.pool_index)
1143            || target.reserved_physical_bytes == 0
1144            || target.reserved_physical_bytes > target.physical_free_at_reservation
1145            || target.physical_free_at_reservation > target.physical_total_at_reservation
1146            || target.consumed_physical_bytes > reservation.consumed_target_physical_bytes
1147            || target.observed_physical_bytes > reservation.observed_target_physical_bytes
1148            || target.inflight_physical_bytes > target.observed_physical_bytes
1149            || temporary_mutation_bytes > target.inflight_physical_bytes
1150        {
1151            return Err(Error::other(
1152                "pool metadata load failed: decommission capacity reservation target allocation is invalid",
1153            ));
1154        }
1155        if reservation.predicted_physical_bytes
1156            < capacity_target_physical_bytes(reservation.source_data_equivalent_bytes, target.layout)?
1157        {
1158            return Err(Error::other(
1159                "pool metadata load failed: decommission capacity reservation target estimate is unsafe",
1160            ));
1161        }
1162        reserved_physical_bytes =
1163            reserved_physical_bytes.saturating_add(target.remaining_reserved_physical_bytes(reservation.temporary_copies));
1164        consumed_physical_bytes = consumed_physical_bytes.saturating_add(target.consumed_physical_bytes);
1165        observed_physical_bytes = observed_physical_bytes.saturating_add(target.observed_physical_bytes);
1166        inflight_physical_bytes = inflight_physical_bytes.saturating_add(target.inflight_physical_bytes);
1167        pending_physical_bytes = pending_physical_bytes.saturating_add(target.pending_physical_bytes);
1168    }
1169    if reserved_physical_bytes != reservation.remaining_peak_physical_bytes() {
1170        return Err(Error::other(
1171            "pool metadata load failed: decommission capacity reservation target allocation is incomplete",
1172        ));
1173    }
1174    if inflight_physical_bytes != reservation.inflight_target_physical_bytes {
1175        return Err(Error::other(
1176            "pool metadata load failed: decommission capacity inflight target accounting is incomplete",
1177        ));
1178    }
1179    if pending_physical_bytes != reservation.pending_target_physical_bytes {
1180        return Err(Error::other(
1181            "pool metadata load failed: decommission capacity pending target accounting is incomplete",
1182        ));
1183    }
1184    if consumed_physical_bytes != reservation.consumed_target_physical_bytes
1185        || observed_physical_bytes != reservation.observed_target_physical_bytes
1186    {
1187        return Err(Error::other(
1188            "pool metadata load failed: decommission capacity target progress accounting is incomplete",
1189        ));
1190    }
1191    if reservation.released_at.is_some() != reservation.release_reason.is_some() {
1192        return Err(Error::other(
1193            "pool metadata load failed: decommission capacity reservation release state is incomplete",
1194        ));
1195    }
1196    Ok(())
1197}
1198
1199fn active_decommission_capacity_model(meta: &PoolMeta) -> Result<Option<u16>> {
1200    let mut active_model = None;
1201    for reservation in meta
1202        .pools
1203        .iter()
1204        .filter_map(|pool| pool.decommission.as_ref()?.capacity_reservation.as_ref())
1205        .filter(|reservation| reservation.active())
1206    {
1207        if !matches!(
1208            reservation.model_version,
1209            DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
1210        ) {
1211            return Err(Error::DecommissionCapacity(format!(
1212                "pool metadata load failed: unsupported active decommission capacity model version {}",
1213                reservation.model_version
1214            )));
1215        }
1216        match active_model {
1217            Some(model_version) if model_version != reservation.model_version => {
1218                return Err(Error::DecommissionCapacity(
1219                    "pool metadata load failed: active decommission capacity reservations use mixed lock models".to_string(),
1220                ));
1221            }
1222            Some(_) => {}
1223            None => active_model = Some(reservation.model_version),
1224        }
1225    }
1226    Ok(active_model)
1227}
1228
1229fn validate_decommission_capacity_model_cohort(meta: &PoolMeta) -> Result<()> {
1230    active_decommission_capacity_model(meta).map(|_| ())
1231}
1232
1233fn select_decommission_capacity_model(meta: &PoolMeta, target_fence_proof_available: bool) -> Result<u16> {
1234    match active_decommission_capacity_model(meta)? {
1235        Some(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION) => Ok(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION),
1236        Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) if target_fence_proof_available => {
1237            Ok(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION)
1238        }
1239        Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) => Err(Error::DecommissionCapacity(
1240            "failed to start decommission: an active per-target capacity cohort requires a live all-v4 fleet proof".to_string(),
1241        )),
1242        Some(version) => Err(Error::DecommissionCapacity(format!(
1243            "failed to start decommission: unsupported active capacity model version {version}"
1244        ))),
1245        None if target_fence_proof_available => Ok(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION),
1246        None => Ok(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION),
1247    }
1248}
1249
1250fn ensure_decommission_target_fence_fleet_proof(
1251    proof: Option<&crate::services::notification_sys::DecommissionTargetFenceFleetProofToken>,
1252    required: bool,
1253) -> Result<()> {
1254    if !required {
1255        return Ok(());
1256    }
1257    if proof.is_some_and(crate::services::notification_sys::decommission_target_fence_fleet_proof_matches) {
1258        return Ok(());
1259    }
1260    Err(Error::other(DECOMMISSION_TARGET_FLEET_PROOF_EXPIRED))
1261}
1262
1263fn release_decommission_capacity_reservation(info: &mut PoolDecommissionInfo, reason: &str, now: OffsetDateTime) -> bool {
1264    let Some(reservation) = info.capacity_reservation.as_mut() else {
1265        return false;
1266    };
1267    if !reservation.active() {
1268        return false;
1269    }
1270    reservation.released_at = Some(now);
1271    reservation.release_reason = Some(reason.to_string());
1272    metrics::gauge!(
1273        METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES,
1274        "pool_index" => reservation.source_pool_index.to_string()
1275    )
1276    .set(0.0);
1277    true
1278}
1279
1280fn renew_decommission_capacity_reservation(
1281    reservation: &mut DecommissionCapacityReservation,
1282    now: OffsetDateTime,
1283    recover_expired: bool,
1284) -> bool {
1285    if !reservation.active() {
1286        return false;
1287    }
1288    let expired = reservation.expires_at <= now;
1289    if expired && recover_expired {
1290        reservation.owner_nonce = uuid::Uuid::new_v4();
1291        reservation.recovered_at = Some(now);
1292    }
1293    reservation.renewed_at = now;
1294    reservation.expires_at = now + DECOMMISSION_CAPACITY_RESERVATION_TTL;
1295    true
1296}
1297
1298fn next_decommission_capacity_generation(meta: &PoolMeta) -> Result<u64> {
1299    meta.pools
1300        .iter()
1301        .filter_map(|pool| pool.decommission.as_ref()?.capacity_reservation.as_ref())
1302        .map(|reservation| reservation.generation)
1303        .max()
1304        .unwrap_or_default()
1305        .checked_add(1)
1306        .ok_or_else(|| Error::other("failed to start decommission: capacity reservation generation overflow"))
1307}
1308
1309fn active_decommission_source_indices(meta: &PoolMeta) -> HashSet<usize> {
1310    meta.pools
1311        .iter()
1312        .enumerate()
1313        .filter_map(|(idx, pool)| {
1314            pool.decommission.as_ref().and_then(|info| {
1315                (info.has_decommission_state() && is_decommission_active(info.complete, info.failed, info.canceled))
1316                    .then_some(idx)
1317            })
1318        })
1319        .collect()
1320}
1321
1322#[cfg(test)]
1323fn build_decommission_capacity_reservation(
1324    source: DecommissionPoolCapacityInfo,
1325    target_layout: DecommissionErasureLayout,
1326    operation_id: uuid::Uuid,
1327    generation: u64,
1328    now: OffsetDateTime,
1329) -> Result<DecommissionCapacityReservation> {
1330    build_decommission_capacity_reservation_with_model(
1331        source,
1332        target_layout,
1333        operation_id,
1334        generation,
1335        now,
1336        DECOMMISSION_CAPACITY_MODEL_VERSION,
1337    )
1338}
1339
1340fn build_decommission_capacity_reservation_with_model(
1341    source: DecommissionPoolCapacityInfo,
1342    target_layout: DecommissionErasureLayout,
1343    operation_id: uuid::Uuid,
1344    generation: u64,
1345    now: OffsetDateTime,
1346    model_version: u16,
1347) -> Result<DecommissionCapacityReservation> {
1348    if !matches!(
1349        model_version,
1350        DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
1351    ) {
1352        return Err(Error::DecommissionCapacity(format!(
1353            "failed to build decommission capacity reservation: unsupported lock model {model_version}"
1354        )));
1355    }
1356    let source_data_equivalent_bytes = capacity_source_data_equivalent(source.physical_used, source.layout)?;
1357    let predicted_physical_bytes = capacity_target_physical_bytes(source_data_equivalent_bytes, target_layout)?;
1358    let temporary_physical_bytes = predicted_physical_bytes.saturating_mul(DECOMMISSION_CAPACITY_TEMPORARY_COPIES);
1359    Ok(DecommissionCapacityReservation {
1360        model_version,
1361        operation_id,
1362        generation,
1363        owner_nonce: uuid::Uuid::new_v4(),
1364        source_pool_index: source.pool_index,
1365        source_layout: source.layout,
1366        source_physical_total_bytes: source.physical_total,
1367        source_physical_bytes: source.physical_used,
1368        source_data_equivalent_bytes,
1369        predicted_physical_bytes,
1370        temporary_copies: DECOMMISSION_CAPACITY_TEMPORARY_COPIES,
1371        temporary_physical_bytes,
1372        peak_physical_bytes: predicted_physical_bytes.saturating_add(temporary_physical_bytes),
1373        committed_data_bytes: 0,
1374        consumed_target_physical_bytes: 0,
1375        observed_target_physical_bytes: 0,
1376        inflight_target_physical_bytes: 0,
1377        pending_target_physical_bytes: 0,
1378        prediction_error_bytes: 0,
1379        targets: Vec::new(),
1380        created_at: now,
1381        renewed_at: now,
1382        expires_at: now + DECOMMISSION_CAPACITY_RESERVATION_TTL,
1383        recovered_at: None,
1384        released_at: None,
1385        release_reason: None,
1386    })
1387}
1388
1389fn reserve_decommission_start_target_capacity(
1390    meta: &mut PoolMeta,
1391    requested_indices: &[usize],
1392    capacity_infos: &[DecommissionPoolCapacityInfo],
1393    operation_id: uuid::Uuid,
1394    generation: u64,
1395    now: OffsetDateTime,
1396    model_version: u16,
1397) -> Result<()> {
1398    validate_decommission_capacity_model_cohort(meta)?;
1399    let active_sources = active_decommission_source_indices(meta);
1400    let targets = capacity_infos
1401        .iter()
1402        .copied()
1403        .filter(|capacity| {
1404            !active_sources.contains(&capacity.pool_index)
1405                && meta
1406                    .pools
1407                    .get(capacity.pool_index)
1408                    .is_some_and(is_decommission_start_active_pool)
1409        })
1410        .collect::<Vec<_>>();
1411    let target_layout = worst_decommission_target_layout(&targets)?;
1412    let requested = requested_indices.iter().copied().collect::<HashSet<_>>();
1413    let mut reservations = Vec::with_capacity(active_sources.len());
1414
1415    let mut source_indices = active_sources.into_iter().collect::<Vec<_>>();
1416    source_indices.sort_unstable();
1417    for source_index in source_indices {
1418        let source = capacity_infos
1419            .iter()
1420            .copied()
1421            .find(|capacity| capacity.pool_index == source_index)
1422            .ok_or_else(|| {
1423                Error::DecommissionCapacity(format!(
1424                    "failed to start decommission: capacity snapshot is missing source pool {source_index}"
1425                ))
1426            })?;
1427        let pool = meta
1428            .pools
1429            .get(source_index)
1430            .ok_or_else(|| invalid_decommission_pool_index_error(meta.pools.len(), source_index))?;
1431        let info = pool
1432            .decommission
1433            .as_ref()
1434            .ok_or_else(|| decommission_metadata_not_initialized_error("reserve decommission capacity"))?;
1435        let new_reservation = requested.contains(&source_index);
1436        let mut reservation = if new_reservation {
1437            build_decommission_capacity_reservation_with_model(
1438                source,
1439                target_layout,
1440                operation_id,
1441                generation,
1442                now,
1443                model_version,
1444            )?
1445        } else {
1446            info.capacity_reservation.clone().ok_or_else(|| {
1447                Error::DecommissionCapacity(format!(
1448                    "failed to start decommission: active source pool {source_index} has no durable capacity reservation"
1449                ))
1450            })?
1451        };
1452        if !reservation.active() {
1453            return Err(Error::DecommissionCapacity(format!(
1454                "failed to start decommission: active source pool {source_index} has a released capacity reservation"
1455            )));
1456        }
1457        renew_decommission_capacity_reservation(&mut reservation, now, true);
1458        reservations.push((source_index, reservation, new_reservation));
1459    }
1460
1461    let mut remaining_target_capacity = targets
1462        .iter()
1463        .map(|target| (target.pool_index, target.physical_free))
1464        .collect::<HashMap<_, _>>();
1465    for (_, reservation, new_reservation) in &reservations {
1466        if *new_reservation {
1467            continue;
1468        }
1469        for target in &reservation.targets {
1470            let available = remaining_target_capacity.entry(target.pool_index).or_default();
1471            *available = available
1472                .saturating_add(target.inflight_physical_bytes)
1473                .saturating_add(target.pending_physical_bytes);
1474            let required = target.remaining_reserved_physical_bytes(reservation.temporary_copies);
1475            if *available < required {
1476                return Err(decommission_capacity_blocked_error(format!(
1477                    "failed to start decommission: existing reservation for source pool {} requires {required} bytes on target pool {}, but only {} bytes remain",
1478                    reservation.source_pool_index, target.pool_index, *available
1479                )));
1480            }
1481            *available -= required;
1482        }
1483    }
1484    let required = reservations
1485        .iter()
1486        .filter(|(_, _, new_reservation)| *new_reservation)
1487        .fold(0usize, |total, (_, reservation, _)| {
1488            total.saturating_add(reservation.remaining_peak_physical_bytes())
1489        });
1490    let available = remaining_target_capacity
1491        .values()
1492        .copied()
1493        .fold(0usize, usize::saturating_add);
1494    if available < required {
1495        metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => "activation").increment(1);
1496        return Err(decommission_capacity_blocked_error(format!(
1497            "failed to start decommission: insufficient reserved physical target capacity: operation {operation_id} generation {generation} requires {required} bytes, but {available} bytes are available after source/target parity and temporary-copy accounting"
1498        )));
1499    }
1500
1501    for (_, reservation, new_reservation) in &mut reservations {
1502        if !*new_reservation {
1503            continue;
1504        }
1505        let mut remaining = reservation.remaining_peak_physical_bytes();
1506        for target in &targets {
1507            if remaining == 0 {
1508                break;
1509            }
1510            let target_remaining = remaining_target_capacity.entry(target.pool_index).or_default();
1511            let allocated = remaining.min(*target_remaining);
1512            if allocated == 0 {
1513                continue;
1514            }
1515            *target_remaining -= allocated;
1516            remaining -= allocated;
1517            reservation.targets.push(DecommissionCapacityTarget {
1518                pool_index: target.pool_index,
1519                layout: target.layout,
1520                physical_total_at_reservation: target.physical_total,
1521                physical_free_at_reservation: target.physical_free,
1522                reserved_physical_bytes: allocated,
1523                consumed_physical_bytes: 0,
1524                observed_physical_bytes: 0,
1525                inflight_physical_bytes: 0,
1526                pending_physical_bytes: 0,
1527                pending_mutation_id: None,
1528                temporary_mutations: Vec::new(),
1529            });
1530        }
1531        debug_assert_eq!(remaining, 0);
1532    }
1533
1534    for (source_index, reservation, _) in reservations {
1535        metrics::gauge!(
1536            METRIC_DECOMMISSION_CAPACITY_PREDICTED_BYTES,
1537            "pool_index" => source_index.to_string()
1538        )
1539        .set(reservation.predicted_physical_bytes as f64);
1540        metrics::gauge!(
1541            METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES,
1542            "pool_index" => source_index.to_string()
1543        )
1544        .set(reservation.remaining_peak_physical_bytes() as f64);
1545        let pool_count = meta.pools.len();
1546        let info = meta
1547            .pools
1548            .get_mut(source_index)
1549            .and_then(|pool| pool.decommission.as_mut())
1550            .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_index))?;
1551        info.capacity_blocked_reason = None;
1552        info.capacity_reservation = Some(reservation);
1553    }
1554    if meta.version != POOL_META_GENERATION_VERSION {
1555        meta.version = POOL_META_VERSION;
1556    }
1557    validate_decommission_capacity_model_cohort(meta)?;
1558    Ok(())
1559}
1560
1561fn ensure_decommission_start_target_capacity(
1562    meta: &PoolMeta,
1563    indices: &[usize],
1564    capacity_infos: &[DecommissionPoolCapacityInfo],
1565    target_fence_proof_available: bool,
1566) -> Result<()> {
1567    let generation = next_decommission_capacity_generation(meta)?;
1568    let mut projected = meta.clone();
1569    let first_idx = indices.first().copied();
1570    for idx in indices.iter().copied() {
1571        let capacity = capacity_infos
1572            .iter()
1573            .find(|capacity| capacity.pool_index == idx)
1574            .ok_or_else(|| Error::DecommissionCapacity(format!("decommission capacity snapshot is missing pool {idx}")))?;
1575        if Some(idx) == first_idx {
1576            projected.decommission(idx, capacity.space)?;
1577        } else {
1578            projected.queue_decommission(idx, capacity.space)?;
1579        }
1580    }
1581    let model_version = select_decommission_capacity_model(&projected, target_fence_proof_available)?;
1582    reserve_decommission_start_target_capacity(
1583        &mut projected,
1584        indices,
1585        capacity_infos,
1586        uuid::Uuid::new_v4(),
1587        generation,
1588        OffsetDateTime::now_utc(),
1589        model_version,
1590    )
1591}
1592
1593fn recover_decommission_capacity_reservations(
1594    meta: &mut PoolMeta,
1595    capacity_infos: &[DecommissionPoolCapacityInfo],
1596    now: OffsetDateTime,
1597    target_fence_proof_available: bool,
1598) -> Result<Vec<usize>> {
1599    ensure_decommission_capacity_writer_supported(meta)?;
1600    let mut active_indices = active_decommission_source_indices(meta).into_iter().collect::<Vec<_>>();
1601    active_indices.sort_unstable();
1602    let missing_indices = active_indices
1603        .iter()
1604        .copied()
1605        .filter(|idx| {
1606            meta.pools
1607                .get(*idx)
1608                .and_then(|pool| pool.decommission.as_ref())
1609                .and_then(|info| info.capacity_reservation.as_ref())
1610                .is_none_or(|reservation| !reservation.active())
1611        })
1612        .collect::<Vec<_>>();
1613    if missing_indices.is_empty() {
1614        validate_decommission_capacity_model_cohort(meta)?;
1615        return Ok(active_indices);
1616    }
1617    let generation = next_decommission_capacity_generation(meta)?;
1618    let model_version = select_decommission_capacity_model(meta, target_fence_proof_available)?;
1619    reserve_decommission_start_target_capacity(
1620        meta,
1621        &missing_indices,
1622        capacity_infos,
1623        uuid::Uuid::new_v4(),
1624        generation,
1625        now,
1626        model_version,
1627    )?;
1628    for idx in missing_indices {
1629        if let Some(reservation) = meta
1630            .pools
1631            .get_mut(idx)
1632            .and_then(|pool| pool.decommission.as_mut())
1633            .and_then(|info| info.capacity_reservation.as_mut())
1634        {
1635            reservation.recovered_at = Some(now);
1636        }
1637    }
1638    Ok(active_indices)
1639}
1640
1641fn signed_capacity_difference(observed: usize, predicted: usize) -> i64 {
1642    if observed >= predicted {
1643        i64::try_from(observed.saturating_sub(predicted)).unwrap_or(i64::MAX)
1644    } else {
1645        -i64::try_from(predicted.saturating_sub(observed)).unwrap_or(i64::MAX)
1646    }
1647}
1648
1649fn active_decommission_target_reservations(meta: &PoolMeta) -> HashMap<usize, usize> {
1650    let mut target_reservations = HashMap::new();
1651    for reservation in meta
1652        .pools
1653        .iter()
1654        .filter_map(|pool| pool.decommission.as_ref()?.capacity_reservation.as_ref())
1655        .filter(|reservation| reservation.active())
1656    {
1657        for target in &reservation.targets {
1658            let reserved = target_reservations.entry(target.pool_index).or_insert(0usize);
1659            *reserved = reserved.saturating_add(target.remaining_reserved_physical_bytes(reservation.temporary_copies));
1660        }
1661    }
1662    target_reservations
1663}
1664
1665fn observe_decommission_capacity_reservation(
1666    info: &mut PoolDecommissionInfo,
1667    _capacity_infos: &[DecommissionPoolCapacityInfo],
1668    _active_target_reservations: &HashMap<usize, usize>,
1669) {
1670    let Some(reservation) = info.capacity_reservation.as_mut() else {
1671        return;
1672    };
1673    reservation.prediction_error_bytes =
1674        signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes);
1675    metrics::gauge!(
1676        METRIC_DECOMMISSION_CAPACITY_PREDICTION_ERROR_BYTES,
1677        "pool_index" => reservation.source_pool_index.to_string()
1678    )
1679    .set(reservation.prediction_error_bytes as f64);
1680    metrics::histogram!(
1681        METRIC_DECOMMISSION_CAPACITY_PREDICTION_ABSOLUTE_ERROR_BYTES,
1682        "pool_index" => reservation.source_pool_index.to_string()
1683    )
1684    .record(reservation.prediction_error_bytes.unsigned_abs() as f64);
1685}
1686
1687fn ensure_decommission_capacity_reservations_available(
1688    meta: &PoolMeta,
1689    capacity_infos: &[DecommissionPoolCapacityInfo],
1690    phase: &'static str,
1691) -> Result<()> {
1692    let mut required_by_target = HashMap::<usize, usize>::new();
1693    let mut inflight_by_target = HashMap::<usize, usize>::new();
1694    let mut pending_by_target = HashMap::<usize, usize>::new();
1695    for source_index in active_decommission_source_indices(meta) {
1696        let info = meta.pools[source_index]
1697            .decommission
1698            .as_ref()
1699            .ok_or_else(|| decommission_metadata_not_initialized_error("check decommission capacity reservation"))?;
1700        let reservation = info.capacity_reservation.as_ref().ok_or_else(|| {
1701            decommission_capacity_blocked_error(format!(
1702                "active decommission source pool {source_index} has no durable capacity reservation"
1703            ))
1704        })?;
1705        if !reservation.active() {
1706            return Err(decommission_capacity_blocked_error(format!(
1707                "active decommission source pool {source_index} has a released capacity reservation"
1708            )));
1709        }
1710        metrics::gauge!(
1711            METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES,
1712            "pool_index" => source_index.to_string()
1713        )
1714        .set(reservation.remaining_peak_physical_bytes() as f64);
1715        for target in &reservation.targets {
1716            let temporary_budget = target.remaining_reserved_physical_bytes(reservation.temporary_copies)
1717                / 1usize.saturating_add(reservation.temporary_copies);
1718            if target.inflight_physical_bytes > temporary_budget {
1719                return Err(decommission_capacity_blocked_error(format!(
1720                    "source pool {source_index} target pool {} has {} inflight physical bytes, exceeding its {temporary_budget}-byte temporary-copy reservation during {phase}",
1721                    target.pool_index, target.inflight_physical_bytes
1722                )));
1723            }
1724            if target.pending_physical_bytes > temporary_budget {
1725                return Err(decommission_capacity_blocked_error(format!(
1726                    "source pool {source_index} target pool {} has {} pending physical bytes, exceeding its {temporary_budget}-byte committed-copy reservation during {phase}",
1727                    target.pool_index, target.pending_physical_bytes
1728                )));
1729            }
1730            let required = required_by_target.entry(target.pool_index).or_default();
1731            *required = required.saturating_add(target.remaining_reserved_physical_bytes(reservation.temporary_copies));
1732            let inflight = inflight_by_target.entry(target.pool_index).or_default();
1733            *inflight = inflight.saturating_add(target.inflight_physical_bytes);
1734            let pending = pending_by_target.entry(target.pool_index).or_default();
1735            *pending = pending.saturating_add(target.pending_physical_bytes);
1736        }
1737    }
1738    for (target_pool_index, required) in required_by_target {
1739        let available = capacity_infos
1740            .iter()
1741            .find(|capacity| capacity.pool_index == target_pool_index)
1742            .map(|capacity| capacity.physical_free)
1743            .ok_or_else(|| {
1744                decommission_capacity_blocked_error(format!(
1745                    "capacity snapshot is missing target pool {target_pool_index} during {phase}"
1746                ))
1747            })?
1748            .saturating_add(inflight_by_target.get(&target_pool_index).copied().unwrap_or_default())
1749            .saturating_add(pending_by_target.get(&target_pool_index).copied().unwrap_or_default());
1750        if available >= required {
1751            continue;
1752        }
1753        metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => phase).increment(1);
1754        return Err(decommission_capacity_blocked_error(format!(
1755            "target pool {target_pool_index} reservation requires {required} physical bytes, but only {available} bytes remain during {phase}"
1756        )));
1757    }
1758    Ok(())
1759}
1760
1761#[derive(Clone, Copy)]
1762pub(crate) enum DecommissionCapacityAdmission {
1763    Mutation,
1764    ExistingMultipart,
1765    ScannerBacklog,
1766    BatchDelete,
1767    Heal,
1768}
1769
1770impl DecommissionCapacityAdmission {
1771    fn phase(self) -> &'static str {
1772        match self {
1773            Self::Mutation => "mutation",
1774            Self::ExistingMultipart => "existing_multipart",
1775            Self::ScannerBacklog => "scanner_backlog",
1776            Self::BatchDelete => "batch_delete",
1777            Self::Heal => "heal",
1778        }
1779    }
1780}
1781
1782fn ensure_external_decommission_target_admission(
1783    meta: &PoolMeta,
1784    target_pool_index: usize,
1785    admission: DecommissionCapacityAdmission,
1786) -> Result<()> {
1787    let phase = admission.phase();
1788    // Pool selection may predate retirement or use a stale node-local snapshot.
1789    // Recheck publication against the fenced durable state. Repair and pure
1790    // capacity release retain their separate admission contracts.
1791    if matches!(admission, DecommissionCapacityAdmission::ScannerBacklog)
1792        && !meta.scanner_pause_backlog_pool_writable(target_pool_index)
1793    {
1794        return Err(Error::SlowDown);
1795    }
1796    let active_sources = active_decommission_source_indices(meta);
1797    if meta.is_suspended(target_pool_index) {
1798        let active_source = active_sources.contains(&target_pool_index);
1799        if !matches!(
1800            admission,
1801            DecommissionCapacityAdmission::Heal | DecommissionCapacityAdmission::ScannerBacklog
1802        ) && !(matches!(admission, DecommissionCapacityAdmission::ExistingMultipart) && active_source)
1803        {
1804            return Err(Error::SlowDown);
1805        }
1806    }
1807    if active_sources.into_iter().any(|source_pool_index| {
1808        meta.pools
1809            .get(source_pool_index)
1810            .and_then(|pool| pool.decommission.as_ref())
1811            .and_then(|info| info.capacity_reservation.as_ref())
1812            .is_none_or(|reservation| !reservation.active())
1813    }) {
1814        metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => phase).increment(1);
1815        return Err(Error::SlowDown);
1816    }
1817    // Migration reservations budget the mover, not exclusive ownership of a
1818    // healthy pool. Foreground publication shares its actual disk capacity;
1819    // migration must retain the source if its capacity or target write fails.
1820    // Repair keeps its separate, conservative reservation admission contract.
1821    if !matches!(admission, DecommissionCapacityAdmission::Heal) {
1822        return Ok(());
1823    }
1824    let reserved = active_decommission_target_reservations(meta)
1825        .get(&target_pool_index)
1826        .copied()
1827        .unwrap_or_default();
1828    if reserved == 0 {
1829        return Ok(());
1830    }
1831    metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => phase).increment(1);
1832    Err(Error::SlowDown)
1833}
1834
1835fn ensure_decommission_target_owner_admission(
1836    meta: &PoolMeta,
1837    owner: DecommissionCapacityOwner,
1838    target_pool_index: usize,
1839    target_physical_bytes: usize,
1840    now: OffsetDateTime,
1841) -> Result<()> {
1842    let reservation = meta
1843        .pools
1844        .get(owner.source_pool_index)
1845        .and_then(|pool| pool.decommission.as_ref())
1846        .and_then(|info| info.capacity_reservation.as_ref())
1847        .filter(|reservation| reservation.admits_owner(owner, now))
1848        .ok_or_else(|| Error::SlowDown)?;
1849    let target = reservation
1850        .targets
1851        .iter()
1852        .find(|target| target.pool_index == target_pool_index)
1853        .ok_or_else(|| Error::SlowDown)?;
1854    let required_peak = target_physical_bytes.saturating_mul(1usize.saturating_add(reservation.temporary_copies));
1855    let remaining = target.remaining_reserved_physical_bytes(reservation.temporary_copies);
1856    if required_peak <= remaining {
1857        return Ok(());
1858    }
1859    Err(decommission_capacity_blocked_error(format!(
1860        "source pool {} target pool {target_pool_index} operation requires {required_peak} physical bytes, but only {remaining} reserved bytes remain",
1861        owner.source_pool_index
1862    )))
1863}
1864
1865fn reserve_decommission_target_pending(
1866    meta: &mut PoolMeta,
1867    source_pool_index: usize,
1868    target_pool_index: usize,
1869    requested_physical_bytes: usize,
1870    mutation_id: uuid::Uuid,
1871    now: OffsetDateTime,
1872) -> Result<usize> {
1873    if requested_physical_bytes == 0 {
1874        return Ok(0);
1875    }
1876    let pool_count = meta.pools.len();
1877    let pool = meta
1878        .pools
1879        .get_mut(source_pool_index)
1880        .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
1881    let info = pool
1882        .decommission
1883        .as_mut()
1884        .ok_or_else(|| decommission_metadata_not_initialized_error("reserve target capacity mutation"))?;
1885    let reservation = info
1886        .capacity_reservation
1887        .as_mut()
1888        .filter(|reservation| reservation.active())
1889        .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared before target write"))?;
1890    let target = reservation
1891        .targets
1892        .iter_mut()
1893        .find(|target| target.pool_index == target_pool_index)
1894        .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared before target write"))?;
1895    if target.pending_physical_bytes == 0 {
1896        target.pending_mutation_id = None;
1897    }
1898    if target.pending_physical_bytes > 0 {
1899        match target.pending_mutation_id {
1900            Some(pending_mutation_id) if pending_mutation_id == mutation_id => {
1901                if requested_physical_bytes <= target.pending_physical_bytes {
1902                    return Ok(0);
1903                }
1904                let budget = target.remaining_reserved_physical_bytes(reservation.temporary_copies)
1905                    / 1usize.saturating_add(reservation.temporary_copies);
1906                if requested_physical_bytes > budget {
1907                    return Err(decommission_capacity_blocked_error(format!(
1908                        "source pool {source_pool_index} target pool {target_pool_index} mutation intent size changed during retry"
1909                    )));
1910                }
1911                let additional = requested_physical_bytes.saturating_sub(target.pending_physical_bytes);
1912                target.pending_physical_bytes = requested_physical_bytes;
1913                reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_add(additional);
1914                renew_decommission_capacity_reservation(reservation, now, true);
1915                pool.last_update = pool.last_update.max(now);
1916                return Ok(additional);
1917            }
1918            pending_mutation_id => {
1919                return Err(decommission_capacity_blocked_error(format!(
1920                    "source pool {source_pool_index} target pool {target_pool_index} has an unresolved target capacity intent {pending_mutation_id:?} while mutation {mutation_id} is waiting"
1921                )));
1922            }
1923        }
1924    }
1925    let budget = target.remaining_reserved_physical_bytes(reservation.temporary_copies)
1926        / 1usize.saturating_add(reservation.temporary_copies);
1927    if requested_physical_bytes > budget {
1928        return Err(decommission_capacity_blocked_error(format!(
1929            "source pool {source_pool_index} target pool {target_pool_index} mutation requires {requested_physical_bytes} physical bytes, but only {budget} committed-copy bytes remain"
1930        )));
1931    }
1932    let additional = requested_physical_bytes;
1933    target.pending_mutation_id = Some(mutation_id);
1934    target.pending_physical_bytes = target.pending_physical_bytes.saturating_add(additional);
1935    reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_add(additional);
1936    renew_decommission_capacity_reservation(reservation, now, true);
1937    pool.last_update = pool.last_update.max(now);
1938    Ok(additional)
1939}
1940
1941fn resolve_decommission_target_pending(
1942    meta: &mut PoolMeta,
1943    source_pool_index: usize,
1944    target_pool_index: usize,
1945    resolved_physical_bytes: usize,
1946    mutation_id: uuid::Uuid,
1947) -> Result<()> {
1948    if resolved_physical_bytes == 0 {
1949        return Ok(());
1950    }
1951    let pool_count = meta.pools.len();
1952    let pool = meta
1953        .pools
1954        .get_mut(source_pool_index)
1955        .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
1956    let reservation = pool
1957        .decommission
1958        .as_mut()
1959        .and_then(|info| info.capacity_reservation.as_mut())
1960        .filter(|reservation| reservation.active())
1961        .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared after target write"))?;
1962    let target = reservation
1963        .targets
1964        .iter_mut()
1965        .find(|target| target.pool_index == target_pool_index)
1966        .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared after target write"))?;
1967    if target.pending_physical_bytes > 0 && target.pending_mutation_id != Some(mutation_id) {
1968        return Err(decommission_capacity_blocked_error(format!(
1969            "source pool {source_pool_index} target pool {target_pool_index} pending capacity intent belongs to another mutation"
1970        )));
1971    }
1972    let resolved = target.pending_physical_bytes.min(resolved_physical_bytes);
1973    target.pending_physical_bytes = target.pending_physical_bytes.saturating_sub(resolved);
1974    reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_sub(resolved);
1975    if target.pending_physical_bytes == 0 {
1976        target.pending_mutation_id = None;
1977    }
1978    Ok(())
1979}
1980
1981fn release_decommission_target_temporary_mutation(
1982    target: &mut DecommissionCapacityTarget,
1983    mutation_id: uuid::Uuid,
1984    maximum_physical_bytes: usize,
1985) -> (usize, bool) {
1986    let Some(index) = target
1987        .temporary_mutations
1988        .iter()
1989        .position(|mutation| mutation.mutation_id == mutation_id)
1990    else {
1991        return (0, false);
1992    };
1993    let released = target.temporary_mutations[index].physical_bytes.min(maximum_physical_bytes);
1994    target.temporary_mutations[index].physical_bytes = target.temporary_mutations[index].physical_bytes.saturating_sub(released);
1995    let removed = target.temporary_mutations[index].physical_bytes == 0;
1996    if removed {
1997        target.temporary_mutations.remove(index);
1998    }
1999    target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released);
2000    (released, released > 0 || removed)
2001}
2002
2003fn settle_decommission_target_non_growing_replacement(
2004    meta: &mut PoolMeta,
2005    source_pool_index: usize,
2006    target_pool_index: usize,
2007    mutation_id: uuid::Uuid,
2008    now: OffsetDateTime,
2009) -> Result<bool> {
2010    let pool_count = meta.pools.len();
2011    let pool = meta
2012        .pools
2013        .get_mut(source_pool_index)
2014        .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2015    let info = pool
2016        .decommission
2017        .as_mut()
2018        .ok_or_else(|| decommission_metadata_not_initialized_error("settle non-growing target replacement"))?;
2019    let reservation = info
2020        .capacity_reservation
2021        .as_mut()
2022        .filter(|reservation| reservation.active())
2023        .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target replacement"))?;
2024    let target = reservation
2025        .targets
2026        .iter_mut()
2027        .find(|target| target.pool_index == target_pool_index)
2028        .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target replacement"))?;
2029    let (released, changed) = release_decommission_target_temporary_mutation(target, mutation_id, usize::MAX);
2030    reservation.inflight_target_physical_bytes = reservation.inflight_target_physical_bytes.saturating_sub(released);
2031    if changed {
2032        renew_decommission_capacity_reservation(reservation, now, true);
2033        pool.last_update = pool.last_update.max(now);
2034    }
2035    Ok(changed)
2036}
2037
2038struct DecommissionTargetConsumption {
2039    committed_data_bytes: usize,
2040    target_physical_bytes: usize,
2041    observed_physical_bytes: usize,
2042}
2043
2044fn record_decommission_target_consumption(
2045    meta: &mut PoolMeta,
2046    source_pool_index: usize,
2047    target_pool_index: usize,
2048    consumption: DecommissionTargetConsumption,
2049    mutation_id: uuid::Uuid,
2050    now: OffsetDateTime,
2051) -> Result<()> {
2052    let DecommissionTargetConsumption {
2053        committed_data_bytes,
2054        target_physical_bytes,
2055        observed_physical_bytes,
2056    } = consumption;
2057    let pool_count = meta.pools.len();
2058    let pool = meta
2059        .pools
2060        .get_mut(source_pool_index)
2061        .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2062    let info = pool
2063        .decommission
2064        .as_mut()
2065        .ok_or_else(|| decommission_metadata_not_initialized_error("record target capacity consumption"))?;
2066    let reservation = info
2067        .capacity_reservation
2068        .as_mut()
2069        .filter(|reservation| reservation.active())
2070        .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target write"))?;
2071    let target = reservation
2072        .targets
2073        .iter_mut()
2074        .find(|target| target.pool_index == target_pool_index)
2075        .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target write"))?;
2076    let remaining_target_bytes = target.remaining_reserved_physical_bytes(reservation.temporary_copies)
2077        / 1usize.saturating_add(reservation.temporary_copies);
2078    let remaining_total_bytes = reservation
2079        .predicted_physical_bytes
2080        .saturating_sub(reservation.consumed_target_physical_bytes);
2081    let consumed = target_physical_bytes.min(remaining_target_bytes).min(remaining_total_bytes);
2082    target.consumed_physical_bytes = target.consumed_physical_bytes.saturating_add(consumed);
2083    target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes);
2084    let has_scoped_temporary_mutations = !target.temporary_mutations.is_empty();
2085    let (released_inflight, _) = release_decommission_target_temporary_mutation(target, mutation_id, usize::MAX);
2086    let released_inflight = if released_inflight == 0 && !has_scoped_temporary_mutations {
2087        let released = target.inflight_physical_bytes.min(consumed);
2088        target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released);
2089        released
2090    } else {
2091        released_inflight
2092    };
2093    reservation.consumed_target_physical_bytes = reservation.consumed_target_physical_bytes.saturating_add(consumed);
2094    let committed = committed_data_bytes.min(
2095        reservation
2096            .source_data_equivalent_bytes
2097            .saturating_sub(reservation.committed_data_bytes),
2098    );
2099    reservation.committed_data_bytes = reservation.committed_data_bytes.saturating_add(committed);
2100    reservation.observed_target_physical_bytes = reservation
2101        .observed_target_physical_bytes
2102        .saturating_add(observed_physical_bytes);
2103    reservation.inflight_target_physical_bytes = reservation.inflight_target_physical_bytes.saturating_sub(released_inflight);
2104    reservation.prediction_error_bytes =
2105        signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes);
2106    renew_decommission_capacity_reservation(reservation, now, true);
2107    info.capacity_blocked_reason = None;
2108    pool.last_update = pool.last_update.max(now);
2109    Ok(())
2110}
2111
2112fn record_decommission_target_inflight(
2113    meta: &mut PoolMeta,
2114    source_pool_index: usize,
2115    target_pool_index: usize,
2116    observed_physical_bytes: usize,
2117    mutation_id: uuid::Uuid,
2118    now: OffsetDateTime,
2119) -> Result<bool> {
2120    let pool_count = meta.pools.len();
2121    let pool = meta
2122        .pools
2123        .get_mut(source_pool_index)
2124        .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2125    let info = pool
2126        .decommission
2127        .as_mut()
2128        .ok_or_else(|| decommission_metadata_not_initialized_error("record target capacity inflight bytes"))?;
2129    let reservation = info
2130        .capacity_reservation
2131        .as_mut()
2132        .filter(|reservation| reservation.active())
2133        .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target write"))?;
2134    let target = reservation
2135        .targets
2136        .iter_mut()
2137        .find(|target| target.pool_index == target_pool_index)
2138        .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target write"))?;
2139    let ledger_changed = if let Some(mutation) = target
2140        .temporary_mutations
2141        .iter_mut()
2142        .find(|mutation| mutation.mutation_id == mutation_id)
2143    {
2144        mutation.physical_bytes = mutation.physical_bytes.saturating_add(observed_physical_bytes);
2145        observed_physical_bytes > 0
2146    } else if observed_physical_bytes > 0 || reservation.model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION {
2147        // A v2 zero-byte record is a durable discovery marker. It keeps
2148        // restart cleanup scoped to mutations that actually staged an MPU
2149        // without charging capacity when statfs observed no physical delta.
2150        target.temporary_mutations.push(DecommissionCapacityTemporaryMutation {
2151            mutation_id,
2152            physical_bytes: observed_physical_bytes,
2153        });
2154        true
2155    } else {
2156        false
2157    };
2158    target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes);
2159    target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_add(observed_physical_bytes);
2160    reservation.observed_target_physical_bytes = reservation
2161        .observed_target_physical_bytes
2162        .saturating_add(observed_physical_bytes);
2163    reservation.inflight_target_physical_bytes = reservation
2164        .inflight_target_physical_bytes
2165        .saturating_add(observed_physical_bytes);
2166    reservation.prediction_error_bytes =
2167        signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes);
2168    renew_decommission_capacity_reservation(reservation, now, true);
2169    pool.last_update = pool.last_update.max(now);
2170    Ok(ledger_changed)
2171}
2172
2173fn record_decommission_target_observation(
2174    meta: &mut PoolMeta,
2175    source_pool_index: usize,
2176    target_pool_index: usize,
2177    observed_physical_bytes: usize,
2178    now: OffsetDateTime,
2179) -> Result<()> {
2180    let pool_count = meta.pools.len();
2181    let pool = meta
2182        .pools
2183        .get_mut(source_pool_index)
2184        .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2185    let info = pool
2186        .decommission
2187        .as_mut()
2188        .ok_or_else(|| decommission_metadata_not_initialized_error("record target capacity observation"))?;
2189    let reservation = info
2190        .capacity_reservation
2191        .as_mut()
2192        .filter(|reservation| reservation.active())
2193        .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target write"))?;
2194    let target = reservation
2195        .targets
2196        .iter_mut()
2197        .find(|target| target.pool_index == target_pool_index)
2198        .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target write"))?;
2199    target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes);
2200    reservation.observed_target_physical_bytes = reservation
2201        .observed_target_physical_bytes
2202        .saturating_add(observed_physical_bytes);
2203    reservation.prediction_error_bytes =
2204        signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes);
2205    renew_decommission_capacity_reservation(reservation, now, true);
2206    pool.last_update = pool.last_update.max(now);
2207    Ok(())
2208}
2209
2210fn release_decommission_target_inflight(
2211    meta: &mut PoolMeta,
2212    source_pool_index: usize,
2213    target_pool_index: usize,
2214    released_physical_bytes: usize,
2215    mutation_id: uuid::Uuid,
2216    proof: DecommissionCapacityReleaseProof,
2217    now: OffsetDateTime,
2218) -> Result<bool> {
2219    let DecommissionCapacityReleaseProof {
2220        confirmed_absent,
2221        clear_pending,
2222    } = proof;
2223    let pool_count = meta.pools.len();
2224    let pool = meta
2225        .pools
2226        .get_mut(source_pool_index)
2227        .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2228    let info = pool
2229        .decommission
2230        .as_mut()
2231        .ok_or_else(|| decommission_metadata_not_initialized_error("release target capacity inflight bytes"))?;
2232    let reservation = info
2233        .capacity_reservation
2234        .as_mut()
2235        .filter(|reservation| reservation.active())
2236        .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target cleanup"))?;
2237    let model_version = reservation.model_version;
2238    let target = reservation
2239        .targets
2240        .iter_mut()
2241        .find(|target| target.pool_index == target_pool_index)
2242        .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target cleanup"))?;
2243    let has_temporary_mutations = !target.temporary_mutations.is_empty();
2244    let pending_belongs_to_mutation = target.pending_mutation_id == Some(mutation_id);
2245    let zero_ledger_cleanup = confirmed_absent
2246        && model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
2247        && !has_temporary_mutations
2248        && target.pending_mutation_id.is_none()
2249        && target.pending_physical_bytes == 0;
2250    let (released, temporary_mutation_changed) = release_decommission_target_temporary_mutation(
2251        target,
2252        mutation_id,
2253        if confirmed_absent {
2254            usize::MAX
2255        } else {
2256            released_physical_bytes
2257        },
2258    );
2259    let released = if released == 0 && !has_temporary_mutations && model_version == DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION {
2260        let released = target.inflight_physical_bytes.min(released_physical_bytes);
2261        target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released);
2262        released
2263    } else {
2264        released
2265    };
2266    let pending_matches = clear_pending && confirmed_absent && pending_belongs_to_mutation;
2267    let published_pending_delayed_release = confirmed_absent
2268        && model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
2269        && !clear_pending
2270        && !has_temporary_mutations
2271        && pending_belongs_to_mutation;
2272    if released == 0
2273        && released_physical_bytes > 0
2274        && !zero_ledger_cleanup
2275        && !published_pending_delayed_release
2276        && (has_temporary_mutations || !pending_matches)
2277    {
2278        return Err(decommission_capacity_blocked_error(
2279            "temporary target cleanup released bytes that cannot be attributed to its mutation",
2280        ));
2281    }
2282    reservation.inflight_target_physical_bytes = reservation.inflight_target_physical_bytes.saturating_sub(released);
2283    let cleared_pending = if pending_matches {
2284        let cleared = target.pending_physical_bytes;
2285        target.pending_physical_bytes = 0;
2286        target.pending_mutation_id = None;
2287        reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_sub(cleared);
2288        cleared
2289    } else {
2290        0
2291    };
2292    let changed = released > 0 || temporary_mutation_changed || cleared_pending > 0;
2293    if changed {
2294        renew_decommission_capacity_reservation(reservation, now, true);
2295        pool.last_update = pool.last_update.max(now);
2296    }
2297    Ok(changed)
2298}
2299
2300fn ensure_valid_decommission_pool_index(pool_count: usize, idx: usize) -> Result<()> {
2301    if idx >= pool_count {
2302        return Err(invalid_decommission_pool_index_error(pool_count, idx));
2303    }
2304
2305    Ok(())
2306}
2307
2308fn get_by_index<'a, T>(items: &'a [T], idx: usize, operation: &'static str) -> Result<&'a T> {
2309    items.get(idx).ok_or_else(|| {
2310        Error::other(format!(
2311            "failed to {operation}: invalid decommission pool index {idx} for {pool_count} pools",
2312            pool_count = items.len()
2313        ))
2314    })
2315}
2316
2317fn decommission_metadata_not_initialized_error(operation: &str) -> Error {
2318    Error::other(format!("failed to {operation}: decommission metadata not initialized"))
2319}
2320
2321fn resolve_decommission_bucket_state(meta: &PoolMeta, idx: usize, bucket: &DecomBucketInfo) -> Result<bool> {
2322    let pool_count = meta.pools.len();
2323    ensure_valid_decommission_pool_index(pool_count, idx)?;
2324
2325    let Some(pool) = meta.pools.get(idx) else {
2326        return Err(invalid_decommission_pool_index_error(pool_count, idx));
2327    };
2328    let Some(info) = pool.decommission.as_ref() else {
2329        return Err(decommission_metadata_not_initialized_error("resolve decommission bucket state"));
2330    };
2331
2332    Ok(info.is_bucket_decommissioned(&bucket.to_string()))
2333}
2334
2335fn mark_decommission_bucket_done(meta: &mut PoolMeta, idx: usize, bucket: &DecomBucketInfo) -> Result<bool> {
2336    let pool_count = meta.pools.len();
2337    ensure_valid_decommission_pool_index(pool_count, idx)?;
2338
2339    let Some(pool) = meta.pools.get_mut(idx) else {
2340        return Err(invalid_decommission_pool_index_error(pool_count, idx));
2341    };
2342    let Some(info) = pool.decommission.as_mut() else {
2343        return Err(decommission_metadata_not_initialized_error("mark decommission bucket done"));
2344    };
2345
2346    Ok(info.bucket_pop(&bucket.to_string()))
2347}
2348
2349fn count_decommission_item(meta: &mut PoolMeta, idx: usize, size: usize, failed: bool) -> Result<()> {
2350    let pool_count = meta.pools.len();
2351    ensure_valid_decommission_pool_index(pool_count, idx)?;
2352
2353    let Some(pool) = meta.pools.get_mut(idx) else {
2354        return Err(invalid_decommission_pool_index_error(pool_count, idx));
2355    };
2356    let Some(info) = pool.decommission.as_mut() else {
2357        return Err(decommission_metadata_not_initialized_error("count decommission item"));
2358    };
2359
2360    if failed {
2361        info.items_decommission_failed += 1;
2362        info.bytes_failed += size;
2363    } else {
2364        info.items_decommissioned += 1;
2365        info.bytes_done += size;
2366    }
2367
2368    Ok(())
2369}
2370
2371fn ensure_decommission_generation(meta: &PoolMeta, idx: usize, generation: OffsetDateTime) -> Result<()> {
2372    let Some(pool) = meta.pools.get(idx) else {
2373        return Err(invalid_decommission_pool_index_error(meta.pools.len(), idx));
2374    };
2375    let Some(info) = pool.decommission.as_ref() else {
2376        return Err(decommission_metadata_not_initialized_error("check decommission generation"));
2377    };
2378
2379    if info.start_time == Some(generation) && !info.queued && is_decommission_active(info.complete, info.failed, info.canceled) {
2380        Ok(())
2381    } else {
2382        Err(Error::OperationCanceled)
2383    }
2384}
2385
2386fn record_decommission_unresolved_entry(
2387    meta: &mut PoolMeta,
2388    idx: usize,
2389    generation: OffsetDateTime,
2390    entry: DecommissionUnresolvedEntry,
2391    now: OffsetDateTime,
2392    rebalance_meta: Option<&RebalanceMeta>,
2393) -> Result<bool> {
2394    ensure_decommission_generation(meta, idx, generation)?;
2395    if entry.pool_index != idx || entry.source_generation != generation {
2396        return Err(Error::other("decommission unresolved entry does not match the active pool generation"));
2397    }
2398
2399    let pool_count = meta.pools.len();
2400    let Some(pool) = meta.pools.get(idx) else {
2401        return Err(invalid_decommission_pool_index_error(pool_count, idx));
2402    };
2403    let Some(info) = pool.decommission.as_ref() else {
2404        return Err(decommission_metadata_not_initialized_error("record decommission unresolved entry"));
2405    };
2406    let existing_index = info.unresolved_entries.iter().position(|existing| {
2407        existing.bucket == entry.bucket
2408            && existing.object == entry.object
2409            && existing.pool_index == entry.pool_index
2410            && existing.set_index == entry.set_index
2411            && existing.source_generation == entry.source_generation
2412    });
2413    if existing_index.is_some_and(|index| info.unresolved_entries[index] == entry) {
2414        return Ok(false);
2415    }
2416
2417    let last_update = meta.next_scanner_data_movement_update(now, rebalance_meta);
2418    let Some(pool) = meta.pools.get_mut(idx) else {
2419        return Err(invalid_decommission_pool_index_error(pool_count, idx));
2420    };
2421    let Some(info) = pool.decommission.as_mut() else {
2422        return Err(decommission_metadata_not_initialized_error("record decommission unresolved entry"));
2423    };
2424    if let Some(index) = existing_index {
2425        info.unresolved_entries[index] = entry;
2426    } else {
2427        info.unresolved_entries.push(entry);
2428    }
2429    pool.last_update = last_update;
2430    Ok(true)
2431}
2432
2433type DecommissionUnresolvedEntryIdentity = (usize, String, String);
2434
2435fn decommission_unresolved_entry_identity(entry: &DecommissionUnresolvedEntry) -> DecommissionUnresolvedEntryIdentity {
2436    (entry.set_index, entry.bucket.clone(), entry.object.clone())
2437}
2438
2439fn reconcile_decommission_unresolved_entries_for_completion(
2440    meta: &mut PoolMeta,
2441    idx: usize,
2442    verified_generation: Option<OffsetDateTime>,
2443    verified_entries: Option<&[DecommissionUnresolvedEntry]>,
2444) -> Result<()> {
2445    let pool_count = meta.pools.len();
2446    let Some(pool) = meta.pools.get(idx) else {
2447        return Err(invalid_decommission_pool_index_error(pool_count, idx));
2448    };
2449    let Some(info) = pool.decommission.as_ref() else {
2450        return Err(decommission_metadata_not_initialized_error("reconcile decommission unresolved entries"));
2451    };
2452    if info.unresolved_entries.is_empty() {
2453        return Ok(());
2454    }
2455
2456    let Some(generation) = verified_generation else {
2457        return Err(Error::other(format!(
2458            "failed to complete decommission for pool {idx}: {} unresolved listing entries remain",
2459            info.unresolved_entries.len()
2460        )));
2461    };
2462    ensure_decommission_generation(meta, idx, generation)?;
2463    if info
2464        .unresolved_entries
2465        .iter()
2466        .any(|entry| entry.source_generation != generation)
2467    {
2468        return Err(Error::other(format!(
2469            "failed to complete decommission for pool {idx}: unresolved listing ledger contains a different generation"
2470        )));
2471    }
2472
2473    let verified_entries = verified_entries.unwrap_or_default();
2474    let unverified_count = info
2475        .unresolved_entries
2476        .iter()
2477        .filter(|entry| !verified_entries.contains(entry))
2478        .count();
2479    if unverified_count > 0 {
2480        return Err(Error::other(format!(
2481            "failed to complete decommission for pool {idx}: {unverified_count} unresolved listing entries were not individually verified"
2482        )));
2483    }
2484
2485    let Some(info) = meta.pools.get_mut(idx).and_then(|pool| pool.decommission.as_mut()) else {
2486        return Err(decommission_metadata_not_initialized_error("reconcile decommission unresolved entries"));
2487    };
2488    info.unresolved_entries.clear();
2489    Ok(())
2490}
2491
2492#[cfg(test)]
2493async fn run_decommission_side_effect<T, E, F, Fut>(
2494    rx: &CancellationToken,
2495    operation_gate: &Arc<tokio::sync::RwLock<()>>,
2496    operation: F,
2497) -> std::result::Result<T, E>
2498where
2499    F: FnOnce() -> Fut,
2500    Fut: std::future::Future<Output = std::result::Result<T, E>>,
2501    E: From<Error>,
2502{
2503    let _operation_guard = tokio::select! {
2504        biased;
2505        _ = rx.cancelled() => return Err(Error::OperationCanceled.into()),
2506        guard = operation_gate.read() => guard,
2507    };
2508
2509    if rx.is_cancelled() {
2510        return Err(Error::OperationCanceled.into());
2511    }
2512
2513    let result = operation().await;
2514    if rx.is_cancelled() {
2515        return Err(Error::OperationCanceled.into());
2516    }
2517    result
2518}
2519
2520fn track_decommission_current_object_stage(
2521    meta: &mut PoolMeta,
2522    idx: usize,
2523    bucket: &str,
2524    object: &str,
2525    stage: &str,
2526) -> Result<()> {
2527    let pool_count = meta.pools.len();
2528    ensure_valid_decommission_pool_index(pool_count, idx)?;
2529
2530    let Some(pool) = meta.pools.get_mut(idx) else {
2531        return Err(invalid_decommission_pool_index_error(pool_count, idx));
2532    };
2533    let Some(info) = pool.decommission.as_mut() else {
2534        return Err(decommission_metadata_not_initialized_error("track decommission current object"));
2535    };
2536
2537    info.object = object.to_string();
2538    info.bucket = bucket.to_string();
2539    info.stage = stage.to_string();
2540    Ok(())
2541}
2542
2543fn track_decommission_current_object(meta: &mut PoolMeta, idx: usize, bucket: &str, object: &str) -> Result<()> {
2544    track_decommission_current_object_stage(meta, idx, bucket, object, "")
2545}
2546
2547fn resolve_decommission_update_after_result(result: Result<bool>) -> Result<bool> {
2548    result.map_err(|err| Error::other(format!("decommission metadata update failed: {err}")))
2549}
2550
2551fn resolve_decommission_progress_save_result(result: Result<()>) -> Option<Error> {
2552    result
2553        .err()
2554        .map(|err| Error::other(format!("decommission progress save failed: {err}")))
2555}
2556
2557fn resolve_decommission_preflight_heal_result<T>(bucket: &str, result: Result<T>) -> Result<T> {
2558    result.map_err(|err| Error::other(format!("decommission preflight heal failed for bucket {bucket}: {err}")))
2559}
2560
2561fn resolve_decommission_optional_bucket_config_result<T>(bucket: &str, stage: &str, result: Result<T>) -> Result<Option<T>> {
2562    match result {
2563        Ok(config) => Ok(Some(config)),
2564        Err(Error::ConfigNotFound) => Ok(None),
2565        Err(err) => Err(Error::other(format!(
2566            "decommission {stage} config load failed for bucket {bucket}: {err}"
2567        ))),
2568    }
2569}
2570
2571fn resolve_decommission_entry_cleanup_delete_result<T>(result: Result<T>, bucket: &str, object_name: &str) -> Result<()> {
2572    match result {
2573        Ok(_) => Ok(()),
2574        Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(()),
2575        Err(err) => Err(Error::other(format!(
2576            "decommission cleanup_delete_object failed for {bucket}/{object_name}: {err}"
2577        ))),
2578    }
2579}
2580
2581fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
2582    result.map_err(|err| Error::other(format!("decommission reload_pool_meta failed for {bucket}/{object_name}: {err}")))
2583}
2584
2585fn resolve_decommission_terminal_mark_result(result: Result<()>, stage: &str, pool_label: &str) -> Result<()> {
2586    result.map_err(|err| Error::other(format!("decommission terminal mark {stage} failed for pool {pool_label}: {err}")))
2587}
2588
2589fn resolve_decommission_terminal_mark_after_error_result(result: Result<()>, idx: usize, primary_err: &Error) -> Result<()> {
2590    result.map_err(|err| {
2591        Error::other(format!(
2592            "decommission terminal mark failed after background error on pool {idx}: {primary_err}; mark error: {err}"
2593        ))
2594    })
2595}
2596
2597fn observe_decommission_terminal_reload_result(result: Result<()>, stage: &str) -> Option<Error> {
2598    result
2599        .err()
2600        .map(|err| Error::other(format!("decommission terminal pool meta reload failed during {stage}: {err}")))
2601}
2602
2603fn decommission_item_size<T>(size: T) -> usize
2604where
2605    usize: TryFrom<T>,
2606{
2607    usize::try_from(size).unwrap_or_default()
2608}
2609
2610fn is_decommission_set_local_usage_cache(bucket: &str, object: &str) -> bool {
2611    if bucket != RUSTFS_META_BUCKET {
2612        return false;
2613    }
2614    let Some(path) = object
2615        .strip_prefix(BUCKET_META_PREFIX)
2616        .and_then(|path| path.strip_prefix('/'))
2617    else {
2618        return false;
2619    };
2620    let name = match path.rsplit_once('/') {
2621        Some((bucket, name)) if !bucket.is_empty() && !bucket.contains('/') && bucket != "." && bucket != ".." => name,
2622        Some(_) => return false,
2623        None => path,
2624    };
2625    name.strip_suffix(".bkp").unwrap_or(name) == DATA_USAGE_CACHE_NAME
2626}
2627
2628fn with_decommission_entry_context<E: Display>(stage: &str, bucket: &str, object: &str, err: E) -> Error {
2629    Error::other(format!("decommission entry {stage} failed for bucket {bucket} object {object}: {err}"))
2630}
2631
2632#[cfg(test)]
2633fn load_decommission_entry_versions(entry: &MetaCacheEntry, bucket: &str, stage: &str) -> Result<FileInfoVersions> {
2634    entry
2635        .file_info_versions(bucket)
2636        .map_err(|err| with_decommission_entry_context(stage, bucket, &entry.name, err))
2637}
2638
2639fn empty_decommission_entry_versions(bucket: &str, object: &str) -> FileInfoVersions {
2640    FileInfoVersions {
2641        volume: bucket.to_string(),
2642        name: object.to_string(),
2643        versions: Vec::new(),
2644        ..Default::default()
2645    }
2646}
2647
2648fn resolve_decommission_entry_exact_versions(
2649    result: Result<Option<FileInfoVersions>>,
2650    entry: &MetaCacheEntry,
2651    bucket: &str,
2652    stage: &str,
2653) -> Result<FileInfoVersions> {
2654    match result {
2655        Ok(Some(fivs)) => Ok(fivs),
2656        Ok(None) => Ok(empty_decommission_entry_versions(bucket, &entry.name)),
2657        Err(err) => Err(with_decommission_entry_context(stage, bucket, &entry.name, err)),
2658    }
2659}
2660
2661async fn load_decommission_entry_exact_versions(
2662    set: &SetDisks,
2663    entry: &MetaCacheEntry,
2664    bucket: &str,
2665    stage: &str,
2666) -> Result<FileInfoVersions> {
2667    resolve_decommission_entry_exact_versions(set.load_file_info_versions_exact(bucket, &entry.name).await, entry, bucket, stage)
2668}
2669
2670fn resolve_decommission_check_after_list_result(list_result: Result<()>, entry_error: Option<Error>) -> Result<()> {
2671    match list_result {
2672        Ok(()) => entry_error.map_or(Ok(()), Err),
2673        Err(list_err) => resolve_decommission_listing_error(Some(list_err), entry_error).map_or(Ok(()), Err),
2674    }
2675}
2676
2677fn resolve_decommission_listing_error(listing_error: Option<Error>, entry_error: Option<Error>) -> Option<Error> {
2678    match (listing_error, entry_error) {
2679        (Some(listing_error), Some(entry_error)) if is_err_operation_canceled(&listing_error) => Some(entry_error),
2680        (Some(listing_error), Some(entry_error)) if is_err_operation_canceled(&entry_error) => Some(listing_error),
2681        (Some(listing_error), _) => Some(listing_error),
2682        (None, entry_error) => entry_error,
2683    }
2684}
2685
2686fn decommission_unresolved_listing_error(entry: &DecommissionUnresolvedEntry) -> Error {
2687    Error::other(format!(
2688        "decommission listing could not resolve metadata for {bucket}/{object} on pool {pool_index} set {set_index} ({candidate_count} candidate(s), {disk_error_count} disk error(s))",
2689        bucket = entry.bucket,
2690        object = entry.object,
2691        pool_index = entry.pool_index,
2692        set_index = entry.set_index,
2693        candidate_count = entry.candidate_count,
2694        disk_error_count = entry.disk_error_count,
2695    ))
2696}
2697
2698// The unresolved-entry payload carries listing context by design; boxing it
2699// would churn every caller without changing behavior.
2700#[allow(clippy::result_large_err)]
2701#[allow(clippy::too_many_arguments)]
2702fn resolve_decommission_partial_listing_entry(
2703    entries: MetaCacheEntries,
2704    resolver: MetadataResolutionParams,
2705    bucket: &str,
2706    prefix: &str,
2707    disk_error_count: usize,
2708    pool_index: usize,
2709    set_index: usize,
2710    source_generation: OffsetDateTime,
2711) -> std::result::Result<MetaCacheEntry, DecommissionUnresolvedEntry> {
2712    let candidate_count = entries.as_ref().iter().flatten().count();
2713    if let Some(entry) = entries.resolve(resolver) {
2714        return Ok(entry);
2715    }
2716
2717    let object = entries
2718        .as_ref()
2719        .iter()
2720        .flatten()
2721        .map(|entry| entry.name.as_str())
2722        .next()
2723        .unwrap_or(prefix)
2724        .to_string();
2725    Err(DecommissionUnresolvedEntry {
2726        bucket: bucket.to_string(),
2727        object,
2728        candidate_count,
2729        disk_error_count,
2730        pool_index,
2731        set_index,
2732        source_generation,
2733        observed_at: OffsetDateTime::now_utc(),
2734        reason: "metadata_resolution_failed".to_string(),
2735    })
2736}
2737
2738fn validate_decommission_durable_ilm_copy(
2739    path: &str,
2740    source_record: &ValidatedDurableIlmRecord,
2741    target: &[u8],
2742) -> Result<ValidatedDurableIlmRecord> {
2743    let target_record = validate_durable_ilm_record(path, target).map_err(|err| {
2744        Error::other(format!(
2745            "target durable ILM record is invalid at path `{path}` {}: {err}",
2746            source_record.context()
2747        ))
2748    })?;
2749    source_record
2750        .checkpoint
2751        .validate_successor(&target_record.checkpoint)
2752        .map_err(|err| {
2753            Error::other(format!(
2754                "target durable ILM record generation mismatch at path `{path}` {}: {err}",
2755                source_record.context()
2756            ))
2757        })?;
2758    Ok(target_record)
2759}
2760
2761#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
2762#[serde(deny_unknown_fields)]
2763struct DecommissionDurableIlmReceipt {
2764    source_path: String,
2765    namespace: String,
2766    id_kind: String,
2767    id: String,
2768    checkpoint: DurableIlmRecordCheckpoint,
2769    terminal_checkpoint: Option<DurableIlmRecordCheckpoint>,
2770    #[serde(default, skip_serializing_if = "Option::is_none")]
2771    fleet_topology_generation: Option<String>,
2772}
2773
2774impl DecommissionDurableIlmReceipt {
2775    fn new(path: &str, record: &ValidatedDurableIlmRecord, fleet_topology_generation: Option<String>) -> Self {
2776        Self {
2777            source_path: path.to_string(),
2778            namespace: record.namespace.to_string(),
2779            id_kind: record.id_kind.to_string(),
2780            id: record.id.clone(),
2781            checkpoint: record.checkpoint.clone(),
2782            terminal_checkpoint: None,
2783            fleet_topology_generation,
2784        }
2785    }
2786
2787    fn context(&self) -> String {
2788        format!("namespace `{}` {} `{}`", self.namespace, self.id_kind, self.id)
2789    }
2790
2791    fn validate(&self) -> Result<()> {
2792        let namespace = classify_durable_ilm_record(&self.source_path)?
2793            .ok_or_else(|| Error::other(format!("receipt source path `{}` is not a durable ILM record", self.source_path)))?;
2794        if namespace.name != self.namespace {
2795            return Err(Error::other(format!(
2796                "receipt namespace `{}` does not match source path `{}`",
2797                self.namespace, self.source_path
2798            )));
2799        }
2800        if self.id_kind.is_empty() || self.id.is_empty() {
2801            return Err(Error::other(format!(
2802                "receipt identity is missing for source path `{}`",
2803                self.source_path
2804            )));
2805        }
2806        if !is_sha256_checksum(self.checkpoint.content_sha256()) {
2807            return Err(Error::other(format!(
2808                "receipt target checksum is invalid for source path `{}` {}",
2809                self.source_path,
2810                self.context()
2811            )));
2812        }
2813        if let Some(terminal_checkpoint) = &self.terminal_checkpoint {
2814            self.checkpoint.validate_successor(terminal_checkpoint).map_err(|err| {
2815                Error::other(format!(
2816                    "receipt terminal checkpoint is invalid for source path `{}` {}: {err}",
2817                    self.source_path,
2818                    self.context()
2819                ))
2820            })?;
2821        }
2822        if self
2823            .fleet_topology_generation
2824            .as_deref()
2825            .is_some_and(|generation| !is_sha256_checksum(generation))
2826        {
2827            return Err(Error::other_with_context(
2828                "receipt fleet topology generation is invalid",
2829                format!("source path `{}` {}", self.source_path, self.context()),
2830            ));
2831        }
2832        Ok(())
2833    }
2834
2835    fn encode(&self) -> Result<Vec<u8>> {
2836        let mut receipt = self.clone();
2837        receipt.checkpoint = receipt.checkpoint.compacted()?;
2838        receipt.terminal_checkpoint = receipt
2839            .terminal_checkpoint
2840            .as_ref()
2841            .map(DurableIlmRecordCheckpoint::compacted)
2842            .transpose()?;
2843        receipt.validate()?;
2844        let receipt_bytes = serde_json::to_vec(&receipt)?;
2845        let persisted = PersistedDecommissionDurableIlmReceipt {
2846            schema: DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA.to_string(),
2847            content_sha256: hex_sha256(&receipt_bytes, ToOwned::to_owned),
2848            receipt,
2849        };
2850        let encoded = serde_json::to_vec(&persisted)?;
2851        if encoded.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE {
2852            return Err(Error::other(format!(
2853                "durable ILM receipt exceeds maximum size for source path `{}` {}",
2854                self.source_path,
2855                self.context()
2856            )));
2857        }
2858        Ok(encoded)
2859    }
2860
2861    fn decode(data: &[u8]) -> Result<Self> {
2862        if data.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE {
2863            return Err(Error::other("durable ILM receipt exceeds maximum size"));
2864        }
2865        let persisted: PersistedDecommissionDurableIlmReceipt = serde_json::from_slice(data)?;
2866        if persisted.schema != DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA {
2867            return Err(Error::other(format!("unsupported durable ILM receipt schema `{}`", persisted.schema)));
2868        }
2869        if !is_sha256_checksum(&persisted.content_sha256) {
2870            return Err(Error::other("durable ILM receipt checksum is invalid"));
2871        }
2872        let receipt_bytes = serde_json::to_vec(&persisted.receipt)?;
2873        let actual_checksum = hex_sha256(&receipt_bytes, ToOwned::to_owned);
2874        if persisted.content_sha256 != actual_checksum {
2875            return Err(Error::other("durable ILM receipt checksum mismatch"));
2876        }
2877        persisted.receipt.validate()?;
2878        Ok(persisted.receipt)
2879    }
2880}
2881
2882fn merge_decommission_durable_ilm_receipts(
2883    existing: &DecommissionDurableIlmReceipt,
2884    incoming: &DecommissionDurableIlmReceipt,
2885) -> Result<DecommissionDurableIlmReceipt> {
2886    if existing.source_path != incoming.source_path
2887        || existing.namespace != incoming.namespace
2888        || existing.id_kind != incoming.id_kind
2889        || existing.id != incoming.id
2890    {
2891        return Err(Error::other(format!(
2892            "durable ILM receipt identity conflict for source path `{}` {}; incoming {}",
2893            existing.source_path,
2894            existing.context(),
2895            incoming.context()
2896        )));
2897    }
2898
2899    let checkpoint =
2900        if existing.checkpoint == incoming.checkpoint || incoming.checkpoint.validate_successor(&existing.checkpoint).is_ok() {
2901            existing.checkpoint.clone()
2902        } else {
2903            existing.checkpoint.validate_successor(&incoming.checkpoint).map_err(|err| {
2904                Error::other(format!(
2905                    "durable ILM receipt checkpoint conflict for source path `{}` {}: {err}",
2906                    existing.source_path,
2907                    existing.context()
2908                ))
2909            })?;
2910            incoming.checkpoint.clone()
2911        };
2912    let terminal_checkpoint = match (&existing.terminal_checkpoint, &incoming.terminal_checkpoint) {
2913        (Some(existing_terminal), Some(incoming_terminal)) if existing_terminal == incoming_terminal => {
2914            Some(existing_terminal.clone())
2915        }
2916        (Some(existing_terminal), Some(incoming_terminal)) if incoming_terminal.validate_successor(existing_terminal).is_ok() => {
2917            Some(existing_terminal.clone())
2918        }
2919        (Some(existing_terminal), Some(incoming_terminal)) => {
2920            existing_terminal.validate_successor(incoming_terminal).map_err(|err| {
2921                Error::other(format!(
2922                    "durable ILM receipt terminal checkpoint conflict for source path `{}` {}: {err}",
2923                    existing.source_path,
2924                    existing.context()
2925                ))
2926            })?;
2927            Some(incoming_terminal.clone())
2928        }
2929        (Some(existing_terminal), None) => Some(existing_terminal.clone()),
2930        (None, Some(incoming_terminal)) => Some(incoming_terminal.clone()),
2931        (None, None) => None,
2932    };
2933    let merged = DecommissionDurableIlmReceipt {
2934        source_path: existing.source_path.clone(),
2935        namespace: existing.namespace.clone(),
2936        id_kind: existing.id_kind.clone(),
2937        id: existing.id.clone(),
2938        checkpoint,
2939        terminal_checkpoint,
2940        fleet_topology_generation: match (
2941            existing.fleet_topology_generation.as_ref(),
2942            incoming.fleet_topology_generation.as_ref(),
2943        ) {
2944            (Some(existing_generation), Some(incoming_generation)) if existing_generation == incoming_generation => {
2945                Some(existing_generation.clone())
2946            }
2947            (None, None) => None,
2948            _ => {
2949                return Err(Error::other_with_context(
2950                    "durable ILM receipt fleet topology conflict",
2951                    format!("source path `{}` {}", existing.source_path, existing.context()),
2952                ));
2953            }
2954        },
2955    };
2956    merged.validate()?;
2957    Ok(merged)
2958}
2959
2960#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
2961#[serde(deny_unknown_fields)]
2962struct PersistedDecommissionDurableIlmReceipt {
2963    schema: String,
2964    content_sha256: String,
2965    receipt: DecommissionDurableIlmReceipt,
2966}
2967
2968#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
2969#[serde(deny_unknown_fields)]
2970struct DecommissionDurableIlmManifest {
2971    schema: String,
2972    run_token: String,
2973    receipt_count: u64,
2974    receipt_paths_sha256: String,
2975}
2976
2977impl DecommissionDurableIlmManifest {
2978    fn new(run_token: &str, receipt_paths: &[String]) -> Result<Self> {
2979        let manifest = Self {
2980            schema: DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA.to_string(),
2981            run_token: run_token.to_string(),
2982            receipt_count: u64::try_from(receipt_paths.len())
2983                .map_err(|_| Error::other("durable ILM expected manifest receipt count exceeds u64"))?,
2984            receipt_paths_sha256: decommission_durable_ilm_manifest_paths_sha256(receipt_paths)?,
2985        };
2986        manifest.validate(run_token, receipt_paths)?;
2987        Ok(manifest)
2988    }
2989
2990    fn validate(&self, run_token: &str, receipt_paths: &[String]) -> Result<()> {
2991        if self.schema != DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA {
2992            return Err(Error::other(format!(
2993                "unsupported durable ILM expected manifest schema `{}`",
2994                self.schema
2995            )));
2996        }
2997        if self.run_token != run_token || !is_sha256_checksum(&self.run_token) {
2998            return Err(Error::other("durable ILM expected manifest run token is invalid"));
2999        }
3000        let receipt_count = u64::try_from(receipt_paths.len())
3001            .map_err(|_| Error::other("durable ILM expected manifest receipt count exceeds u64"))?;
3002        if self.receipt_count != receipt_count {
3003            return Err(Error::other(format!(
3004                "durable ILM expected manifest receipt count mismatch: expected {}, found {receipt_count}",
3005                self.receipt_count
3006            )));
3007        }
3008        if !is_sha256_checksum(&self.receipt_paths_sha256)
3009            || self.receipt_paths_sha256 != decommission_durable_ilm_manifest_paths_sha256(receipt_paths)?
3010        {
3011            return Err(Error::other("durable ILM expected manifest receipt paths checksum mismatch"));
3012        }
3013        Ok(())
3014    }
3015
3016    fn encode(&self) -> Result<Vec<u8>> {
3017        let encoded = serde_json::to_vec(self)?;
3018        if encoded.len() > DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE {
3019            return Err(Error::other("durable ILM expected manifest exceeds maximum size"));
3020        }
3021        Ok(encoded)
3022    }
3023
3024    fn decode(data: &[u8], run_token: &str, receipt_paths: &[String]) -> Result<Self> {
3025        if data.len() > DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE {
3026            return Err(Error::other("durable ILM expected manifest exceeds maximum size"));
3027        }
3028        let manifest: Self = serde_json::from_slice(data)?;
3029        manifest.validate(run_token, receipt_paths)?;
3030        Ok(manifest)
3031    }
3032}
3033
3034#[derive(Debug, Clone, PartialEq, Eq)]
3035struct DecommissionDurableIlmReceiptLocator {
3036    run_token: String,
3037    source_path: String,
3038    id_kind: String,
3039    id: String,
3040}
3041
3042impl DecommissionDurableIlmReceiptLocator {
3043    fn context(&self) -> String {
3044        format!("source path `{}` {} `{}`", self.source_path, self.id_kind, self.id)
3045    }
3046}
3047
3048fn decommission_durable_ilm_receipt_run_token(cmd_line: &str, start_time: OffsetDateTime) -> String {
3049    let identity = format!("{cmd_line}\0{}", start_time.unix_timestamp_nanos());
3050    hex_sha256(identity.as_bytes(), ToOwned::to_owned)
3051}
3052
3053fn decommission_durable_ilm_receipt_run_prefix(run_token: &str) -> String {
3054    format!("{DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT}/{run_token}/")
3055}
3056
3057fn decommission_durable_ilm_receipt_path(run_token: &str, source_path: &str, id_kind: &str, id: &str) -> String {
3058    format!(
3059        "{}{}/{}/{}.json",
3060        decommission_durable_ilm_receipt_run_prefix(run_token),
3061        source_path,
3062        id_kind,
3063        id
3064    )
3065}
3066
3067fn decommission_durable_ilm_manifest_path(run_token: &str) -> String {
3068    format!("{DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT}/{run_token}.json")
3069}
3070
3071fn decommission_durable_ilm_manifest_paths_sha256(receipt_paths: &[String]) -> Result<String> {
3072    let mut sorted_paths = receipt_paths.iter().map(String::as_str).collect::<Vec<_>>();
3073    sorted_paths.sort_unstable();
3074    let encoded = serde_json::to_vec(&sorted_paths)?;
3075    Ok(hex_sha256(&encoded, ToOwned::to_owned))
3076}
3077
3078fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result<DecommissionDurableIlmReceiptLocator> {
3079    let prefix = format!("{DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT}/");
3080    let suffix = path
3081        .strip_prefix(&prefix)
3082        .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has the wrong root")))?;
3083    let (run_token, record_path) = suffix
3084        .split_once('/')
3085        .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record path")))?;
3086    let mut parts = record_path.rsplitn(3, '/');
3087    let id = parts
3088        .next()
3089        .and_then(|file| file.strip_suffix(".json"))
3090        .filter(|id| !id.is_empty())
3091        .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record id")))?;
3092    let id_kind = parts
3093        .next()
3094        .filter(|id_kind| matches!(*id_kind, "operation_id" | "transaction_id" | "job_id" | "control_id"))
3095        .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has an invalid id kind")))?;
3096    let source_path = parts
3097        .next()
3098        .filter(|source_path| !source_path.is_empty())
3099        .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its source path")))?;
3100    if !is_sha256_checksum(run_token) {
3101        return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid run token")));
3102    }
3103    match id_kind {
3104        "operation_id" | "control_id" if !is_sha256_checksum(id) => {
3105            let id_label = id_kind.trim_end_matches("_id");
3106            return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid {id_label} id")));
3107        }
3108        "transaction_id" | "job_id" if uuid::Uuid::parse_str(id).is_err() => {
3109            return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid UUID")));
3110        }
3111        _ => {}
3112    }
3113    classify_durable_ilm_record(source_path)?
3114        .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` does not identify a durable ILM source path")))?;
3115    Ok(DecommissionDurableIlmReceiptLocator {
3116        run_token: run_token.to_string(),
3117        source_path: source_path.to_string(),
3118        id_kind: id_kind.to_string(),
3119        id: id.to_string(),
3120    })
3121}
3122
3123fn resolve_decommission_pool_meta_reload_result(result: Result<()>, stage: &str) -> Result<()> {
3124    result.map_err(|err| Error::other(format!("decommission pool meta reload failed during {stage}: {err}")))
3125}
3126
3127fn apply_decommission_status_space_info(mut pool_info: PoolStatus, space_info: PoolSpaceInfo) -> PoolStatus {
3128    match pool_info.decommission.as_mut() {
3129        Some(d) => {
3130            d.total_size = space_info.total;
3131            d.current_size = space_info.free;
3132        }
3133        None => {
3134            pool_info.decommission = Some(PoolDecommissionInfo {
3135                total_size: space_info.total,
3136                current_size: space_info.free,
3137                ..Default::default()
3138            });
3139        }
3140    }
3141
3142    pool_info
3143}
3144
3145fn should_replace_pool_status_for_status_refresh(
3146    current: Option<&PoolStatus>,
3147    persisted: &PoolStatus,
3148    has_active_worker: bool,
3149) -> bool {
3150    let Some(current) = current else {
3151        return true;
3152    };
3153
3154    !has_active_worker && persisted.last_update > current.last_update
3155}
3156
3157fn pool_decommission_movement_snapshot(
3158    info: Option<&PoolDecommissionInfo>,
3159) -> (bool, bool, bool, bool, bool, Option<OffsetDateTime>) {
3160    info.map(|info| {
3161        (
3162            info.has_decommission_state(),
3163            info.complete,
3164            info.failed,
3165            info.canceled,
3166            info.queued,
3167            info.start_time,
3168        )
3169    })
3170    .unwrap_or_default()
3171}
3172
3173pub(crate) fn pool_meta_movement_snapshot_changed(before: &PoolMeta, after: &PoolMeta) -> bool {
3174    before.pools.len() != after.pools.len()
3175        || before.pools.iter().zip(after.pools.iter()).any(|(before, after)| {
3176            pool_decommission_movement_snapshot(before.decommission.as_ref())
3177                != pool_decommission_movement_snapshot(after.decommission.as_ref())
3178        })
3179}
3180
3181/// Merges a persisted pool metadata snapshot into `current` monotonically:
3182/// a pool entry is replaced only when no active worker covers it and the
3183/// snapshot is strictly newer, so delayed snapshots never roll back local
3184/// queued/terminal progressions. Returns whether any entry was replaced or
3185/// appended.
3186pub(crate) fn merge_pool_status_refresh(current: &mut PoolMeta, persisted: PoolMeta, active_workers: &[bool]) -> bool {
3187    let observed_version = current.version.max(persisted.version);
3188    if persisted.pools.is_empty() {
3189        current.version = observed_version;
3190        return false;
3191    }
3192
3193    if current.pools.is_empty() {
3194        *current = persisted;
3195        current.version = observed_version;
3196        return true;
3197    }
3198
3199    current.version = observed_version;
3200
3201    let mut merged_newer = false;
3202    for (idx, persisted_pool) in persisted.pools.into_iter().enumerate() {
3203        if persisted_pool.id != idx {
3204            continue;
3205        }
3206
3207        let has_active_worker = active_workers.get(idx).copied().unwrap_or(false);
3208        if idx < current.pools.len() {
3209            if should_replace_pool_status_for_status_refresh(current.pools.get(idx), &persisted_pool, has_active_worker) {
3210                current.pools[idx] = persisted_pool;
3211                merged_newer = true;
3212            }
3213        } else if idx == current.pools.len() && !has_active_worker {
3214            current.pools.push(persisted_pool);
3215            merged_newer = true;
3216        }
3217    }
3218    merged_newer
3219}
3220
3221fn merge_pool_meta_updates_for_save(
3222    persisted: &mut PoolMeta,
3223    current: &PoolMeta,
3224    indices: &[usize],
3225    operation: &str,
3226) -> Result<()> {
3227    if persisted.pools.is_empty() {
3228        *persisted = current.clone();
3229        return Ok(());
3230    }
3231    if persisted.version != current.version {
3232        return Err(Error::other(format!(
3233            "{operation}: pool metadata version changed from {} to {}",
3234            current.version, persisted.version
3235        )));
3236    }
3237
3238    for &idx in indices {
3239        let current_pool = current
3240            .pools
3241            .get(idx)
3242            .ok_or_else(|| invalid_decommission_pool_index_error(current.pools.len(), idx))?;
3243        let persisted_count = persisted.pools.len();
3244        let persisted_pool = persisted
3245            .pools
3246            .get_mut(idx)
3247            .ok_or_else(|| invalid_decommission_pool_index_error(persisted_count, idx))?;
3248        if current_pool.id != idx || persisted_pool.id != idx || current_pool.cmd_line != persisted_pool.cmd_line {
3249            return Err(Error::other(format!("{operation}: pool metadata layout changed for pool {idx}")));
3250        }
3251        let current_clears_decommission = current_pool
3252            .decommission
3253            .as_ref()
3254            .is_none_or(|info| !info.has_decommission_state());
3255        if current_clears_decommission
3256            && persisted_pool.decommission.as_ref().is_some_and(|info| {
3257                info.has_decommission_state() && is_decommission_active(info.complete, info.failed, info.canceled)
3258            })
3259        {
3260            record_pool_meta_stale_write_rejection("nonterminal_decommission_clear");
3261            return Err(Error::other(format!(
3262                "{operation}: stale pool metadata update rejected for pool {idx}; persisted active or queued decommission cannot be cleared"
3263            )));
3264        }
3265        if current_clears_decommission
3266            && persisted_pool
3267                .decommission
3268                .as_ref()
3269                .is_some_and(|info| info.complete || !info.unresolved_entries.is_empty())
3270        {
3271            record_pool_meta_stale_write_rejection("unsafe_terminal_decommission_clear");
3272            return Err(Error::StalePoolMetadataUpdate {
3273                operation: operation.to_string(),
3274                pool_index: idx,
3275                reason: "completed or unresolved decommission state cannot be cleared",
3276            });
3277        }
3278        if current_pool.last_update < persisted_pool.last_update {
3279            record_pool_meta_stale_write_rejection("older_pool_revision");
3280            return Err(Error::other(format!(
3281                "{operation}: stale pool metadata update rejected for pool {idx}; persisted update is newer"
3282            )));
3283        }
3284        if let (Some(persisted_info), Some(current_info)) =
3285            (persisted_pool.decommission.as_ref(), current_pool.decommission.as_ref())
3286            && current_info.has_decommission_state()
3287        {
3288            let persisted_terminal = (persisted_info.complete, persisted_info.failed, persisted_info.canceled);
3289            let current_terminal = (current_info.complete, current_info.failed, current_info.canceled);
3290            if persisted_terminal != (false, false, false) && persisted_terminal != current_terminal {
3291                record_pool_meta_stale_write_rejection("terminal_state_regression");
3292                return Err(Error::other(format!(
3293                    "{operation}: stale pool metadata update rejected for pool {idx}; terminal state cannot be replaced"
3294                )));
3295            }
3296            if current_info.items_decommissioned < persisted_info.items_decommissioned
3297                || current_info.items_decommission_failed < persisted_info.items_decommission_failed
3298                || current_info.bytes_done < persisted_info.bytes_done
3299                || current_info.bytes_failed < persisted_info.bytes_failed
3300            {
3301                record_pool_meta_stale_write_rejection("progress_regression");
3302                return Err(Error::other(format!(
3303                    "{operation}: stale pool metadata update rejected for pool {idx}; durable progress cannot decrease"
3304                )));
3305            }
3306        }
3307        *persisted_pool = current_pool.clone();
3308    }
3309
3310    Ok(())
3311}
3312
3313fn publish_pool_meta_updates(current: &mut PoolMeta, saved: &PoolMeta, indices: &[usize]) {
3314    current.version = current.version.max(saved.version);
3315    for &idx in indices {
3316        let Some(saved_pool) = saved.pools.get(idx) else {
3317            continue;
3318        };
3319        let Some(current_pool) = current.pools.get_mut(idx) else {
3320            continue;
3321        };
3322        if current_pool.id == idx && saved_pool.id == idx && current_pool.cmd_line == saved_pool.cmd_line {
3323            *current_pool = saved_pool.clone();
3324        }
3325    }
3326}
3327
3328/// Callers retain the save gate and namespace fence from the durable read/save
3329/// through publication, so capacity snapshots cannot publish out of order.
3330fn publish_decommission_capacity_update(current: &mut PoolMeta, saved: &PoolMeta, idx: usize) -> Result<()> {
3331    let stale = |reason| Error::StalePoolMetadataUpdate {
3332        operation: "publish decommission capacity".to_string(),
3333        pool_index: idx,
3334        reason,
3335    };
3336    let saved_pool = saved
3337        .pools
3338        .get(idx)
3339        .ok_or_else(|| invalid_decommission_pool_index_error(saved.pools.len(), idx))?;
3340    let current_count = current.pools.len();
3341    let pool = current
3342        .pools
3343        .get_mut(idx)
3344        .ok_or_else(|| invalid_decommission_pool_index_error(current_count, idx))?;
3345    if pool.id != idx || saved_pool.id != idx || pool.cmd_line != saved_pool.cmd_line || saved.version < current.version {
3346        return Err(stale("pool identity changed before capacity publication"));
3347    }
3348    let info = pool
3349        .decommission
3350        .as_mut()
3351        .ok_or_else(|| stale("decommission cleared before capacity publication"))?;
3352    let saved_info = saved_pool
3353        .decommission
3354        .as_ref()
3355        .ok_or_else(|| stale("persisted decommission is missing during capacity publication"))?;
3356    if (info.start_time, info.queued, info.complete, info.failed, info.canceled)
3357        != (
3358            saved_info.start_time,
3359            saved_info.queued,
3360            saved_info.complete,
3361            saved_info.failed,
3362            saved_info.canceled,
3363        )
3364    {
3365        return Err(stale("decommission generation or terminal state changed before capacity publication"));
3366    }
3367    let identity = |reservation: &DecommissionCapacityReservation| {
3368        (
3369            reservation.source_pool_index,
3370            reservation.operation_id,
3371            reservation.generation,
3372            reservation.owner_nonce,
3373            reservation.model_version,
3374        )
3375    };
3376    if info.capacity_reservation.as_ref().map(identity) != saved_info.capacity_reservation.as_ref().map(identity) {
3377        return Err(stale("decommission capacity owner changed before publication"));
3378    }
3379
3380    // Capacity transactions do not checkpoint the worker's counters or queue.
3381    // Keep their clock separate before advancing the shared pool revision.
3382    info.progress_save_last_at.get_or_insert(pool.last_update);
3383    info.capacity_reservation = saved_info.capacity_reservation.clone();
3384    info.capacity_blocked_reason = saved_info.capacity_blocked_reason.clone();
3385    pool.last_update = pool.last_update.max(saved_pool.last_update);
3386    current.version = current.version.max(saved.version);
3387    Ok(())
3388}
3389
3390fn resolve_start_decommission_pool_meta_reload_result(result: Result<()>) -> Result<()> {
3391    resolve_decommission_pool_meta_reload_result(result, "start_decommission")
3392}
3393
3394fn activation_rebalance_meta_lock_error(err: rustfs_lock::LockError) -> Error {
3395    match err {
3396        rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
3397            mode: "write",
3398            bucket: RUSTFS_META_BUCKET.to_string(),
3399            object: REBAL_META_NAME.to_string(),
3400            required,
3401            achieved,
3402        },
3403        other => Error::other(format!(
3404            "failed to acquire rebalance activation lock on {RUSTFS_META_BUCKET}/{REBAL_META_NAME}: {other}"
3405        )),
3406    }
3407}
3408
3409fn activation_pool_meta_lock_error(err: rustfs_lock::LockError) -> Error {
3410    match err {
3411        rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
3412            mode: "write",
3413            bucket: RUSTFS_META_BUCKET.to_string(),
3414            object: POOL_META_NAME.to_string(),
3415            required,
3416            achieved,
3417        },
3418        other => Error::other(format!(
3419            "failed to acquire pool activation lock on {RUSTFS_META_BUCKET}/{POOL_META_NAME}: {other}"
3420        )),
3421    }
3422}
3423
3424pub(crate) struct PoolRebalanceActivationFence {
3425    pool_meta_guard: rustfs_lock::NamespaceLockGuard,
3426    rebalance_meta_guard: rustfs_lock::NamespaceLockGuard,
3427    fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
3428    decommission_target_fence_proof: Option<crate::services::notification_sys::DecommissionTargetFenceFleetProofToken>,
3429    #[cfg(test)]
3430    forced_lost: Arc<AtomicBool>,
3431}
3432
3433impl PoolRebalanceActivationFence {
3434    pub(crate) fn set_fleet_proof(
3435        &mut self,
3436        fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
3437    ) {
3438        self.fleet_proof = fleet_proof;
3439    }
3440
3441    fn set_decommission_target_fence_proof(
3442        &mut self,
3443        fleet_proof: Option<crate::services::notification_sys::DecommissionTargetFenceFleetProofToken>,
3444    ) {
3445        self.decommission_target_fence_proof = fleet_proof;
3446    }
3447
3448    pub(crate) fn ensure_held(&self) -> Result<()> {
3449        #[cfg(test)]
3450        let forced_lost = self.forced_lost.load(Ordering::Acquire);
3451        #[cfg(not(test))]
3452        let forced_lost = false;
3453        if forced_lost || self.pool_meta_guard.is_lock_lost() || self.rebalance_meta_guard.is_lock_lost() {
3454            return Err(Error::other("activation lock lost before metadata commit or worker admission"));
3455        }
3456        if self
3457            .fleet_proof
3458            .as_ref()
3459            .is_some_and(|proof| !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof))
3460        {
3461            return Err(Error::other(POOL_ACTIVATION_FLEET_PROOF_EXPIRED));
3462        }
3463        if self
3464            .decommission_target_fence_proof
3465            .as_ref()
3466            .is_some_and(|proof| !crate::services::notification_sys::decommission_target_fence_fleet_proof_matches(proof))
3467        {
3468            return Err(Error::other(DECOMMISSION_TARGET_FLEET_PROOF_EXPIRED));
3469        }
3470
3471        Ok(())
3472    }
3473
3474    pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
3475        opts.add_namespace_lock_guard(&self.pool_meta_guard);
3476        opts.add_namespace_lock_guard(&self.rebalance_meta_guard);
3477    }
3478
3479    #[cfg(test)]
3480    fn force_lost_for_test(&self) {
3481        self.forced_lost.store(true, Ordering::Release);
3482    }
3483}
3484
3485#[cfg(test)]
3486tokio::task_local! {
3487    pub(crate) static REBALANCE_ACTIVATION_LOCK_ATTEMPT: Arc<tokio::sync::Notify>;
3488}
3489
3490pub(crate) async fn acquire_pool_rebalance_activation_locks<S>(
3491    pool: Arc<S>,
3492    fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
3493) -> Result<PoolRebalanceActivationFence>
3494where
3495    S: crate::storage_api_contracts::namespace::NamespaceLocking<
3496            Error = Error,
3497            NamespaceLock = rustfs_lock::NamespaceLockWrapper,
3498        >,
3499{
3500    // Match entry admission: rebalance.bin -> pool.bin. An entry retains its
3501    // run read fence while target mutations acquire the pool metadata fence;
3502    // activation must not hold pool.bin while waiting for that entry to drain.
3503    let rebalance_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
3504    #[cfg(test)]
3505    let _ = REBALANCE_ACTIVATION_LOCK_ATTEMPT.try_with(|attempted| attempted.notify_one());
3506    let rebalance_meta_guard = rebalance_meta_lock
3507        .get_write_lock(get_lock_acquire_timeout())
3508        .await
3509        .map_err(activation_rebalance_meta_lock_error)?;
3510    let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
3511    let pool_meta_guard = pool_meta_lock
3512        .get_write_lock(get_lock_acquire_timeout())
3513        .await
3514        .map_err(activation_pool_meta_lock_error)?;
3515
3516    Ok(PoolRebalanceActivationFence {
3517        pool_meta_guard,
3518        rebalance_meta_guard,
3519        fleet_proof,
3520        decommission_target_fence_proof: None,
3521        #[cfg(test)]
3522        forced_lost: Arc::new(AtomicBool::new(false)),
3523    })
3524}
3525
3526pub(crate) async fn acquire_pool_activation_fleet_proof(
3527    ctx: &crate::runtime::instance::InstanceContext,
3528) -> Result<Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>> {
3529    if !ctx.is_dist_erasure().await {
3530        return Ok(None);
3531    }
3532    crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof()
3533        .map(Some)
3534        .ok_or_else(|| Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED))
3535}
3536
3537pub fn is_pool_activation_fleet_proof_error(err: &Error) -> bool {
3538    // Save-stage helpers add context by formatting the original error, so the
3539    // marker may be nested in the display string. Restrict matching to the
3540    // `Error::other` I/O shape used by this activation path.
3541    matches!(err, Error::Io(io_error) if io_error.kind() == std::io::ErrorKind::Other && {
3542        let message = io_error.to_string();
3543        message.contains(POOL_ACTIVATION_FLEET_PROOF_REQUIRED) || message.contains(POOL_ACTIVATION_FLEET_PROOF_EXPIRED)
3544    })
3545}
3546
3547#[cfg(test)]
3548#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3549pub(crate) enum PoolActivationStartKind {
3550    Rebalance,
3551    Decommission,
3552}
3553
3554#[cfg(test)]
3555struct PoolActivationDurableSaveBarrierState {
3556    pool_key: usize,
3557    arrived: tokio::sync::Notify,
3558    release: tokio::sync::Notify,
3559    force_fence_loss: AtomicBool,
3560}
3561
3562#[cfg(test)]
3563static POOL_ACTIVATION_DURABLE_SAVE_BARRIER: std::sync::OnceLock<
3564    std::sync::Mutex<Option<Arc<PoolActivationDurableSaveBarrierState>>>,
3565> = std::sync::OnceLock::new();
3566
3567#[cfg(test)]
3568pub(crate) struct PoolActivationDurableSaveBarrier {
3569    state: Arc<PoolActivationDurableSaveBarrierState>,
3570}
3571
3572#[cfg(test)]
3573fn pool_activation_test_pool_key<S>(pool: &Arc<S>) -> usize {
3574    Arc::as_ptr(pool).cast::<()>() as usize
3575}
3576
3577#[cfg(test)]
3578impl PoolActivationDurableSaveBarrier {
3579    pub(crate) fn install<S>(pool: &Arc<S>) -> Self {
3580        let state = Arc::new(PoolActivationDurableSaveBarrierState {
3581            pool_key: pool_activation_test_pool_key(pool),
3582            arrived: tokio::sync::Notify::new(),
3583            release: tokio::sync::Notify::new(),
3584            force_fence_loss: AtomicBool::new(false),
3585        });
3586        let mut barrier = POOL_ACTIVATION_DURABLE_SAVE_BARRIER
3587            .get_or_init(|| std::sync::Mutex::new(None))
3588            .lock()
3589            .expect("pool activation durable save barrier should not be poisoned");
3590        assert!(barrier.is_none(), "pool activation durable save barrier must be unique");
3591        *barrier = Some(Arc::clone(&state));
3592        Self { state }
3593    }
3594
3595    pub(crate) async fn wait_until_paused(&self) {
3596        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.arrived.notified())
3597            .await
3598            .expect("activation should reach the post-durable-save barrier");
3599    }
3600
3601    pub(crate) fn release_after_fence_loss(&self) {
3602        self.state.force_fence_loss.store(true, Ordering::Release);
3603        self.state.release.notify_one();
3604    }
3605
3606    pub(crate) fn release_without_fence_loss(&self) {
3607        self.state.release.notify_one();
3608    }
3609}
3610
3611#[cfg(test)]
3612impl Drop for PoolActivationDurableSaveBarrier {
3613    fn drop(&mut self) {
3614        self.state.release.notify_one();
3615        let mut barrier = POOL_ACTIVATION_DURABLE_SAVE_BARRIER
3616            .get_or_init(|| std::sync::Mutex::new(None))
3617            .lock()
3618            .expect("pool activation durable save barrier should not be poisoned");
3619        if barrier.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
3620            *barrier = None;
3621        }
3622    }
3623}
3624
3625#[cfg(test)]
3626pub(crate) async fn pause_pool_activation_after_durable_save<S>(pool: &Arc<S>, fence: &PoolRebalanceActivationFence) {
3627    let pool_key = pool_activation_test_pool_key(pool);
3628    let barrier = {
3629        let mut barrier = POOL_ACTIVATION_DURABLE_SAVE_BARRIER
3630            .get_or_init(|| std::sync::Mutex::new(None))
3631            .lock()
3632            .expect("pool activation durable save barrier should not be poisoned");
3633        if barrier.as_ref().is_some_and(|state| state.pool_key == pool_key) {
3634            barrier.take()
3635        } else {
3636            None
3637        }
3638    };
3639    if let Some(barrier) = barrier {
3640        barrier.arrived.notify_one();
3641        barrier.release.notified().await;
3642        if barrier.force_fence_loss.load(Ordering::Acquire) {
3643            fence.force_lost_for_test();
3644        }
3645    }
3646}
3647
3648#[cfg(test)]
3649struct PoolActivationStartProbeState {
3650    kind: PoolActivationStartKind,
3651    preflight_side_effect_attempted: std::sync::atomic::AtomicBool,
3652    attempted: std::sync::atomic::AtomicBool,
3653    notify: tokio::sync::Notify,
3654}
3655
3656#[cfg(test)]
3657static POOL_ACTIVATION_START_PROBES: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<PoolActivationStartProbeState>>>> =
3658    std::sync::OnceLock::new();
3659
3660#[cfg(test)]
3661pub(crate) struct PoolActivationStartProbe {
3662    state: Arc<PoolActivationStartProbeState>,
3663}
3664
3665#[cfg(test)]
3666impl PoolActivationStartProbe {
3667    pub(crate) fn install(kind: PoolActivationStartKind) -> Self {
3668        let state = Arc::new(PoolActivationStartProbeState {
3669            kind,
3670            preflight_side_effect_attempted: std::sync::atomic::AtomicBool::new(false),
3671            attempted: std::sync::atomic::AtomicBool::new(false),
3672            notify: tokio::sync::Notify::new(),
3673        });
3674        POOL_ACTIVATION_START_PROBES
3675            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
3676            .lock()
3677            .expect("pool activation start probe should not be poisoned")
3678            .push(Arc::clone(&state));
3679        Self { state }
3680    }
3681
3682    pub(crate) async fn wait_until_attempted(&self) {
3683        while !self.state.attempted.load(Ordering::Acquire) {
3684            self.state.notify.notified().await;
3685        }
3686    }
3687
3688    pub(crate) fn preflight_side_effect_was_attempted(&self) -> bool {
3689        self.state.preflight_side_effect_attempted.load(Ordering::Acquire)
3690    }
3691
3692    pub(crate) fn activation_was_attempted(&self) -> bool {
3693        self.state.attempted.load(Ordering::Acquire)
3694    }
3695}
3696
3697#[cfg(test)]
3698impl Drop for PoolActivationStartProbe {
3699    fn drop(&mut self) {
3700        let mut probes = POOL_ACTIVATION_START_PROBES
3701            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
3702            .lock()
3703            .expect("pool activation start probe should not be poisoned");
3704        probes.retain(|state| !Arc::ptr_eq(state, &self.state));
3705    }
3706}
3707
3708#[cfg(test)]
3709pub(crate) fn observe_pool_activation_start_attempt(kind: PoolActivationStartKind) {
3710    let probes = POOL_ACTIVATION_START_PROBES
3711        .get_or_init(|| std::sync::Mutex::new(Vec::new()))
3712        .lock()
3713        .expect("pool activation start probe should not be poisoned")
3714        .iter()
3715        .filter(|state| state.kind == kind)
3716        .cloned()
3717        .collect::<Vec<_>>();
3718    for state in probes {
3719        state.attempted.store(true, Ordering::Release);
3720        state.notify.notify_one();
3721    }
3722}
3723
3724#[cfg(test)]
3725fn observe_pool_activation_preflight_side_effect_attempt(kind: PoolActivationStartKind) {
3726    let probes = POOL_ACTIVATION_START_PROBES
3727        .get_or_init(|| std::sync::Mutex::new(Vec::new()))
3728        .lock()
3729        .expect("pool activation start probe should not be poisoned")
3730        .iter()
3731        .filter(|state| state.kind == kind)
3732        .cloned()
3733        .collect::<Vec<_>>();
3734    for state in probes {
3735        state.preflight_side_effect_attempted.store(true, Ordering::Release);
3736    }
3737}
3738
3739fn rollback_decommission_pool_meta(pool_meta: &mut PoolMeta, previous_pool_meta: &PoolMeta, indices: &[usize]) {
3740    publish_pool_meta_updates(pool_meta, previous_pool_meta, indices);
3741}
3742
3743#[derive(Debug)]
3744struct DecommissionCancelCommit {
3745    previous_start_time: Option<OffsetDateTime>,
3746    previous_queued: bool,
3747    previous_last_update: OffsetDateTime,
3748    canceled_pool: PoolStatus,
3749}
3750
3751fn commit_decommission_cancel(pool_meta: &mut PoolMeta, idx: usize, commit: DecommissionCancelCommit) -> Result<()> {
3752    let pool_count = pool_meta.pools.len();
3753    let Some(current) = pool_meta.pools.get(idx) else {
3754        return Err(invalid_decommission_pool_index_error(pool_count, idx));
3755    };
3756    // A peer reload can install the saved cancel while runtime-only fields are reconstructed.
3757    let cancel_already_published = commit
3758        .canceled_pool
3759        .decommission
3760        .as_ref()
3761        .is_some_and(|info| info.canceled && !info.complete && !info.failed && info.start_time.is_none())
3762        && PersistedPoolStatus::from(current) == PersistedPoolStatus::from(&commit.canceled_pool);
3763    if cancel_already_published {
3764        return Ok(());
3765    }
3766
3767    let matches_generation = current.id == commit.canceled_pool.id
3768        && current.cmd_line == commit.canceled_pool.cmd_line
3769        && current.decommission.as_ref().is_some_and(|info| {
3770            info.start_time == commit.previous_start_time
3771                && info.queued == commit.previous_queued
3772                && is_decommission_active(info.complete, info.failed, info.canceled)
3773                && (commit.previous_start_time.is_some() || current.last_update == commit.previous_last_update)
3774        });
3775    if !matches_generation {
3776        return Err(Error::other(format!(
3777            "failed to publish decommission cancel for pool {idx}: operation generation changed"
3778        )));
3779    }
3780
3781    pool_meta.pools[idx] = commit.canceled_pool;
3782    Ok(())
3783}
3784
3785fn rollback_start_decommission_pool_meta(pool_meta: &mut PoolMeta, previous_pool_meta: &PoolMeta, indices: &[usize]) {
3786    let active_updates = indices
3787        .iter()
3788        .filter_map(|&idx| pool_meta.pools.get(idx).map(|pool| (idx, pool.last_update)))
3789        .collect::<Vec<_>>();
3790    rollback_decommission_pool_meta(pool_meta, previous_pool_meta, indices);
3791    let rollback_at = OffsetDateTime::now_utc();
3792    for (idx, active_update) in active_updates {
3793        if let Some(pool) = pool_meta.pools.get_mut(idx) {
3794            pool.last_update = std::cmp::max(rollback_at, active_update + Duration::nanoseconds(1));
3795        }
3796    }
3797}
3798
3799fn ensure_pool_meta_write_fence(guard: &rustfs_lock::NamespaceLockGuard, operation: &str) -> Result<()> {
3800    if guard.is_lock_lost() {
3801        return Err(Error::other(format!("{operation}: pool metadata distributed fence was lost")));
3802    }
3803    Ok(())
3804}
3805
3806fn ensure_pool_not_left_in_cmdline_after_decommission(position: usize, cmd_line: &str, completed: bool) -> Result<()> {
3807    if completed {
3808        return Err(Error::other(format!(
3809            "pool({}) = {} is decommissioned, please remove from server command line",
3810            position + 1,
3811            cmd_line
3812        )));
3813    }
3814
3815    Ok(())
3816}
3817
3818fn resolve_decommission_listing_worker_result(
3819    set_idx: usize,
3820    worker_result: std::result::Result<Result<()>, tokio::task::JoinError>,
3821) -> Result<()> {
3822    worker_result.map_err(|err| Error::other(format!("decommission listing worker {set_idx} task join error: {err}")))?
3823}
3824
3825fn should_retry_decommission_listing(err: &Error, attempt: usize, max_attempts: usize) -> bool {
3826    !is_err_bucket_not_found(err) && attempt + 1 < max_attempts
3827}
3828
3829async fn wait_decommission_retry_backoff(rx: &CancellationToken, delay: std::time::Duration) -> bool {
3830    tokio::select! {
3831        _ = rx.cancelled() => true,
3832        _ = tokio::time::sleep(delay) => false,
3833    }
3834}
3835
3836fn decommission_retry_backoff_delay(base: std::time::Duration, attempt: usize) -> std::time::Duration {
3837    base.saturating_mul(u32::try_from(attempt).unwrap_or(u32::MAX))
3838}
3839
3840#[cfg(test)]
3841async fn run_decommission_listing_with_retry<List, ListFuture>(
3842    rx: CancellationToken,
3843    bucket: String,
3844    cb: ListCallback,
3845    pool_idx: usize,
3846    set_idx: usize,
3847    max_attempts: usize,
3848    list: List,
3849) -> Result<()>
3850where
3851    List: FnMut(ListCallback) -> ListFuture,
3852    ListFuture: std::future::Future<Output = Result<()>>,
3853{
3854    run_decommission_listing_with_retry_and_drain(rx, bucket, cb, pool_idx, set_idx, max_attempts, list, || async { false }).await
3855}
3856
3857#[allow(clippy::too_many_arguments)]
3858async fn run_decommission_listing_with_retry_and_drain<List, ListFuture, Drain, DrainFuture>(
3859    rx: CancellationToken,
3860    bucket: String,
3861    cb: ListCallback,
3862    pool_idx: usize,
3863    set_idx: usize,
3864    max_attempts: usize,
3865    mut list: List,
3866    mut drain: Drain,
3867) -> Result<()>
3868where
3869    List: FnMut(ListCallback) -> ListFuture,
3870    ListFuture: std::future::Future<Output = Result<()>>,
3871    Drain: FnMut() -> DrainFuture,
3872    DrainFuture: std::future::Future<Output = bool>,
3873{
3874    let max_attempts = max_attempts.max(1);
3875
3876    for attempt in 0..max_attempts {
3877        if rx.is_cancelled() {
3878            debug!(
3879                event = EVENT_DECOMMISSION_BUCKET,
3880                component = LOG_COMPONENT_ECSTORE,
3881                subsystem = LOG_SUBSYSTEM_POOLS,
3882                pool_index = pool_idx,
3883                set_index = set_idx,
3884                bucket = %bucket,
3885                state = "listing_worker_cancelled",
3886                "Decommission listing worker cancelled"
3887            );
3888            return Ok(());
3889        }
3890
3891        debug!(
3892            event = EVENT_DECOMMISSION_BUCKET,
3893            component = LOG_COMPONENT_ECSTORE,
3894            subsystem = LOG_SUBSYSTEM_POOLS,
3895            pool_index = pool_idx,
3896            set_index = set_idx,
3897            bucket = %bucket,
3898            attempt = attempt + 1,
3899            max_attempts,
3900            state = "listing_started",
3901            "Decommission listing started"
3902        );
3903
3904        let list_result = list(cb.clone()).await;
3905        if drain().await {
3906            return Ok(());
3907        }
3908
3909        match list_result {
3910            Ok(()) => {
3911                debug!(
3912                    event = EVENT_DECOMMISSION_BUCKET,
3913                    component = LOG_COMPONENT_ECSTORE,
3914                    subsystem = LOG_SUBSYSTEM_POOLS,
3915                    pool_index = pool_idx,
3916                    set_index = set_idx,
3917                    bucket = %bucket,
3918                    attempt = attempt + 1,
3919                    max_attempts,
3920                    state = "listing_completed",
3921                    "Decommission listing completed"
3922                );
3923                return Ok(());
3924            }
3925            Err(err) if is_err_bucket_not_found(&err) => {
3926                warn!(
3927                    event = EVENT_DECOMMISSION_BUCKET,
3928                    component = LOG_COMPONENT_ECSTORE,
3929                    subsystem = LOG_SUBSYSTEM_POOLS,
3930                    pool_index = pool_idx,
3931                    set_index = set_idx,
3932                    bucket = %bucket,
3933                    attempt = attempt + 1,
3934                    max_attempts,
3935                    state = "listing_bucket_missing",
3936                    "Decommission listing bucket missing"
3937                );
3938                return Ok(());
3939            }
3940            Err(err) if should_retry_decommission_listing(&err, attempt, max_attempts) => {
3941                error!(
3942                    event = EVENT_DECOMMISSION_BUCKET,
3943                    component = LOG_COMPONENT_ECSTORE,
3944                    subsystem = LOG_SUBSYSTEM_POOLS,
3945                    pool_index = pool_idx,
3946                    set_index = set_idx,
3947                    bucket = %bucket,
3948                    attempt = attempt + 1,
3949                    max_attempts,
3950                    retry_delay_ms = DECOMMISSION_LISTING_RETRY_DELAY.as_millis(),
3951                    state = "listing_failed_retrying",
3952                    error = ?err,
3953                    "Decommission listing failed; retrying"
3954                );
3955                if wait_decommission_retry_backoff(&rx, DECOMMISSION_LISTING_RETRY_DELAY).await {
3956                    debug!(
3957                        event = EVENT_DECOMMISSION_BUCKET,
3958                        component = LOG_COMPONENT_ECSTORE,
3959                        subsystem = LOG_SUBSYSTEM_POOLS,
3960                        pool_index = pool_idx,
3961                        set_index = set_idx,
3962                        bucket = %bucket,
3963                        state = "listing_worker_cancelled",
3964                        "Decommission listing worker cancelled during retry wait"
3965                    );
3966                    return Ok(());
3967                }
3968            }
3969            Err(err) => {
3970                error!(
3971                    event = EVENT_DECOMMISSION_BUCKET,
3972                    component = LOG_COMPONENT_ECSTORE,
3973                    subsystem = LOG_SUBSYSTEM_POOLS,
3974                    pool_index = pool_idx,
3975                    set_index = set_idx,
3976                    bucket = %bucket,
3977                    attempt = attempt + 1,
3978                    max_attempts,
3979                    state = "listing_failed",
3980                    error = ?err,
3981                    "Decommission listing failed"
3982                );
3983                return Err(Error::other(format!(
3984                    "decommission listing failed for bucket {bucket} pool {pool_idx} set {set_idx} attempt {}/{}: {err}",
3985                    attempt + 1,
3986                    max_attempts
3987                )));
3988            }
3989        }
3990    }
3991
3992    Ok(())
3993}
3994
3995fn should_count_decommission_version_complete(ignore: bool, cleanup_ignored: bool, failure: bool) -> bool {
3996    cleanup_ignored || (!ignore && !failure)
3997}
3998
3999fn is_decommission_copy_cleanup_safe_error(err: &Error) -> bool {
4000    // DataMovementOverwriteErr only means source and destination pool resolved to
4001    // the same pool. Without a target equivalence check it is not cleanup-safe.
4002    if is_err_object_not_found(err) || is_err_version_not_found(err) {
4003        return true;
4004    }
4005
4006    // A not-found surfacing from inside a data-movement stage is the same
4007    // condition once the wrapper is unwrapped (backlog#1827 T2).
4008    data_movement::data_movement_stage_source(err).is_some_and(is_decommission_copy_cleanup_safe_error)
4009}
4010
4011fn is_decommission_target_capacity_error(err: &Error) -> bool {
4012    if matches!(err, Error::DiskFull | Error::StorageFull) {
4013        return true;
4014    }
4015
4016    // A stage failure keeps the error it wrapped, so classify by type rather
4017    // than by the rendered message (backlog#1827 T2). The substring fallback
4018    // stays for errors that reached here through some other wrapper.
4019    if let Some(source) = data_movement::data_movement_stage_source(err) {
4020        return is_decommission_target_capacity_error(source);
4021    }
4022
4023    let message = err.to_string();
4024    let disk_full = Error::DiskFull.to_string();
4025    let storage_full = Error::StorageFull.to_string();
4026    message.contains(&disk_full) || message.contains(&storage_full)
4027}
4028
4029fn should_cleanup_decommission_source_entry(decommissioned: usize, total_versions: usize, expired: usize) -> bool {
4030    decommissioned.saturating_add(expired) == total_versions
4031}
4032
4033fn should_fail_decommission_pool_after_exhausted_source_changed(exhausted_entries: usize) -> bool {
4034    exhausted_entries > DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT
4035}
4036
4037#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4038enum DecommissionEntryAttemptOutcome {
4039    Complete,
4040    SourceChanged,
4041}
4042
4043#[cfg(test)]
4044pub(crate) type DecommissionTestFaultDecision = Arc<dyn Fn(&'static str, &str, &str, usize, bool) -> bool + Send + Sync>;
4045
4046#[cfg(test)]
4047static DECOMMISSION_TEST_FAULT_HOOK: std::sync::OnceLock<std::sync::Mutex<Option<DecommissionTestFaultDecision>>> =
4048    std::sync::OnceLock::new();
4049
4050#[cfg(test)]
4051pub(crate) struct DecommissionTestFaultGuard(DecommissionTestFaultDecision);
4052
4053#[cfg(test)]
4054impl DecommissionTestFaultGuard {
4055    pub(crate) fn install(decision: DecommissionTestFaultDecision) -> Self {
4056        let mut slot = DECOMMISSION_TEST_FAULT_HOOK
4057            .get_or_init(|| std::sync::Mutex::new(None))
4058            .lock()
4059            .expect("decommission test fault hook mutex should not poison");
4060        assert!(slot.is_none(), "decommission test fault hook must be unique");
4061        let stored = Arc::clone(&decision);
4062        *slot = Some(decision);
4063        Self(stored)
4064    }
4065}
4066
4067#[cfg(test)]
4068impl Drop for DecommissionTestFaultGuard {
4069    fn drop(&mut self) {
4070        let mut slot = DECOMMISSION_TEST_FAULT_HOOK
4071            .get_or_init(|| std::sync::Mutex::new(None))
4072            .lock()
4073            .expect("decommission test fault hook mutex should not poison");
4074        if slot.as_ref().is_some_and(|decision| Arc::ptr_eq(decision, &self.0)) {
4075            *slot = None;
4076        }
4077    }
4078}
4079
4080#[cfg(test)]
4081fn decommission_test_wrap_result<T>(
4082    stage: &'static str,
4083    bucket: &str,
4084    object: &str,
4085    attempt: usize,
4086    result: Result<T>,
4087) -> Result<T> {
4088    let decision = DECOMMISSION_TEST_FAULT_HOOK
4089        .get_or_init(|| std::sync::Mutex::new(None))
4090        .lock()
4091        .expect("decommission test fault hook mutex should not poison")
4092        .clone();
4093    let inject = decision.is_some_and(|decision| decision(stage, bucket, object, attempt, result.is_ok()));
4094    if result.is_ok() && inject {
4095        return Err(Error::other(format!(
4096            "injected decommission test fault at {stage} attempt {attempt} for {bucket}/{object}"
4097        )));
4098    }
4099    result
4100}
4101
4102#[cfg(test)]
4103pub(crate) type DecommissionCleanupMutationHook = Arc<dyn Fn(&str, &str, usize) -> BoxFuture<'static, ()> + Send + Sync>;
4104
4105#[cfg(test)]
4106static DECOMMISSION_CLEANUP_MUTATION_HOOK: std::sync::OnceLock<std::sync::Mutex<Option<DecommissionCleanupMutationHook>>> =
4107    std::sync::OnceLock::new();
4108
4109#[cfg(test)]
4110pub(crate) struct DecommissionCleanupMutationGuard(DecommissionCleanupMutationHook);
4111
4112#[cfg(test)]
4113impl DecommissionCleanupMutationGuard {
4114    pub(crate) fn install(hook: DecommissionCleanupMutationHook) -> Self {
4115        let mut slot = DECOMMISSION_CLEANUP_MUTATION_HOOK
4116            .get_or_init(|| std::sync::Mutex::new(None))
4117            .lock()
4118            .expect("decommission cleanup mutation hook mutex should not poison");
4119        assert!(slot.is_none(), "decommission cleanup mutation hook must be unique");
4120        let stored = Arc::clone(&hook);
4121        *slot = Some(hook);
4122        Self(stored)
4123    }
4124}
4125
4126#[cfg(test)]
4127impl Drop for DecommissionCleanupMutationGuard {
4128    fn drop(&mut self) {
4129        let mut slot = DECOMMISSION_CLEANUP_MUTATION_HOOK
4130            .get_or_init(|| std::sync::Mutex::new(None))
4131            .lock()
4132            .expect("decommission cleanup mutation hook mutex should not poison");
4133        if slot.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.0)) {
4134            *slot = None;
4135        }
4136    }
4137}
4138
4139#[cfg(test)]
4140async fn run_decommission_cleanup_mutation_hook(bucket: &str, object: &str, attempt: usize) {
4141    let hook = DECOMMISSION_CLEANUP_MUTATION_HOOK
4142        .get_or_init(|| std::sync::Mutex::new(None))
4143        .lock()
4144        .expect("decommission cleanup mutation hook mutex should not poison")
4145        .clone();
4146    if let Some(hook) = hook {
4147        hook(bucket, object, attempt).await;
4148    }
4149}
4150
4151const DECOMMISSION_FREE_VERSION_MIGRATED_REASON: &str = "tier_free_version_migrated";
4152const DECOMMISSION_FREE_VERSION_CONSUMED_REASON: &str = "tier_free_version_already_consumed";
4153const DECOMMISSION_FREE_VERSION_RETAINED_REASON: &str = "tier_free_version_migration_failed";
4154const DECOMMISSION_FREE_VERSION_SWEEP_REASON: &str = "tier_free_version_unresolved_after_decommission";
4155const DECOMMISSION_FREE_VERSION_DISPOSITION_REASON: &str = "tier_free_version_disposition_recorded";
4156
4157#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
4158struct DecommissionFreeVersionDisposition {
4159    migrated: usize,
4160    consumed: usize,
4161    retained: usize,
4162}
4163
4164impl DecommissionFreeVersionDisposition {
4165    fn record_migrated(&mut self) {
4166        self.migrated += 1;
4167    }
4168
4169    fn record_consumed(&mut self) {
4170        self.consumed += 1;
4171    }
4172
4173    fn record_retained(&mut self) {
4174        self.retained += 1;
4175    }
4176
4177    fn total(self) -> usize {
4178        self.migrated.saturating_add(self.consumed).saturating_add(self.retained)
4179    }
4180}
4181
4182enum DecommissionFreeVersionAttempt {
4183    Migrated,
4184    Consumed,
4185    CapacityFailure(Error),
4186    Retry(Error),
4187}
4188
4189fn classify_decommission_free_version_attempt(result: Result<()>) -> DecommissionFreeVersionAttempt {
4190    match result {
4191        Ok(()) => DecommissionFreeVersionAttempt::Migrated,
4192        Err(err) if is_decommission_copy_cleanup_safe_error(&err) => DecommissionFreeVersionAttempt::Consumed,
4193        Err(err) if is_decommission_target_capacity_error(&err) => DecommissionFreeVersionAttempt::CapacityFailure(err),
4194        Err(err) => DecommissionFreeVersionAttempt::Retry(err),
4195    }
4196}
4197
4198#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4199#[allow(
4200    dead_code,
4201    reason = "terminal-state classification asserted by this file's tests (backlog#1823)"
4202)]
4203enum DecommissionTerminalState {
4204    Completed,
4205    Failed,
4206}
4207
4208#[allow(
4209    dead_code,
4210    reason = "terminal-state classification asserted by this file's tests (backlog#1823)"
4211)]
4212fn classify_decommission_terminal_state(failed_items_present: bool) -> DecommissionTerminalState {
4213    if failed_items_present {
4214        DecommissionTerminalState::Failed
4215    } else {
4216        DecommissionTerminalState::Completed
4217    }
4218}
4219
4220fn should_preserve_decommission_canceled_state(meta_canceled: bool, _cancel_signal: bool) -> bool {
4221    meta_canceled
4222}
4223
4224fn should_continue_decommission_queue(meta: &PoolMeta, idx: usize) -> bool {
4225    meta.pools
4226        .get(idx)
4227        .and_then(|pool| pool.decommission.as_ref())
4228        .is_some_and(|info| info.complete && !info.failed && !info.canceled)
4229}
4230
4231fn decommission_cancel_signal_result(cancel_signal: bool) -> Result<()> {
4232    if cancel_signal {
4233        Err(StorageError::OperationCanceled)
4234    } else {
4235        Ok(())
4236    }
4237}
4238
4239fn is_decommission_cancel_requested(cancel_signal: bool, pool: Option<&PoolStatus>) -> bool {
4240    cancel_signal
4241        || pool
4242            .and_then(|pool| pool.decommission.as_ref())
4243            .is_some_and(|info| info.canceled)
4244}
4245
4246fn should_skip_canceled_decommission_routine(cancel_signal: bool, pool: Option<&PoolStatus>) -> bool {
4247    cancel_signal
4248        && pool
4249            .and_then(|pool| pool.decommission.as_ref())
4250            .is_some_and(|info| info.canceled)
4251}
4252
4253async fn run_decommission_buckets_bounded<F>(
4254    rx: CancellationToken,
4255    buckets: Vec<DecomBucketInfo>,
4256    limit: usize,
4257    mut start_bucket: F,
4258) -> Result<()>
4259where
4260    F: FnMut(DecomBucketInfo, CancellationToken) -> BoxFuture<'static, Result<()>>,
4261{
4262    let mut pending = buckets.into_iter();
4263    let mut active: FuturesUnordered<BoxFuture<'static, Result<()>>> = FuturesUnordered::new();
4264    let mut first_err = None;
4265    let limit = limit.max(1);
4266
4267    for _ in 0..limit {
4268        let Some(bucket) = pending.next() else {
4269            break;
4270        };
4271
4272        active.push(start_bucket(bucket, rx.clone()));
4273    }
4274
4275    while let Some(result) = active.next().await {
4276        if let Err(err) = result {
4277            rx.cancel();
4278            if first_err.is_none() {
4279                first_err = Some(err);
4280            }
4281            continue;
4282        }
4283
4284        if first_err.is_some() || rx.is_cancelled() {
4285            continue;
4286        }
4287
4288        let Some(bucket) = pending.next() else {
4289            continue;
4290        };
4291
4292        active.push(start_bucket(bucket, rx.clone()));
4293    }
4294
4295    if first_err.is_none() && rx.is_cancelled() && pending.len() > 0 {
4296        return decommission_cancel_signal_result(true);
4297    }
4298
4299    if let Some(err) = first_err {
4300        return Err(err);
4301    }
4302
4303    Ok(())
4304}
4305
4306async fn run_decommission_phases<F>(
4307    rx: CancellationToken,
4308    regular_buckets: Vec<DecomBucketInfo>,
4309    meta_buckets: Vec<DecomBucketInfo>,
4310    bucket_concurrency: usize,
4311    mut start_bucket: F,
4312) -> Result<()>
4313where
4314    F: FnMut(DecomBucketInfo, CancellationToken) -> BoxFuture<'static, Result<()>>,
4315{
4316    decommission_cancel_signal_result(rx.is_cancelled())?;
4317
4318    for bucket in meta_buckets {
4319        decommission_cancel_signal_result(rx.is_cancelled())?;
4320        start_bucket(bucket, rx.clone()).await?;
4321    }
4322
4323    decommission_cancel_signal_result(rx.is_cancelled())?;
4324
4325    if bucket_concurrency <= 1 {
4326        for bucket in regular_buckets {
4327            decommission_cancel_signal_result(rx.is_cancelled())?;
4328            start_bucket(bucket, rx.clone()).await?;
4329        }
4330        return Ok(());
4331    }
4332
4333    run_decommission_buckets_bounded(rx, regular_buckets, bucket_concurrency, start_bucket).await
4334}
4335
4336#[cfg(test)]
4337async fn wait_decommission_worker_drain(workers: &Semaphore, limit: usize) -> Result<()> {
4338    let permits = u32::try_from(limit)
4339        .map_err(|_| Error::other(format!("decommission worker limit {limit} exceeds semaphore drain capacity")))?;
4340    let _drain = workers
4341        .acquire_many(permits)
4342        .await
4343        .map_err(|err| Error::other(format!("decommission worker drain failed: {err}")))?;
4344    Ok(())
4345}
4346
4347fn should_reject_decommission_cancel_as_terminal(complete: bool, failed: bool) -> bool {
4348    complete || failed
4349}
4350
4351fn should_retry_decommission_cancel_reload(changed: bool, already_canceled: bool) -> bool {
4352    changed || already_canceled
4353}
4354
4355fn ensure_decommission_cancel_allowed(pool_present: bool, decommission_present: bool, terminal: bool) -> Result<()> {
4356    if !pool_present {
4357        return Err(invalid_decommission_request("failed to cancel decommission: target pool was not found"));
4358    }
4359
4360    if !decommission_present || terminal {
4361        return Err(StorageError::DecommissionNotStarted);
4362    }
4363
4364    Ok(())
4365}
4366
4367fn ensure_decommission_clear_allowed(
4368    pool_present: bool,
4369    decommission_present: bool,
4370    complete: bool,
4371    failed: bool,
4372    canceled: bool,
4373    unresolved_entries: usize,
4374) -> Result<()> {
4375    if !pool_present {
4376        return Err(invalid_decommission_request("failed to clear decommission: target pool was not found"));
4377    }
4378
4379    if !decommission_present {
4380        return Err(StorageError::DecommissionNotStarted);
4381    }
4382
4383    if complete {
4384        return Err(StorageError::DecommissionNotStarted);
4385    }
4386
4387    if !failed && !canceled {
4388        return Err(StorageError::DecommissionAlreadyRunning);
4389    }
4390
4391    if unresolved_entries > 0 {
4392        return Err(invalid_decommission_request(format!(
4393            "failed to clear decommission: {unresolved_entries} unresolved listing entries must be reconciled by retrying decommission"
4394        )));
4395    }
4396
4397    Ok(())
4398}
4399
4400fn ensure_decommission_terminal_operation_supported(single_pool: bool, operation: &str) -> Result<()> {
4401    if single_pool {
4402        return Err(invalid_decommission_request(format!(
4403            "failed to {operation}: single pool deployments do not support decommission"
4404        )));
4405    }
4406
4407    Ok(())
4408}
4409
4410fn validate_start_decommission_request(indices: &[usize], single_pool: bool) -> Result<()> {
4411    if indices.is_empty() {
4412        return Err(invalid_decommission_request(
4413            "failed to start decommission: no target pools were provided",
4414        ));
4415    }
4416
4417    ensure_decommission_terminal_operation_supported(single_pool, "start decommission")
4418}
4419
4420fn require_decommission_store<T>(store: Option<T>, operation: &str) -> Result<T> {
4421    store.ok_or_else(|| Error::other(format!("failed to {operation}: store not initialized")))
4422}
4423
4424fn ensure_decommission_listing_disks_available(has_disks: bool, bucket: &str) -> Result<()> {
4425    if !has_disks {
4426        return Err(Error::other(format!(
4427            "failed to list objects to decommission for bucket {bucket}: no disks available"
4428        )));
4429    }
4430
4431    Ok(())
4432}
4433
4434#[derive(Debug, Clone, Serialize, Deserialize)]
4435pub struct PoolStatus {
4436    #[serde(rename = "id")]
4437    pub id: usize,
4438    #[serde(rename = "cmdline")]
4439    pub cmd_line: String,
4440    #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
4441    pub last_update: OffsetDateTime,
4442    #[serde(rename = "decommissionInfo")]
4443    pub decommission: Option<PoolDecommissionInfo>,
4444}
4445
4446#[derive(Debug, Clone, Default, Serialize, Deserialize)]
4447pub struct PoolMeta {
4448    pub version: u16,
4449    pub pools: Vec<PoolStatus>,
4450    pub dont_save: bool,
4451}
4452
4453#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4454struct PoolMetaRevision {
4455    version: u16,
4456    cluster_id: Option<uuid::Uuid>,
4457    epoch: u64,
4458    generation: u64,
4459    transaction_id: Option<uuid::Uuid>,
4460}
4461
4462impl PoolMetaRevision {
4463    fn legacy(version: u16) -> Self {
4464        Self {
4465            version,
4466            cluster_id: None,
4467            epoch: 0,
4468            generation: 0,
4469            transaction_id: None,
4470        }
4471    }
4472
4473    fn is_generation_protocol(self) -> bool {
4474        self.version == POOL_META_GENERATION_VERSION
4475    }
4476}
4477
4478#[derive(Debug, Clone)]
4479struct PoolMetaCommittedCandidate {
4480    canonical: Vec<u8>,
4481    meta: PoolMeta,
4482    revision: PoolMetaRevision,
4483}
4484
4485#[derive(Debug)]
4486enum PoolMetaReplica {
4487    Missing,
4488    Valid {
4489        raw: Vec<u8>,
4490        canonical: Vec<u8>,
4491        meta: PoolMeta,
4492        revision: PoolMetaRevision,
4493        committed: bool,
4494        previous: Option<Box<PoolMetaCommittedCandidate>>,
4495    },
4496    Corrupt(String),
4497    Incompatible(String),
4498    Unreadable(Arc<Error>),
4499}
4500
4501#[derive(Debug, Clone)]
4502enum PoolMetaCasToken {
4503    Missing,
4504    Existing(String),
4505    Unsafe,
4506}
4507
4508#[derive(Debug)]
4509struct PoolMetaReplicaRead {
4510    replica: PoolMetaReplica,
4511    cas: PoolMetaCasToken,
4512}
4513
4514impl From<PoolMetaReplica> for PoolMetaReplicaRead {
4515    fn from(replica: PoolMetaReplica) -> Self {
4516        Self {
4517            replica,
4518            cas: PoolMetaCasToken::Unsafe,
4519        }
4520    }
4521}
4522
4523#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4524pub(crate) struct PoolMetaReplicaState {
4525    pub(crate) needs_repair: bool,
4526    pub(crate) repair_write_safe: bool,
4527}
4528
4529impl PoolMetaReplicaState {
4530    pub(crate) fn ensure_write_safe(self, operation: &str) -> Result<()> {
4531        if self.repair_write_safe {
4532            return Ok(());
4533        }
4534        Err(Error::other(format!(
4535            "{operation}: pool metadata update cannot overwrite an unreadable replica"
4536        )))
4537    }
4538}
4539
4540#[derive(Debug, Clone, Default)]
4541pub(crate) struct PoolMetaWriteState {
4542    write_blocked: bool,
4543    aborted_transaction: Arc<AtomicBool>,
4544    block_context: Option<crate::error::PoolMetadataError>,
4545    transaction_failure: Arc<std::sync::Mutex<Option<crate::error::PoolMetadataError>>>,
4546    active_transactions: Arc<AtomicUsize>,
4547    recovery_failure: Option<Arc<Error>>,
4548    block_started_at: Arc<std::sync::Mutex<Option<OffsetDateTime>>>,
4549    expected_cluster_id: Option<uuid::Uuid>,
4550    cluster_epoch: Option<u64>,
4551    pool_meta_absent: bool,
4552    bootstrap_authority: PoolMetaBootstrapAuthority,
4553    /// First-hand bootstrap authority this process holds for each pool
4554    /// (index = pool index): `Fresh` only for pools it formatted itself,
4555    /// `LegacyAdoption` only for pools whose migration it verified. Empty when
4556    /// the caller tracks deployment-wide authority only.
4557    pool_bootstrap_authorities: Vec<PoolMetaBootstrapAuthority>,
4558    /// Whether this process hosts the first endpoint of the first pool and is
4559    /// therefore the only writer allowed to publish the initial `pool.bin`.
4560    /// `None` when the caller did not say; unknown writers are treated as
4561    /// elected so every fail-closed rule still applies to them.
4562    elected_bootstrap_writer: Option<bool>,
4563    identity_initialized: Option<bool>,
4564    identity_fresh_bootstrap_nonce: Option<uuid::Uuid>,
4565    identity_needs_repair: bool,
4566    /// At least one pool has no identity replica at all.
4567    identity_replicas_missing: bool,
4568    /// At least one pool has a replica that is present but not a valid identity.
4569    identity_replicas_invalid: bool,
4570}
4571
4572/// Why an all-missing `pool.bin` set may not be initialized right now.
4573enum MissingMetadataRejection {
4574    /// Another node still has to act (mint, attest, or publish); retrying the
4575    /// startup loop is the remedy, so the write gate stays open.
4576    BootstrapPending(Error),
4577    /// The durable state contradicts a fresh bootstrap; writes stay blocked
4578    /// until an operator recovers the metadata.
4579    RecoveryRequired(Error),
4580}
4581
4582impl MissingMetadataRejection {
4583    fn into_error(self) -> Error {
4584        match self {
4585            Self::BootstrapPending(err) | Self::RecoveryRequired(err) => err,
4586        }
4587    }
4588}
4589
4590#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
4591pub(crate) enum PoolMetaBootstrapAuthority {
4592    #[default]
4593    None,
4594    Fresh,
4595    LegacyAdoption,
4596}
4597
4598impl PoolMetaBootstrapAuthority {
4599    pub(crate) fn combine_across_pools(self, other: Self) -> Self {
4600        if self == other { self } else { Self::None }
4601    }
4602
4603    fn is_proven(self) -> bool {
4604        !matches!(self, Self::None)
4605    }
4606}
4607
4608impl PoolMetaWriteState {
4609    #[cfg(test)]
4610    pub(crate) fn for_startup(cluster_id: uuid::Uuid, fresh_bootstrap_proven: bool) -> Self {
4611        let bootstrap_authority = if fresh_bootstrap_proven {
4612            PoolMetaBootstrapAuthority::Fresh
4613        } else {
4614            PoolMetaBootstrapAuthority::None
4615        };
4616        Self::for_startup_with_bootstrap_authority(cluster_id, bootstrap_authority)
4617    }
4618
4619    #[cfg(test)]
4620    pub(crate) fn for_startup_with_bootstrap_authority(
4621        cluster_id: uuid::Uuid,
4622        bootstrap_authority: PoolMetaBootstrapAuthority,
4623    ) -> Self {
4624        Self {
4625            expected_cluster_id: Some(cluster_id),
4626            bootstrap_authority,
4627            ..Default::default()
4628        }
4629    }
4630
4631    /// Startup state for a process that loaded every pool format itself and
4632    /// remembers, per pool, whether it created (or adopted) that pool
4633    /// first-hand. Deployment-wide authority is the conjunction across pools:
4634    /// any pool this process merely read yields `None`, exactly as before.
4635    pub(crate) fn for_startup_with_pool_bootstrap_authorities(
4636        cluster_id: uuid::Uuid,
4637        pool_bootstrap_authorities: Vec<PoolMetaBootstrapAuthority>,
4638        elected_bootstrap_writer: bool,
4639    ) -> Self {
4640        let bootstrap_authority = pool_bootstrap_authorities
4641            .iter()
4642            .copied()
4643            .reduce(PoolMetaBootstrapAuthority::combine_across_pools)
4644            .unwrap_or_default();
4645        Self {
4646            expected_cluster_id: Some(cluster_id),
4647            bootstrap_authority,
4648            pool_bootstrap_authorities,
4649            elected_bootstrap_writer: Some(elected_bootstrap_writer),
4650            ..Default::default()
4651        }
4652    }
4653
4654    pub(crate) fn bootstrap_identity_proven(&self) -> bool {
4655        self.bootstrap_authority.is_proven()
4656    }
4657
4658    fn pool_bootstrap_authority_proven(&self, pool_idx: usize) -> bool {
4659        self.pool_bootstrap_authorities
4660            .get(pool_idx)
4661            .is_some_and(|authority| authority.is_proven())
4662    }
4663
4664    /// Pools this process formatted or adopted first-hand during this startup.
4665    pub(crate) fn attested_pool_indices(&self) -> Vec<usize> {
4666        self.pool_bootstrap_authorities
4667            .iter()
4668            .enumerate()
4669            .filter(|(_, authority)| authority.is_proven())
4670            .map(|(pool_idx, _)| pool_idx)
4671            .collect()
4672    }
4673
4674    /// Deployment-level proof assembled from per-pool creators: this process
4675    /// created the first pool itself, and every pool replica carries the same
4676    /// pending identity. A pending replica is only ever written by the process
4677    /// that formatted that pool with first-hand proof (see
4678    /// [`PoolMetaIdentityWriteScope::Pools`]), so a complete, agreeing pending
4679    /// set proves that every pool joined this bootstrap fresh. A missing,
4680    /// corrupt, or disagreeing replica keeps the writer fail-closed, and a
4681    /// restart without first-hand proof never reopens bootstrap on its own.
4682    fn pending_identity_attested_by_every_pool(&self) -> bool {
4683        self.elected_bootstrap_writer == Some(true)
4684            && self.pool_bootstrap_authority_proven(0)
4685            && self.identity_initialized == Some(false)
4686            && !self.identity_needs_repair
4687            && self.identity_fresh_bootstrap_nonce.is_some()
4688    }
4689
4690    /// The elected writer minted (or holds) the nonce and the only thing
4691    /// standing between it and a complete attestation is a pool whose creator
4692    /// has not written its replica yet. Corrupt replicas are never transient.
4693    fn awaiting_creator_attestation(&self) -> bool {
4694        self.elected_bootstrap_writer == Some(true)
4695            && self.pool_bootstrap_authority_proven(0)
4696            && self.identity_initialized == Some(false)
4697            && self.identity_fresh_bootstrap_nonce.is_some()
4698            && self.identity_needs_repair
4699            && self.identity_replicas_missing
4700            && !self.identity_replicas_invalid
4701    }
4702
4703    fn is_non_elected_bootstrap_observer(&self) -> bool {
4704        self.elected_bootstrap_writer == Some(false)
4705    }
4706
4707    pub(crate) fn identity_is_pending(&self) -> bool {
4708        self.identity_initialized == Some(false)
4709    }
4710
4711    #[cfg(test)]
4712    pub(crate) fn aborted_transaction_latch_for_test(&self) -> Arc<AtomicBool> {
4713        Arc::clone(&self.aborted_transaction)
4714    }
4715
4716    #[cfg(test)]
4717    pub(crate) fn independent_clone_for_test(&self) -> Self {
4718        let mut cloned = self.clone();
4719        cloned.aborted_transaction = Arc::new(AtomicBool::new(self.aborted_transaction.load(Ordering::Acquire)));
4720        cloned.transaction_failure = Arc::new(std::sync::Mutex::new(
4721            self.transaction_failure
4722                .lock()
4723                .unwrap_or_else(std::sync::PoisonError::into_inner)
4724                .clone(),
4725        ));
4726        cloned.active_transactions = Arc::new(AtomicUsize::new(0));
4727        cloned.block_started_at = Arc::new(std::sync::Mutex::new(
4728            *self
4729                .block_started_at
4730                .lock()
4731                .unwrap_or_else(std::sync::PoisonError::into_inner),
4732        ));
4733        cloned
4734    }
4735
4736    #[cfg(any(test, feature = "test-util"))]
4737    fn for_test_bootstrap() -> Self {
4738        Self {
4739            bootstrap_authority: PoolMetaBootstrapAuthority::Fresh,
4740            identity_initialized: Some(false),
4741            identity_fresh_bootstrap_nonce: Some(uuid::Uuid::new_v4()),
4742            ..Default::default()
4743        }
4744    }
4745
4746    pub(crate) fn observe_replicas(&mut self, replica_state: PoolMetaReplicaState) {
4747        if !replica_state.repair_write_safe {
4748            self.block_writes();
4749        }
4750    }
4751
4752    fn block_writes(&mut self) {
4753        self.block_with_context(pool_metadata_error(
4754            crate::error::PoolMetadataFailure::RecoveryRequired,
4755            "validation",
4756            None,
4757        ));
4758    }
4759
4760    fn block_with_context(&mut self, mut context: crate::error::PoolMetadataError) {
4761        if !self.write_blocked {
4762            record_pool_meta_block_once(&self.block_started_at, &mut context);
4763            self.block_context = Some(context);
4764        } else if let Some(previous) = &self.block_context
4765            && previous.source.is_none()
4766            && context.source.is_some()
4767        {
4768            context.since = previous.since;
4769            self.block_context = Some(context);
4770        }
4771        self.write_blocked = true;
4772    }
4773
4774    pub(crate) fn block_writes_after_fence_loss(&mut self) {
4775        self.block_with_context(pool_metadata_error(crate::error::PoolMetadataFailure::FenceLost, "format_heal", None));
4776    }
4777
4778    fn arm_transaction(&self) -> PoolMetaTransactionArm {
4779        self.active_transactions.fetch_add(1, Ordering::SeqCst);
4780        PoolMetaTransactionArm {
4781            aborted_transaction: Arc::clone(&self.aborted_transaction),
4782            failure: Arc::clone(&self.transaction_failure),
4783            active_transactions: Arc::clone(&self.active_transactions),
4784            block_started_at: Arc::clone(&self.block_started_at),
4785            phase: None,
4786            source: None,
4787        }
4788    }
4789
4790    fn observe_selection(&mut self, selection: &PoolMetaSelection) -> Result<()> {
4791        self.pool_meta_absent = selection.absent;
4792        self.validate_selection(selection)?;
4793        if self.cluster_epoch.is_none()
4794            && let Some((_, metadata_epoch)) = selection.generation_identity
4795        {
4796            self.cluster_epoch = Some(metadata_epoch);
4797        }
4798        Ok(())
4799    }
4800
4801    fn validate_selection(&self, selection: &PoolMetaSelection) -> Result<()> {
4802        if let Some(expected_cluster_id) = self.expected_cluster_id
4803            && let Some((cluster_id, _)) = selection.generation_identity
4804            && cluster_id != expected_cluster_id
4805        {
4806            return Err(Error::other(format!(
4807                "pool metadata incompatible: cluster identity {cluster_id} does not match deployment {expected_cluster_id}"
4808            )));
4809        }
4810        if let Some(identity_epoch) = self.cluster_epoch
4811            && let Some((_, metadata_epoch)) = selection.generation_identity
4812            && metadata_epoch != identity_epoch
4813        {
4814            return Err(Error::other(format!(
4815                "pool metadata recovery required: committed epoch {} does not match cluster identity epoch {identity_epoch}",
4816                metadata_epoch
4817            )));
4818        }
4819        Ok(())
4820    }
4821
4822    fn observe_identity(&mut self, selection: &PoolMetaIdentitySelection) -> Result<()> {
4823        self.identity_needs_repair = selection.needs_repair;
4824        self.identity_initialized = selection.identity.map(|identity| identity.initialized);
4825        self.identity_fresh_bootstrap_nonce = selection.identity.and_then(|identity| identity.fresh_bootstrap_nonce);
4826        self.identity_replicas_missing = selection
4827            .cas_tokens
4828            .iter()
4829            .any(|token| matches!(token, PoolMetaCasToken::Missing));
4830        self.identity_replicas_invalid = selection
4831            .valid_replicas
4832            .iter()
4833            .zip(&selection.cas_tokens)
4834            .any(|(valid, token)| !valid && !matches!(token, PoolMetaCasToken::Missing));
4835        if let Some(identity) = selection.identity {
4836            if identity.initialized {
4837                self.bootstrap_authority = PoolMetaBootstrapAuthority::None;
4838                self.pool_bootstrap_authorities.clear();
4839            }
4840            if let Some(metadata_epoch) = self.cluster_epoch
4841                && metadata_epoch != identity.epoch
4842            {
4843                self.block_writes();
4844                return Err(Error::other(format!(
4845                    "pool metadata recovery required: metadata epoch {metadata_epoch} does not match cluster identity epoch {}",
4846                    identity.epoch
4847                )));
4848            }
4849            self.cluster_epoch = Some(identity.epoch);
4850        }
4851        if !selection.repair_write_safe {
4852            self.block_writes();
4853        }
4854        Ok(())
4855    }
4856
4857    pub(crate) fn ensure_missing_metadata_can_initialize(&mut self) -> Result<()> {
4858        if !self.pool_meta_absent {
4859            return Ok(());
4860        }
4861        match self.validate_missing_metadata_can_initialize() {
4862            Ok(()) => Ok(()),
4863            Err(MissingMetadataRejection::BootstrapPending(err)) => Err(err),
4864            Err(MissingMetadataRejection::RecoveryRequired(err)) => {
4865                Err(block_pool_meta_validation(self, err, "metadata_absence"))
4866            }
4867        }
4868    }
4869
4870    fn validate_missing_metadata_can_initialize(&self) -> std::result::Result<(), MissingMetadataRejection> {
4871        use MissingMetadataRejection::{BootstrapPending, RecoveryRequired};
4872        match self.identity_initialized {
4873            Some(false)
4874                if self.identity_fresh_bootstrap_nonce.is_some()
4875                    && (self.bootstrap_identity_proven() || self.pending_identity_attested_by_every_pool()) =>
4876            {
4877                Ok(())
4878            }
4879            Some(false) if self.awaiting_creator_attestation() => Err(BootstrapPending(Error::other(
4880                "pool metadata bootstrap pending: waiting for every pool creator to attest the pending cluster identity",
4881            ))),
4882            Some(false) if self.is_non_elected_bootstrap_observer() && self.identity_fresh_bootstrap_nonce.is_some() => {
4883                Err(BootstrapPending(Error::other(
4884                    "pool metadata bootstrap pending: waiting for the elected writer to publish the initial pool.bin",
4885                )))
4886            }
4887            Some(false) => Err(RecoveryRequired(Error::other(
4888                "pool metadata recovery required: pending cluster identity exists but this startup has no verified fresh-bootstrap proof, legacy-adoption proof, or complete per-pool creator attestation",
4889            ))),
4890            Some(true) => Err(RecoveryRequired(Error::other(
4891                "pool metadata recovery required: initialized cluster identity exists but every pool.bin replica is missing",
4892            ))),
4893            None if self.is_non_elected_bootstrap_observer() => Err(BootstrapPending(Error::other(
4894                "pool metadata bootstrap pending: waiting for the elected writer to establish the cluster identity",
4895            ))),
4896            None => Err(RecoveryRequired(Error::other(
4897                "pool metadata recovery required: no durable bootstrap identity or pool.bin replica is available",
4898            ))),
4899        }
4900    }
4901
4902    pub(crate) fn identity_requires_repair(&self) -> bool {
4903        self.expected_cluster_id.is_some() && (self.identity_needs_repair || self.identity_initialized != Some(true))
4904    }
4905
4906    pub(crate) fn ensure_write_safe(&self, operation: &str) -> Result<()> {
4907        if !self.write_blocked && !self.aborted_transaction.load(Ordering::SeqCst) {
4908            return Ok(());
4909        }
4910        let mut context = self
4911            .block_context
4912            .clone()
4913            .or_else(|| {
4914                self.transaction_failure
4915                    .lock()
4916                    .unwrap_or_else(std::sync::PoisonError::into_inner)
4917                    .clone()
4918            })
4919            .unwrap_or_else(|| pool_metadata_error(crate::error::PoolMetadataFailure::FenceLost, "external_fence", None));
4920        context.operation = operation.to_owned();
4921        Err(Error::other(context))
4922    }
4923}
4924
4925fn pool_metadata_error(
4926    kind: crate::error::PoolMetadataFailure,
4927    phase: &'static str,
4928    source: Option<Arc<Error>>,
4929) -> crate::error::PoolMetadataError {
4930    crate::error::PoolMetadataError {
4931        kind,
4932        operation: phase.to_owned(),
4933        phase,
4934        since: OffsetDateTime::now_utc(),
4935        source,
4936    }
4937}
4938
4939fn record_pool_meta_block_once(
4940    started_at: &std::sync::Mutex<Option<OffsetDateTime>>,
4941    context: &mut crate::error::PoolMetadataError,
4942) {
4943    let mut started_at = started_at.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
4944    if let Some(since) = *started_at {
4945        context.since = since;
4946        return;
4947    }
4948    *started_at = Some(context.since);
4949    error!(
4950        event = EVENT_DECOMMISSION_STATE,
4951        component = LOG_COMPONENT_ECSTORE,
4952        subsystem = LOG_SUBSYSTEM_POOLS,
4953        state = "pool_metadata_blocked",
4954        reason = context.kind.as_str(),
4955        phase = context.phase,
4956        blocked_since = %context.since,
4957        "Pool metadata writes blocked"
4958    );
4959    metrics::counter!("rustfs_pool_metadata_blocks_total", "reason" => context.kind.as_str()).increment(1);
4960}
4961
4962#[derive(Debug)]
4963struct PoolMetaTransactionArm {
4964    aborted_transaction: Arc<AtomicBool>,
4965    failure: Arc<std::sync::Mutex<Option<crate::error::PoolMetadataError>>>,
4966    active_transactions: Arc<AtomicUsize>,
4967    block_started_at: Arc<std::sync::Mutex<Option<OffsetDateTime>>>,
4968    phase: Option<&'static str>,
4969    source: Option<Arc<Error>>,
4970}
4971
4972impl PoolMetaTransactionArm {
4973    fn failure(&mut self, err: Error) -> Error {
4974        let Some(phase) = self.phase else {
4975            return err;
4976        };
4977        let source = Arc::new(err);
4978        self.source = Some(Arc::clone(&source));
4979        Error::other(pool_metadata_error(
4980            crate::error::PoolMetadataFailure::TransactionUnknown,
4981            phase,
4982            Some(source),
4983        ))
4984    }
4985
4986    fn disarm(&mut self) {
4987        self.phase = None;
4988    }
4989}
4990
4991impl Drop for PoolMetaTransactionArm {
4992    fn drop(&mut self) {
4993        if let Some(phase) = self.phase {
4994            let mut context =
4995                pool_metadata_error(crate::error::PoolMetadataFailure::TransactionUnknown, phase, self.source.take());
4996            let mut failure = self.failure.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
4997            if !self.aborted_transaction.swap(true, Ordering::SeqCst) {
4998                record_pool_meta_block_once(&self.block_started_at, &mut context);
4999                *failure = Some(context);
5000            }
5001        }
5002        self.active_transactions.fetch_sub(1, Ordering::SeqCst);
5003    }
5004}
5005
5006#[derive(Debug)]
5007struct PoolMetaSelection {
5008    meta: PoolMeta,
5009    revision: PoolMetaRevision,
5010    canonical: Option<Vec<u8>>,
5011    replica_state: PoolMetaReplicaState,
5012    cas_tokens: Vec<PoolMetaCasToken>,
5013    absent: bool,
5014    generation_protocol_observed: bool,
5015    generation_identity: Option<(uuid::Uuid, u64)>,
5016}
5017
5018fn classify_pool_meta_tuple_decode_error(kind: &str, err: rmp_serde::decode::Error) -> PoolMetaReplica {
5019    let truncated = matches!(
5020        &err,
5021        rmp_serde::decode::Error::InvalidMarkerRead(source)
5022            | rmp_serde::decode::Error::InvalidDataRead(source)
5023            if source.kind() == std::io::ErrorKind::UnexpectedEof
5024    );
5025    if truncated {
5026        PoolMetaReplica::Corrupt(format!("{kind} tuple payload is truncated: {err}"))
5027    } else {
5028        PoolMetaReplica::Incompatible(format!("{kind} tuple payload is not decodable: {err}"))
5029    }
5030}
5031
5032fn parse_pool_meta_uuid(value: &str, field: &str) -> Result<uuid::Uuid> {
5033    let parsed = uuid::Uuid::parse_str(value)
5034        .map_err(|err| Error::other(format!("pool metadata corrupt: invalid {field} `{value}`: {err}")))?;
5035    if parsed.is_nil() || parsed == uuid::Uuid::max() {
5036        return Err(Error::other(format!("pool metadata corrupt: {field} must be a non-reserved UUID")));
5037    }
5038    Ok(parsed)
5039}
5040
5041fn pool_meta_generation_revision(
5042    cluster_id: &str,
5043    epoch: u64,
5044    generation: u64,
5045    transaction_id: &str,
5046) -> Result<PoolMetaRevision> {
5047    if epoch == 0 || generation == 0 {
5048        return Err(Error::other(
5049            "pool metadata corrupt: version 3 epoch and generation must both be non-zero",
5050        ));
5051    }
5052    Ok(PoolMetaRevision {
5053        version: POOL_META_GENERATION_VERSION,
5054        cluster_id: Some(parse_pool_meta_uuid(cluster_id, "cluster identity")?),
5055        epoch,
5056        generation,
5057        transaction_id: Some(parse_pool_meta_uuid(transaction_id, "transaction id")?),
5058    })
5059}
5060
5061fn pool_meta_from_v3_statuses(version: u16, pools: Vec<PersistedPoolStatus>) -> Result<PoolMeta> {
5062    if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) {
5063        return Err(Error::other(format!(
5064            "pool metadata corrupt: version 3 previous snapshot has unsupported source version {version}"
5065        )));
5066    }
5067    let meta = PoolMeta {
5068        version,
5069        pools: pools.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?,
5070        dont_save: false,
5071    };
5072    validate_decommission_capacity_model_cohort(&meta)?;
5073    Ok(meta)
5074}
5075
5076fn pool_meta_previous_candidate(value: PersistedPoolMetaV3Previous) -> Result<PoolMetaCommittedCandidate> {
5077    let revision = match value.version {
5078        POOL_META_V1_VERSION | POOL_META_VERSION => {
5079            if value.cluster_id.is_some() || value.epoch != 0 || value.generation != 0 || value.transaction_id.is_some() {
5080                return Err(Error::other(
5081                    "pool metadata corrupt: legacy previous snapshot carries version 3 revision fields",
5082                ));
5083            }
5084            PoolMetaRevision::legacy(value.version)
5085        }
5086        POOL_META_GENERATION_VERSION => pool_meta_generation_revision(
5087            value
5088                .cluster_id
5089                .as_deref()
5090                .ok_or_else(|| Error::other("pool metadata corrupt: version 3 previous snapshot has no cluster identity"))?,
5091            value.epoch,
5092            value.generation,
5093            value
5094                .transaction_id
5095                .as_deref()
5096                .ok_or_else(|| Error::other("pool metadata corrupt: version 3 previous snapshot has no transaction id"))?,
5097        )?,
5098        version => {
5099            return Err(Error::other(format!(
5100                "pool metadata corrupt: previous snapshot has unsupported version {version}"
5101            )));
5102        }
5103    };
5104    let meta = pool_meta_from_v3_statuses(value.version, value.pools)?;
5105    let canonical = if revision.is_generation_protocol() {
5106        encode_pool_meta_v3_envelope(&meta, revision, true, None)?
5107    } else {
5108        meta.encode_config_data_for_v2_gate(true)?
5109    };
5110    Ok(PoolMetaCommittedCandidate {
5111        canonical,
5112        meta,
5113        revision,
5114    })
5115}
5116
5117fn decode_pool_meta_v3(data: Vec<u8>) -> PoolMetaReplica {
5118    let persisted = match rmp_serde::from_slice::<PersistedPoolMetaV3>(&data[4..]) {
5119        Ok(persisted) => persisted,
5120        Err(err) => return classify_pool_meta_tuple_decode_error("v3", err),
5121    };
5122    if persisted.version != POOL_META_GENERATION_VERSION {
5123        return PoolMetaReplica::Corrupt(format!(
5124            "v3 payload has version {}, expected {POOL_META_GENERATION_VERSION}",
5125            persisted.version
5126        ));
5127    }
5128    let revision = match pool_meta_generation_revision(
5129        &persisted.cluster_id,
5130        persisted.epoch,
5131        persisted.generation,
5132        &persisted.transaction_id,
5133    ) {
5134        Ok(revision) => revision,
5135        Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5136    };
5137    let meta = match pool_meta_from_v3_statuses(POOL_META_GENERATION_VERSION, persisted.pools) {
5138        Ok(meta) => meta,
5139        Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5140    };
5141    let previous = match persisted.previous.map(pool_meta_previous_candidate).transpose() {
5142        Ok(previous) => previous,
5143        Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5144    };
5145    if persisted.committed && previous.is_some() {
5146        return PoolMetaReplica::Corrupt("committed v3 payload unexpectedly retains a previous snapshot".to_string());
5147    }
5148    if !persisted.committed {
5149        match previous.as_ref() {
5150            Some(previous) if previous.revision.is_generation_protocol() => {
5151                if previous.revision.cluster_id != revision.cluster_id
5152                    || previous.revision.epoch != revision.epoch
5153                    || previous.revision.generation.checked_add(1) != Some(revision.generation)
5154                {
5155                    return PoolMetaReplica::Corrupt(
5156                        "pending v3 payload does not advance exactly one generation from its previous snapshot".to_string(),
5157                    );
5158                }
5159            }
5160            Some(_) if revision.generation != 1 => {
5161                return PoolMetaReplica::Corrupt(
5162                    "first v3 generation must be generation 1 when migrating a legacy snapshot".to_string(),
5163                );
5164            }
5165            None if revision.generation != 1 => {
5166                return PoolMetaReplica::Corrupt(
5167                    "pending v3 payload without a previous snapshot must be the initial generation".to_string(),
5168                );
5169            }
5170            _ => {}
5171        }
5172    }
5173    let canonical = match encode_pool_meta_v3_envelope(&meta, revision, true, None) {
5174        Ok(canonical) => canonical,
5175        Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5176    };
5177    PoolMetaReplica::Valid {
5178        raw: data,
5179        canonical,
5180        meta,
5181        revision,
5182        committed: persisted.committed,
5183        previous: previous.map(Box::new),
5184    }
5185}
5186
5187fn decode_pool_meta_replica(data: Vec<u8>) -> PoolMetaReplica {
5188    if data.len() <= 4 {
5189        return PoolMetaReplica::Corrupt("metadata payload is empty or truncated".to_string());
5190    }
5191
5192    let format = LittleEndian::read_u16(&data[0..2]);
5193    if format != POOL_META_FORMAT {
5194        return PoolMetaReplica::Incompatible(format!("unsupported format {format}"));
5195    }
5196    let version = LittleEndian::read_u16(&data[2..4]);
5197    if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) {
5198        return PoolMetaReplica::Incompatible(format!("unsupported version {version}"));
5199    }
5200    if version == POOL_META_GENERATION_VERSION {
5201        return decode_pool_meta_v3(data);
5202    }
5203
5204    let payload = &data[4..];
5205    let meta = match (version, rmp::decode::read_array_len(&mut &payload[..])) {
5206        (POOL_META_VERSION, Ok(2)) => match rmp_serde::from_slice::<PersistedPoolMeta>(payload) {
5207            Ok(meta) => match PoolMeta::try_from(meta) {
5208                Ok(meta) => meta,
5209                Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5210            },
5211            Err(err) => return classify_pool_meta_tuple_decode_error("current", err),
5212        },
5213        (POOL_META_V1_VERSION, Ok(2)) => match rmp_serde::from_slice::<PersistedPoolMetaV1>(payload) {
5214            Ok(meta) => match PoolMeta::try_from(meta) {
5215                Ok(meta) => meta,
5216                Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5217            },
5218            Err(err) => return classify_pool_meta_tuple_decode_error("v1", err),
5219        },
5220        // The older V1 tuple includes the runtime-only `dont_save` flag.
5221        (POOL_META_V1_VERSION, Ok(3)) => match rmp_serde::from_slice::<LegacyPoolMeta>(payload) {
5222            Ok(meta) => match PoolMeta::try_from(meta) {
5223                Ok(meta) => meta,
5224                Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5225            },
5226            Err(err) => return classify_pool_meta_tuple_decode_error("legacy v1", err),
5227        },
5228        (_, Ok(field_count)) if field_count < 2 => {
5229            return PoolMetaReplica::Corrupt(format!("pool metadata tuple has only {field_count} fields"));
5230        }
5231        (_, Ok(field_count)) => {
5232            return PoolMetaReplica::Incompatible(format!(
5233                "pool metadata version {version} tuple has unsupported field count {field_count}"
5234            ));
5235        }
5236        (_, Err(_)) => {
5237            let mut meta = PoolMeta::default();
5238            if let Err(err) = meta.load_from_config_data(data.clone()) {
5239                let reason = err.to_string();
5240                if reason.contains("unknown field") {
5241                    return PoolMetaReplica::Incompatible(format!("current-version payload uses unsupported fields: {reason}"));
5242                }
5243                return PoolMetaReplica::Corrupt(reason);
5244            }
5245            meta
5246        }
5247    };
5248
5249    match meta.encode_config_data_for_v2_gate(true) {
5250        Ok(canonical) => PoolMetaReplica::Valid {
5251            raw: data,
5252            canonical,
5253            meta,
5254            revision: PoolMetaRevision::legacy(version),
5255            committed: true,
5256            previous: None,
5257        },
5258        Err(err) => PoolMetaReplica::Corrupt(err.to_string()),
5259    }
5260}
5261
5262#[cfg(test)]
5263pub(crate) fn pool_meta_v3_commit_state_for_test(data: Vec<u8>) -> Result<(u64, bool)> {
5264    match decode_pool_meta_replica(data) {
5265        PoolMetaReplica::Valid { revision, committed, .. } if revision.is_generation_protocol() => {
5266            Ok((revision.generation, committed))
5267        }
5268        _ => Err(Error::other("test pool metadata is not a valid V3 replica")),
5269    }
5270}
5271
5272async fn read_pool_meta_replica<S>(pool: Arc<S>, no_lock: bool) -> PoolMetaReplicaRead
5273where
5274    S: EcstoreObjectIO,
5275{
5276    let result = if no_lock {
5277        read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME)
5278            .await
5279            .map(|(data, object_info)| (data, object_info.etag))
5280    } else {
5281        read_config_preserve_empty(pool, POOL_META_NAME)
5282            .await
5283            .map(|data| (data, None))
5284    };
5285    match result {
5286        Ok((data, etag)) => PoolMetaReplicaRead {
5287            replica: decode_pool_meta_replica(data),
5288            cas: etag
5289                .filter(|etag| !etag.trim().is_empty())
5290                .map(PoolMetaCasToken::Existing)
5291                .unwrap_or(PoolMetaCasToken::Unsafe),
5292        },
5293        Err(Error::ConfigNotFound) => PoolMetaReplicaRead {
5294            replica: PoolMetaReplica::Missing,
5295            cas: PoolMetaCasToken::Missing,
5296        },
5297        Err(err) => PoolMetaReplicaRead {
5298            replica: PoolMetaReplica::Unreadable(Arc::new(err)),
5299            cas: PoolMetaCasToken::Unsafe,
5300        },
5301    }
5302}
5303
5304fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<PoolMetaSelection> {
5305    if reads.is_empty() {
5306        return Err(Error::other("pool metadata recovery required: no storage pools available"));
5307    }
5308
5309    let cas_tokens = reads.iter().map(|read| read.cas.clone()).collect();
5310    let mut committed = Vec::<(usize, Vec<u8>, PoolMetaCommittedCandidate)>::new();
5311    let mut needs_repair = false;
5312    let mut repair_write_safe = true;
5313    let mut missing = 0usize;
5314    let mut unusable = Vec::new();
5315    let mut observed_version = POOL_META_V1_VERSION;
5316    let mut generation_protocol_observed = false;
5317    let mut generation_identity = None;
5318
5319    for (idx, read) in reads.into_iter().enumerate() {
5320        match read.replica {
5321            PoolMetaReplica::Missing => {
5322                missing += 1;
5323                needs_repair = true;
5324            }
5325            PoolMetaReplica::Corrupt(reason) => {
5326                needs_repair = true;
5327                unusable.push(format!("pool {idx} is corrupt: {reason}"));
5328            }
5329            PoolMetaReplica::Unreadable(reason) => {
5330                needs_repair = true;
5331                repair_write_safe = false;
5332                unusable.push(format!("pool {idx} is unreadable: {reason}"));
5333            }
5334            PoolMetaReplica::Incompatible(reason) => {
5335                return Err(Error::other(format!(
5336                    "pool metadata recovery required: pool {idx} is incompatible ({reason}); upgrade or restore a compatible replica without overwriting it"
5337                )));
5338            }
5339            PoolMetaReplica::Valid {
5340                raw,
5341                canonical,
5342                meta,
5343                revision,
5344                committed: is_committed,
5345                previous,
5346            } => {
5347                generation_protocol_observed |= revision.is_generation_protocol();
5348                if revision.is_generation_protocol() {
5349                    let identity = (
5350                        revision
5351                            .cluster_id
5352                            .expect("validated V3 revision should carry a cluster identity"),
5353                        revision.epoch,
5354                    );
5355                    if generation_identity.is_some_and(|current| current != identity) {
5356                        return Err(Error::other(
5357                            "pool metadata recovery required: V3 replicas disagree on cluster identity or epoch",
5358                        ));
5359                    }
5360                    generation_identity = Some(identity);
5361                }
5362                if is_committed {
5363                    observed_version = observed_version.max(meta.version);
5364                    committed.push((
5365                        idx,
5366                        raw,
5367                        PoolMetaCommittedCandidate {
5368                            canonical,
5369                            meta,
5370                            revision,
5371                        },
5372                    ));
5373                } else if let Some(previous) = previous {
5374                    observed_version = observed_version.max(previous.meta.version);
5375                    needs_repair = true;
5376                    committed.push((idx, raw, *previous));
5377                } else {
5378                    needs_repair = true;
5379                    unusable.push(format!("pool {idx} contains an uncommitted initial generation"));
5380                }
5381            }
5382        }
5383    }
5384
5385    if committed.is_empty() && missing > 0 && unusable.is_empty() {
5386        return Ok(PoolMetaSelection {
5387            meta: PoolMeta::default(),
5388            revision: PoolMetaRevision::legacy(0),
5389            canonical: None,
5390            replica_state: PoolMetaReplicaState {
5391                needs_repair: false,
5392                repair_write_safe: true,
5393            },
5394            cas_tokens,
5395            absent: true,
5396            generation_protocol_observed,
5397            generation_identity,
5398        });
5399    }
5400    if committed.is_empty() {
5401        return Err(Error::other(format!(
5402            "pool metadata recovery required: no valid committed replica is available ({})",
5403            unusable.join("; ")
5404        )));
5405    }
5406
5407    if committed
5408        .iter()
5409        .any(|(_, _, candidate)| candidate.revision.is_generation_protocol())
5410    {
5411        let highest = committed
5412            .iter()
5413            .filter(|(_, _, candidate)| candidate.revision.is_generation_protocol())
5414            .map(|(_, _, candidate)| candidate.revision.generation)
5415            .max()
5416            .ok_or_else(|| Error::other("pool metadata recovery required: no committed V3 generation is available"))?;
5417        let mut selected: Option<(usize, &PoolMetaCommittedCandidate)> = None;
5418        for (idx, _, candidate) in &committed {
5419            if !candidate.revision.is_generation_protocol() || candidate.revision.generation != highest {
5420                needs_repair = true;
5421                continue;
5422            }
5423            if let Some((selected_idx, selected_candidate)) = selected {
5424                if selected_candidate.canonical != candidate.canonical
5425                    || selected_candidate.revision.transaction_id != candidate.revision.transaction_id
5426                {
5427                    return Err(Error::other(format!(
5428                        "pool metadata recovery required: committed generation {highest} diverges between pools {selected_idx} and {idx}"
5429                    )));
5430                }
5431            } else {
5432                selected = Some((*idx, candidate));
5433            }
5434        }
5435        let (_, selected) = selected
5436            .ok_or_else(|| Error::other("pool metadata recovery required: highest V3 generation has no valid snapshot"))?;
5437        for (_, raw, candidate) in &committed {
5438            needs_repair |= candidate.canonical != selected.canonical || raw != &selected.canonical;
5439        }
5440        return Ok(PoolMetaSelection {
5441            meta: selected.meta.clone(),
5442            revision: selected.revision,
5443            canonical: Some(selected.canonical.clone()),
5444            replica_state: PoolMetaReplicaState {
5445                needs_repair,
5446                repair_write_safe,
5447            },
5448            cas_tokens,
5449            absent: false,
5450            generation_protocol_observed,
5451            generation_identity,
5452        });
5453    }
5454
5455    // Legacy V1/V2 has no durable generation. Pool zero remains the commit
5456    // record; divergent backups without it are ambiguous and fail closed.
5457    let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
5458    for (idx, raw, candidate) in committed {
5459        if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
5460            if selected_candidate.canonical != candidate.canonical {
5461                if selected_candidate.meta.version == candidate.meta.version && *selected_idx == 0 {
5462                    needs_repair = true;
5463                } else {
5464                    return Err(Error::other(format!(
5465                        "pool metadata recovery required: valid replicas in pools {selected_idx} and {idx} diverge; restore one matching pool.bin snapshot before restart"
5466                    )));
5467                }
5468            } else {
5469                needs_repair |= selected_raw != &raw;
5470            }
5471        } else {
5472            selected = Some((idx, raw, candidate));
5473        }
5474    }
5475    let (_, _, mut selected) =
5476        selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
5477    selected.meta.version = observed_version;
5478    selected.revision.version = observed_version;
5479    Ok(PoolMetaSelection {
5480        meta: selected.meta,
5481        revision: selected.revision,
5482        canonical: Some(selected.canonical),
5483        replica_state: PoolMetaReplicaState {
5484            needs_repair,
5485            repair_write_safe,
5486        },
5487        cas_tokens,
5488        absent: false,
5489        generation_protocol_observed,
5490        generation_identity,
5491    })
5492}
5493
5494#[cfg(test)]
5495fn select_pool_meta_replica(replicas: Vec<PoolMetaReplica>) -> Result<PoolMetaSelection> {
5496    select_pool_meta_replica_reads(
5497        replicas
5498            .into_iter()
5499            .map(|replica| PoolMetaReplicaRead {
5500                replica,
5501                cas: PoolMetaCasToken::Unsafe,
5502            })
5503            .collect(),
5504    )
5505}
5506
5507async fn read_pool_meta_replicas<S>(pools: Vec<Arc<S>>, no_lock: bool) -> Vec<PoolMetaReplicaRead>
5508where
5509    S: EcstoreObjectIO,
5510{
5511    let reads = join_all(pools.into_iter().map(|pool| read_pool_meta_replica(pool, no_lock))).await;
5512    #[cfg(feature = "e2e-test-hooks")]
5513    if STARTUP_CAS_OBSERVATION.try_with(|_| ()).is_ok() {
5514        let batch = uuid::Uuid::new_v4();
5515        for (pool, read) in reads.iter().enumerate() {
5516            let mut observation = serde_json::json!({
5517                "kind": "replica-read", "object": POOL_META_NAME, "batch": batch, "pool": pool,
5518                "cas": match &read.cas {
5519                    PoolMetaCasToken::Missing => "missing",
5520                    PoolMetaCasToken::Existing(_) => "existing",
5521                    PoolMetaCasToken::Unsafe => "unsafe",
5522                },
5523                "etag": match &read.cas { PoolMetaCasToken::Existing(etag) => Some(etag), _ => None },
5524            });
5525            match &read.replica {
5526                PoolMetaReplica::Valid {
5527                    raw,
5528                    canonical,
5529                    meta,
5530                    revision,
5531                    committed,
5532                    previous,
5533                } => {
5534                    observation["state"] = serde_json::json!("valid");
5535                    observation["committed"] = serde_json::json!(committed);
5536                    observation["version"] = serde_json::json!(revision.version);
5537                    observation["cluster_id"] = serde_json::json!(revision.cluster_id);
5538                    observation["epoch"] = serde_json::json!(revision.epoch);
5539                    observation["generation"] = serde_json::json!(revision.generation);
5540                    observation["transaction_id"] = serde_json::json!(revision.transaction_id);
5541                    observation["pool_count"] = serde_json::json!(meta.pools.len());
5542                    observation["payload_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(canonical)));
5543                    observation["raw_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(raw)));
5544                    match pool_meta_test_pools_sha256(meta) {
5545                        Ok(digest) => observation["persisted_pools_sha256"] = serde_json::json!(digest),
5546                        Err(err) => observation["observation_error"] = serde_json::json!(err.to_string()),
5547                    }
5548                    if let Some(previous) = previous {
5549                        match pool_meta_test_snapshot(&previous.meta, previous.revision, &previous.canonical) {
5550                            Ok(summary) => observation["previous"] = summary,
5551                            Err(err) => observation["observation_error"] = serde_json::json!(err.to_string()),
5552                        }
5553                    }
5554                }
5555                PoolMetaReplica::Missing => observation["state"] = serde_json::json!("missing"),
5556                PoolMetaReplica::Corrupt(_) => observation["state"] = serde_json::json!("corrupt"),
5557                PoolMetaReplica::Incompatible(_) => observation["state"] = serde_json::json!("incompatible"),
5558                PoolMetaReplica::Unreadable(_) => observation["state"] = serde_json::json!("unreadable"),
5559            }
5560            startup_cas_test_observe(observation);
5561        }
5562    }
5563    reads
5564}
5565
5566fn select_pool_meta_replicas_observing<R>(write_state: &mut PoolMetaWriteState, replicas: Vec<R>) -> Result<PoolMetaSelection>
5567where
5568    R: Into<PoolMetaReplicaRead>,
5569{
5570    let replicas = replicas.into_iter().map(Into::into).collect::<Vec<_>>();
5571    if let Some(err) = pool_meta_replica_read_failure(&replicas, "pool_read") {
5572        return Err(err);
5573    }
5574    match select_pool_meta_replica_reads(replicas) {
5575        Ok(selection) => {
5576            if let Err(err) = write_state.observe_selection(&selection) {
5577                return Err(block_pool_meta_validation(write_state, err, "pool_selection"));
5578            }
5579            Ok(selection)
5580        }
5581        Err(err) => Err(block_pool_meta_validation(write_state, err, "pool_selection")),
5582    }
5583}
5584
5585fn block_pool_meta_validation(write_state: &mut PoolMetaWriteState, err: Error, phase: &'static str) -> Error {
5586    let context = pool_metadata_error(crate::error::PoolMetadataFailure::RecoveryRequired, phase, Some(Arc::new(err)));
5587    write_state.block_with_context(context.clone());
5588    Error::other(context)
5589}
5590
5591fn pool_meta_replica_read_failure(replicas: &[PoolMetaReplicaRead], operation: &str) -> Option<Error> {
5592    replicas.iter().find_map(|read| match &read.replica {
5593        PoolMetaReplica::Unreadable(source) => {
5594            let mut context =
5595                pool_metadata_error(crate::error::PoolMetadataFailure::ReadUnavailable, "pool_read", Some(Arc::clone(source)));
5596            context.operation = operation.to_owned();
5597            Some(Error::other(context))
5598        }
5599        _ => None,
5600    })
5601}
5602
5603fn select_pool_meta_replicas_for_read_probe<R>(
5604    write_state: &PoolMetaWriteState,
5605    replicas: Vec<R>,
5606    operation: &str,
5607) -> Result<PoolMetaSelection>
5608where
5609    R: Into<PoolMetaReplicaRead>,
5610{
5611    let replicas = replicas.into_iter().map(Into::into).collect::<Vec<_>>();
5612    if let Some(err) = pool_meta_replica_read_failure(&replicas, operation) {
5613        return Err(err);
5614    }
5615    let selection = select_pool_meta_replica_reads(replicas)?;
5616    write_state.validate_selection(&selection)?;
5617    selection.replica_state.ensure_write_safe(operation)?;
5618    if selection.absent && (write_state.expected_cluster_id.is_some() || write_state.identity_initialized.is_some()) {
5619        write_state
5620            .validate_missing_metadata_can_initialize()
5621            .map_err(MissingMetadataRejection::into_error)?;
5622    }
5623    Ok(selection)
5624}
5625
5626async fn load_pool_meta_replicas<S>(pools: Vec<Arc<S>>, no_lock: bool) -> Result<PoolMetaSelection>
5627where
5628    S: EcstoreObjectIO,
5629{
5630    select_pool_meta_replica_reads(read_pool_meta_replicas(pools, no_lock).await)
5631}
5632
5633async fn load_pool_meta_replicas_observing<S>(
5634    pools: Vec<Arc<S>>,
5635    no_lock: bool,
5636    write_state: &mut PoolMetaWriteState,
5637) -> Result<PoolMetaSelection>
5638where
5639    S: EcstoreObjectIO,
5640{
5641    let replicas = read_pool_meta_replicas(pools, no_lock).await;
5642    select_pool_meta_replicas_observing(write_state, replicas)
5643}
5644
5645async fn load_pool_meta_replicas_for_read_probe<S>(
5646    pools: Vec<Arc<S>>,
5647    no_lock: bool,
5648    write_state: &PoolMetaWriteState,
5649    operation: &str,
5650) -> Result<PoolMetaSelection>
5651where
5652    S: EcstoreObjectIO,
5653{
5654    let replicas = read_pool_meta_replicas(pools, no_lock).await;
5655    select_pool_meta_replicas_for_read_probe(write_state, replicas, operation).map_err(|err| {
5656        if err.pool_metadata_failure().is_some() {
5657            err
5658        } else {
5659            Error::other(pool_metadata_error(
5660                crate::error::PoolMetadataFailure::RecoveryRequired,
5661                "read_probe",
5662                Some(Arc::new(err)),
5663            ))
5664        }
5665    })
5666}
5667
5668#[derive(Debug, Clone, Serialize, Deserialize)]
5669#[serde(deny_unknown_fields)]
5670struct PersistedPoolMetaV3 {
5671    version: u16,
5672    cluster_id: String,
5673    epoch: u64,
5674    generation: u64,
5675    transaction_id: String,
5676    committed: bool,
5677    pools: Vec<PersistedPoolStatus>,
5678    previous: Option<PersistedPoolMetaV3Previous>,
5679}
5680
5681#[derive(Debug, Clone, Serialize, Deserialize)]
5682#[serde(deny_unknown_fields)]
5683struct PersistedPoolMetaV3Previous {
5684    version: u16,
5685    cluster_id: Option<String>,
5686    epoch: u64,
5687    generation: u64,
5688    transaction_id: Option<String>,
5689    pools: Vec<PersistedPoolStatus>,
5690}
5691
5692#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
5693#[serde(deny_unknown_fields)]
5694struct PersistedPoolMetaIdentity {
5695    version: u16,
5696    cluster_id: uuid::Uuid,
5697    epoch: u64,
5698    initialized: bool,
5699    fresh_bootstrap_nonce: Option<uuid::Uuid>,
5700}
5701
5702#[derive(Debug)]
5703enum PoolMetaIdentityReplica {
5704    Missing,
5705    Valid(PersistedPoolMetaIdentity),
5706    Corrupt(String),
5707    Incompatible(String),
5708    Unreadable(Arc<Error>),
5709}
5710
5711#[derive(Debug)]
5712struct PoolMetaIdentityRead {
5713    replica: PoolMetaIdentityReplica,
5714    cas: PoolMetaCasToken,
5715}
5716
5717#[derive(Debug)]
5718struct PoolMetaIdentitySelection {
5719    identity: Option<PersistedPoolMetaIdentity>,
5720    needs_repair: bool,
5721    repair_write_safe: bool,
5722    cas_tokens: Vec<PoolMetaCasToken>,
5723    /// Per pool: whether the replica decoded as a valid identity.
5724    valid_replicas: Vec<bool>,
5725}
5726
5727fn encode_pool_meta_identity(identity: PersistedPoolMetaIdentity) -> Result<Vec<u8>> {
5728    let mut data = Vec::new();
5729    data.write_u16::<LittleEndian>(POOL_META_IDENTITY_FORMAT)?;
5730    data.write_u16::<LittleEndian>(POOL_META_IDENTITY_VERSION)?;
5731    identity.serialize(&mut Serializer::new(&mut data))?;
5732    Ok(data)
5733}
5734
5735fn decode_pool_meta_identity(data: &[u8]) -> PoolMetaIdentityReplica {
5736    if data.len() <= 4 {
5737        return PoolMetaIdentityReplica::Corrupt("identity payload is empty or truncated".to_string());
5738    }
5739    let format = LittleEndian::read_u16(&data[0..2]);
5740    let version = LittleEndian::read_u16(&data[2..4]);
5741    if format != POOL_META_IDENTITY_FORMAT || version != POOL_META_IDENTITY_VERSION {
5742        return PoolMetaIdentityReplica::Incompatible(format!("unsupported identity format {format} version {version}"));
5743    }
5744    let identity = match rmp_serde::from_slice::<PersistedPoolMetaIdentity>(&data[4..]) {
5745        Ok(identity) => identity,
5746        Err(err) => return PoolMetaIdentityReplica::Corrupt(format!("identity payload is not decodable: {err}")),
5747    };
5748    let invalid_bootstrap_nonce = identity
5749        .fresh_bootstrap_nonce
5750        .is_some_and(|nonce| nonce.is_nil() || nonce == uuid::Uuid::max());
5751    if identity.version != POOL_META_IDENTITY_VERSION
5752        || identity.cluster_id.is_nil()
5753        || identity.cluster_id == uuid::Uuid::max()
5754        || identity.epoch == 0
5755        || invalid_bootstrap_nonce
5756        || identity.initialized == identity.fresh_bootstrap_nonce.is_some()
5757    {
5758        return PoolMetaIdentityReplica::Corrupt(
5759            "identity payload contains an invalid version, UUID, epoch, or fresh-bootstrap proof".to_string(),
5760        );
5761    }
5762    PoolMetaIdentityReplica::Valid(identity)
5763}
5764
5765#[cfg(test)]
5766pub(crate) fn pool_meta_identity_initialized_for_test(data: &[u8]) -> Result<bool> {
5767    match decode_pool_meta_identity(data) {
5768        PoolMetaIdentityReplica::Valid(identity) => Ok(identity.initialized),
5769        _ => Err(Error::other("test pool metadata identity is not valid")),
5770    }
5771}
5772
5773#[cfg(test)]
5774pub(crate) fn pending_pool_meta_identity_for_test(cluster_id: uuid::Uuid, epoch: u64, nonce: uuid::Uuid) -> Result<Vec<u8>> {
5775    encode_pool_meta_identity(PersistedPoolMetaIdentity {
5776        version: POOL_META_IDENTITY_VERSION,
5777        cluster_id,
5778        epoch,
5779        initialized: false,
5780        fresh_bootstrap_nonce: Some(nonce),
5781    })
5782}
5783
5784#[cfg(test)]
5785pub(crate) fn initialized_pool_meta_identity_for_test(cluster_id: uuid::Uuid, epoch: u64) -> Result<Vec<u8>> {
5786    encode_pool_meta_identity(PersistedPoolMetaIdentity {
5787        version: POOL_META_IDENTITY_VERSION,
5788        cluster_id,
5789        epoch,
5790        initialized: true,
5791        fresh_bootstrap_nonce: None,
5792    })
5793}
5794
5795async fn read_pool_meta_identity_replica<S>(pool: Arc<S>) -> PoolMetaIdentityRead
5796where
5797    S: EcstoreObjectIO,
5798{
5799    match read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_IDENTITY_NAME).await {
5800        Ok((data, object_info)) => PoolMetaIdentityRead {
5801            replica: decode_pool_meta_identity(&data),
5802            cas: object_info
5803                .etag
5804                .filter(|etag| !etag.trim().is_empty())
5805                .map(PoolMetaCasToken::Existing)
5806                .unwrap_or(PoolMetaCasToken::Unsafe),
5807        },
5808        Err(Error::ConfigNotFound) => PoolMetaIdentityRead {
5809            replica: PoolMetaIdentityReplica::Missing,
5810            cas: PoolMetaCasToken::Missing,
5811        },
5812        Err(err) => PoolMetaIdentityRead {
5813            replica: PoolMetaIdentityReplica::Unreadable(Arc::new(err)),
5814            cas: PoolMetaCasToken::Unsafe,
5815        },
5816    }
5817}
5818
5819fn select_pool_meta_identity(
5820    reads: Vec<PoolMetaIdentityRead>,
5821    expected_cluster_id: uuid::Uuid,
5822) -> Result<PoolMetaIdentitySelection> {
5823    let cas_tokens = reads.iter().map(|read| read.cas.clone()).collect();
5824    let valid_replicas = reads
5825        .iter()
5826        .map(|read| matches!(read.replica, PoolMetaIdentityReplica::Valid(_)))
5827        .collect();
5828    let mut selected: Option<PersistedPoolMetaIdentity> = None;
5829    let mut needs_repair = false;
5830    let mut repair_write_safe = true;
5831    let mut unusable = Vec::new();
5832    for (idx, read) in reads.into_iter().enumerate() {
5833        match read.replica {
5834            PoolMetaIdentityReplica::Missing => needs_repair = true,
5835            PoolMetaIdentityReplica::Corrupt(reason) => {
5836                needs_repair = true;
5837                unusable.push(format!("pool {idx} identity is corrupt: {reason}"));
5838            }
5839            PoolMetaIdentityReplica::Unreadable(reason) => {
5840                needs_repair = true;
5841                repair_write_safe = false;
5842                unusable.push(format!("pool {idx} identity is unreadable: {reason}"));
5843            }
5844            PoolMetaIdentityReplica::Incompatible(reason) => {
5845                return Err(Error::other(format!("pool metadata incompatible: pool {idx} identity uses {reason}")));
5846            }
5847            PoolMetaIdentityReplica::Valid(identity) => {
5848                if identity.cluster_id != expected_cluster_id {
5849                    return Err(Error::other(format!(
5850                        "pool metadata incompatible: pool {idx} identity {} does not match deployment {expected_cluster_id}",
5851                        identity.cluster_id
5852                    )));
5853                }
5854                if let Some(current) = selected {
5855                    if current.cluster_id != identity.cluster_id || current.epoch != identity.epoch {
5856                        return Err(Error::other(
5857                            "pool metadata recovery required: identity replicas disagree on cluster identity or epoch",
5858                        ));
5859                    }
5860                    if current.initialized != identity.initialized {
5861                        needs_repair = true;
5862                        selected = Some(if current.initialized { current } else { identity });
5863                    } else if !current.initialized && current.fresh_bootstrap_nonce != identity.fresh_bootstrap_nonce {
5864                        return Err(Error::other(
5865                            "pool metadata recovery required: pending identity replicas disagree on fresh-bootstrap proof",
5866                        ));
5867                    }
5868                } else {
5869                    selected = Some(identity);
5870                }
5871            }
5872        }
5873    }
5874    if selected.is_none() && !unusable.is_empty() {
5875        return Err(Error::other(format!(
5876            "pool metadata recovery required: no valid cluster identity replica is available ({})",
5877            unusable.join("; ")
5878        )));
5879    }
5880    Ok(PoolMetaIdentitySelection {
5881        identity: selected,
5882        needs_repair,
5883        repair_write_safe,
5884        cas_tokens,
5885        valid_replicas,
5886    })
5887}
5888
5889async fn load_pool_meta_identity<S>(pools: Vec<Arc<S>>, expected_cluster_id: uuid::Uuid) -> Result<PoolMetaIdentitySelection>
5890where
5891    S: EcstoreObjectIO,
5892{
5893    let reads = join_all(pools.into_iter().map(read_pool_meta_identity_replica)).await;
5894    if let Some(source) = reads.iter().find_map(|read| match &read.replica {
5895        PoolMetaIdentityReplica::Unreadable(source) => Some(Arc::clone(source)),
5896        _ => None,
5897    }) {
5898        return Err(Error::other(pool_metadata_error(
5899            crate::error::PoolMetadataFailure::ReadUnavailable,
5900            "identity_read",
5901            Some(source),
5902        )));
5903    }
5904    select_pool_meta_identity(reads, expected_cluster_id)
5905}
5906
5907async fn load_pool_meta_identity_selection_observing<S>(
5908    pools: Vec<Arc<S>>,
5909    write_state: &mut PoolMetaWriteState,
5910    expected_cluster_id: uuid::Uuid,
5911) -> Result<PoolMetaIdentitySelection>
5912where
5913    S: EcstoreObjectIO,
5914{
5915    let selection = match load_pool_meta_identity(pools, expected_cluster_id).await {
5916        Ok(selection) => selection,
5917        Err(err) => {
5918            if err
5919                .pool_metadata_failure()
5920                .is_some_and(|context| context.kind == crate::error::PoolMetadataFailure::ReadUnavailable)
5921            {
5922                return Err(err);
5923            }
5924            return Err(block_pool_meta_validation(write_state, err, "identity_selection"));
5925        }
5926    };
5927    if let Err(err) = write_state.observe_identity(&selection) {
5928        return Err(block_pool_meta_validation(write_state, err, "identity_selection"));
5929    }
5930    Ok(selection)
5931}
5932
5933fn persisted_pool_meta_v3_previous(candidate: &PoolMetaCommittedCandidate) -> PersistedPoolMetaV3Previous {
5934    PersistedPoolMetaV3Previous {
5935        version: candidate.revision.version,
5936        cluster_id: candidate.revision.cluster_id.map(|id| id.to_string()),
5937        epoch: candidate.revision.epoch,
5938        generation: candidate.revision.generation,
5939        transaction_id: candidate.revision.transaction_id.map(|id| id.to_string()),
5940        pools: candidate.meta.pools.iter().map(Into::into).collect(),
5941    }
5942}
5943
5944fn encode_pool_meta_v3_envelope(
5945    meta: &PoolMeta,
5946    revision: PoolMetaRevision,
5947    committed: bool,
5948    previous: Option<&PoolMetaCommittedCandidate>,
5949) -> Result<Vec<u8>> {
5950    if meta.dont_save {
5951        return Ok(Vec::new());
5952    }
5953    let cluster_id = revision
5954        .cluster_id
5955        .ok_or_else(|| Error::other("pool metadata V3 save failed: cluster identity is not initialized"))?;
5956    let transaction_id = revision
5957        .transaction_id
5958        .ok_or_else(|| Error::other("pool metadata V3 save failed: transaction id is not initialized"))?;
5959    if revision.version != POOL_META_GENERATION_VERSION || revision.epoch == 0 || revision.generation == 0 {
5960        return Err(Error::other("pool metadata V3 save failed: invalid durable revision"));
5961    }
5962    let persisted = PersistedPoolMetaV3 {
5963        version: POOL_META_GENERATION_VERSION,
5964        cluster_id: cluster_id.to_string(),
5965        epoch: revision.epoch,
5966        generation: revision.generation,
5967        transaction_id: transaction_id.to_string(),
5968        committed,
5969        pools: meta.pools.iter().map(Into::into).collect(),
5970        previous: previous.map(persisted_pool_meta_v3_previous),
5971    };
5972    let mut data = Vec::new();
5973    data.write_u16::<LittleEndian>(POOL_META_FORMAT)?;
5974    data.write_u16::<LittleEndian>(POOL_META_GENERATION_VERSION)?;
5975    persisted.serialize(&mut Serializer::new(&mut data))?;
5976    Ok(data)
5977}
5978
5979enum PoolMetaPersistenceFence<'a> {
5980    Distributed(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>),
5981    Activation(&'a PoolRebalanceActivationFence),
5982}
5983
5984impl PoolMetaPersistenceFence<'_> {
5985    fn ensure_held(&self) -> Result<()> {
5986        match self {
5987            Self::Distributed(Some(signal)) if signal.is_lost() => {
5988                Err(Error::other("pool metadata distributed fence was lost before a replica write"))
5989            }
5990            Self::Activation(fence) => fence.ensure_held(),
5991            _ => Ok(()),
5992        }
5993    }
5994
5995    fn add_to_options(&self, opts: &mut ObjectOptions) {
5996        match self {
5997            Self::Distributed(Some(signal)) => opts.add_namespace_lock_lost_signal(Arc::clone(signal)),
5998            Self::Activation(fence) => fence.add_namespace_lock_fence(opts),
5999            Self::Distributed(None) => {}
6000        }
6001    }
6002
6003    fn is_activation(&self) -> bool {
6004        matches!(self, Self::Activation(_))
6005    }
6006}
6007
6008fn pool_meta_cas_preconditions(token: &PoolMetaCasToken, object: &str) -> Result<HTTPPreconditions> {
6009    match token {
6010        PoolMetaCasToken::Missing => Ok(HTTPPreconditions {
6011            if_none_match: Some("*".to_string()),
6012            ..Default::default()
6013        }),
6014        PoolMetaCasToken::Existing(etag) => Ok(HTTPPreconditions {
6015            if_match: Some(etag.clone()),
6016            ..Default::default()
6017        }),
6018        PoolMetaCasToken::Unsafe => Err(Error::other(format!(
6019            "pool metadata recovery required: {object} replica has no safe conditional-write revision"
6020        ))),
6021    }
6022}
6023
6024#[cfg(feature = "e2e-test-hooks")]
6025struct StartupCasObservation {
6026    attempt: uuid::Uuid,
6027    phase: &'static str,
6028    pools: Vec<usize>,
6029}
6030
6031#[cfg(feature = "e2e-test-hooks")]
6032tokio::task_local! {
6033    static STARTUP_CAS_OBSERVATION: StartupCasObservation;
6034}
6035
6036// This scope follows only the directly polled startup future. Spawned work
6037// does not inherit it; receiver evidence retains its existing RPC tuple.
6038#[cfg(feature = "e2e-test-hooks")]
6039pub(crate) async fn startup_cas_test_scope<S, F: std::future::Future>(
6040    attempt: uuid::Uuid,
6041    phase: &'static str,
6042    pools: &[Arc<S>],
6043    future: F,
6044) -> F::Output {
6045    STARTUP_CAS_OBSERVATION
6046        .scope(
6047            StartupCasObservation {
6048                attempt,
6049                phase,
6050                // These identities are never dereferenced or logged. The
6051                // caller and operation keep the same pool Arcs alive.
6052                pools: pools.iter().map(|pool| Arc::as_ptr(pool) as usize).collect(),
6053            },
6054            future,
6055        )
6056        .await
6057}
6058
6059// Direct JSON diagnostics are independent of the startup tracing subscriber.
6060#[cfg(feature = "e2e-test-hooks")]
6061pub(crate) fn startup_cas_test_observe(mut observation: serde_json::Value) {
6062    let Some(nonce) = std::env::var("RUSTFS_E2E_STARTUP_CAS_NONCE")
6063        .ok()
6064        .and_then(|value| uuid::Uuid::parse_str(&value).ok())
6065    else {
6066        return;
6067    };
6068    observation["nonce"] = serde_json::json!(nonce);
6069    observation["pid"] = serde_json::json!(std::process::id());
6070    let _ = STARTUP_CAS_OBSERVATION.try_with(|scope| {
6071        observation["attempt"] = serde_json::json!(scope.attempt);
6072        observation["startup_phase"] = serde_json::json!(scope.phase);
6073    });
6074    let line = format!("RUSTFS_E2E_STARTUP_CAS {observation}\n");
6075    let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes());
6076}
6077
6078#[cfg(feature = "e2e-test-hooks")]
6079fn pool_meta_test_pools_sha256(meta: &PoolMeta) -> Result<String> {
6080    let pools = meta.pools.iter().map(PersistedPoolStatus::from).collect::<Vec<_>>();
6081    Ok(rustfs_utils::crypto::hex(Sha256::digest(serde_json::to_vec(&pools)?)))
6082}
6083
6084#[cfg(feature = "e2e-test-hooks")]
6085fn pool_meta_test_snapshot(meta: &PoolMeta, revision: PoolMetaRevision, canonical: &[u8]) -> Result<serde_json::Value> {
6086    Ok(serde_json::json!({
6087        "version": revision.version, "cluster_id": revision.cluster_id, "epoch": revision.epoch,
6088        "generation": revision.generation, "transaction_id": revision.transaction_id,
6089        "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(canonical)),
6090        "persisted_pools_sha256": pool_meta_test_pools_sha256(meta)?,
6091    }))
6092}
6093
6094#[cfg(feature = "e2e-test-hooks")]
6095#[derive(Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
6096#[serde(rename_all = "snake_case")]
6097enum PoolMetaPhaseBarrierCase {
6098    PrepareSubset,
6099    PreparedAll,
6100    CommitOne,
6101    BeforePublish,
6102}
6103
6104#[cfg(feature = "e2e-test-hooks")]
6105#[derive(Deserialize)]
6106#[serde(deny_unknown_fields)]
6107struct PoolMetaPhaseBarrierArm {
6108    nonce: uuid::Uuid,
6109    case: PoolMetaPhaseBarrierCase,
6110}
6111
6112#[cfg(feature = "e2e-test-hooks")]
6113struct PoolMetaPhaseBarrier {
6114    directory: std::path::PathBuf,
6115    arm: PoolMetaPhaseBarrierArm,
6116    transaction_id: uuid::Uuid,
6117    event_gate: tokio::sync::Mutex<()>,
6118    blocked_pools: AtomicUsize,
6119    completed_writes: AtomicUsize,
6120    ready_emitted: AtomicBool,
6121}
6122
6123#[cfg(feature = "e2e-test-hooks")]
6124impl PoolMetaPhaseBarrier {
6125    async fn bind(
6126        previous: &PoolMetaCommittedCandidate,
6127        candidate: &PoolMeta,
6128        revision: PoolMetaRevision,
6129        durable: &[u8],
6130    ) -> Result<Option<Self>> {
6131        let Some(directory) = std::env::var_os("RUSTFS_E2E_POOL_META_BARRIER_DIR").map(std::path::PathBuf::from) else {
6132            return Ok(None);
6133        };
6134        Self::bind_in_directory(directory, previous, candidate, revision, durable).await
6135    }
6136
6137    async fn bind_in_directory(
6138        directory: std::path::PathBuf,
6139        previous: &PoolMetaCommittedCandidate,
6140        candidate: &PoolMeta,
6141        revision: PoolMetaRevision,
6142        durable: &[u8],
6143    ) -> Result<Option<Self>> {
6144        let data = match tokio::fs::read(directory.join("arm.json")).await {
6145            Ok(data) => data,
6146            Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
6147            Err(err) => return Err(err.into()),
6148        };
6149        let arm: PoolMetaPhaseBarrierArm = serde_json::from_slice(&data)?;
6150        if arm.nonce.is_nil() || !directory.is_absolute() {
6151            return Err(Error::other(
6152                "pool metadata test barrier requires an absolute directory and non-nil nonce",
6153            ));
6154        }
6155        let transaction_id = revision
6156            .transaction_id
6157            .ok_or_else(|| Error::other("pool metadata test barrier requires a V3 transaction"))?;
6158        // A new revision can still carry the same persisted pool state. Leave
6159        // the external arm available until recovery can distinguish P from G.
6160        if pool_meta_test_pools_sha256(&previous.meta)? == pool_meta_test_pools_sha256(candidate)? {
6161            return Ok(None);
6162        }
6163        // One external arm binds one attempt, including if its CAS subsequently retries.
6164        let claim = tokio::fs::OpenOptions::new()
6165            .write(true)
6166            .create_new(true)
6167            .open(directory.join("claimed"))
6168            .await;
6169        match claim {
6170            Ok(_) => {}
6171            Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => return Ok(None),
6172            Err(err) => return Err(err.into()),
6173        }
6174        let barrier = Self {
6175            directory,
6176            arm,
6177            transaction_id,
6178            event_gate: tokio::sync::Mutex::new(()),
6179            blocked_pools: AtomicUsize::new(0),
6180            completed_writes: AtomicUsize::new(0),
6181            ready_emitted: AtomicBool::new(false),
6182        };
6183        barrier
6184            .observe(serde_json::json!({
6185                "kind": "armed",
6186                "previous": pool_meta_test_snapshot(&previous.meta, previous.revision, &previous.canonical)?,
6187                "candidate": pool_meta_test_snapshot(candidate, revision, durable)?,
6188            }))
6189            .await?;
6190        Ok(Some(barrier))
6191    }
6192
6193    async fn observe(&self, mut event: serde_json::Value) -> Result<()> {
6194        use tokio::io::AsyncWriteExt as _;
6195        event["nonce"] = serde_json::json!(self.arm.nonce);
6196        event["pid"] = serde_json::json!(std::process::id());
6197        event["transaction_id"] = serde_json::json!(self.transaction_id);
6198        event["case"] = serde_json::json!(self.arm.case);
6199        let mut line = serde_json::to_vec(&event)?;
6200        line.push(b'\n');
6201        let _event_guard = self.event_gate.lock().await;
6202        let mut file = tokio::fs::OpenOptions::new()
6203            .create(true)
6204            .append(true)
6205            .open(self.directory.join("events.jsonl"))
6206            .await?;
6207        file.write_all(&line).await?;
6208        file.flush().await?;
6209        Ok(())
6210    }
6211
6212    async fn wait_for_release(&self) -> Result<()> {
6213        tokio::time::timeout(std::time::Duration::from_secs(120), async {
6214            loop {
6215                match tokio::fs::metadata(self.directory.join("release")).await {
6216                    Ok(_) => return Ok(()),
6217                    Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
6218                    Err(err) => return Err(Error::from(err)),
6219                }
6220                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
6221            }
6222        })
6223        .await
6224        .map_err(|_| Error::other("pool metadata test barrier release timed out"))?
6225    }
6226
6227    async fn ready(&self) -> Result<()> {
6228        self.observe(serde_json::json!({"kind": "ready", "cutpoint": self.arm.case}))
6229            .await?;
6230        self.wait_for_release().await
6231    }
6232
6233    async fn ready_after_all_participants(&self) -> Result<()> {
6234        let (blocked, completed) = match self.arm.case {
6235            PoolMetaPhaseBarrierCase::PrepareSubset => (0b0101, 0b1010),
6236            PoolMetaPhaseBarrierCase::CommitOne => (0b0111, 0b1000),
6237            _ => return Ok(()),
6238        };
6239        if self.blocked_pools.load(Ordering::SeqCst) == blocked
6240            && self.completed_writes.load(Ordering::SeqCst) == completed
6241            && !self.ready_emitted.swap(true, Ordering::SeqCst)
6242        {
6243            self.ready().await?;
6244        }
6245        Ok(())
6246    }
6247
6248    async fn before_write(&self, phase: &'static str, pool: usize, data: &[u8]) -> Result<()> {
6249        let blocked = match (self.arm.case, phase) {
6250            (PoolMetaPhaseBarrierCase::PrepareSubset, "prepare_cas") => ![1, 3].contains(&pool),
6251            (PoolMetaPhaseBarrierCase::CommitOne, "commit_cas") => pool != 3,
6252            _ => false,
6253        };
6254        if blocked {
6255            self.observe(serde_json::json!({
6256                "kind": "blocked", "phase": phase, "pool": pool,
6257                "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(data)),
6258            }))
6259            .await?;
6260            self.blocked_pools.fetch_or(1 << pool, Ordering::SeqCst);
6261            self.ready_after_all_participants().await?;
6262            self.wait_for_release().await?;
6263        }
6264        self.observe(serde_json::json!({
6265            "kind": "before-dispatch", "phase": phase, "pool": pool,
6266            "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(data)),
6267        }))
6268        .await
6269    }
6270
6271    async fn after_write(&self, phase: &'static str, pool: usize, data: &[u8], outcome: &PoolMetaCasWriteOutcome) -> Result<()> {
6272        self.observe(serde_json::json!({
6273            "kind": "after-cas", "phase": phase, "pool": pool,
6274            "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(data)),
6275            "ok": outcome.result.is_ok(), "tail_drained": outcome.result.is_ok(),
6276            "etag": outcome.result.as_ref().ok().and_then(|info| info.etag.as_deref()),
6277            "may_have_mutated": outcome.may_have_mutated,
6278        }))
6279        .await?;
6280        if outcome.result.is_ok() {
6281            match (self.arm.case, phase, pool) {
6282                (PoolMetaPhaseBarrierCase::PrepareSubset, "prepare_cas", 1 | 3)
6283                | (PoolMetaPhaseBarrierCase::CommitOne, "commit_cas", 3) => {
6284                    self.completed_writes.fetch_or(1 << pool, Ordering::SeqCst);
6285                    self.ready_after_all_participants().await?;
6286                }
6287                _ => {}
6288            }
6289        }
6290        Ok(())
6291    }
6292}
6293
6294fn pool_meta_cas_options(object: &str, token: &PoolMetaCasToken, fence: &PoolMetaPersistenceFence<'_>) -> Result<ObjectOptions> {
6295    fence.ensure_held()?;
6296    let mut opts = ObjectOptions {
6297        max_parity: true,
6298        write_completion: crate::object_api::WriteCompletion::TailDrained,
6299        no_lock: true,
6300        http_preconditions: Some(pool_meta_cas_preconditions(token, object)?),
6301        ..Default::default()
6302    };
6303    fence.add_to_options(&mut opts);
6304    Ok(opts)
6305}
6306
6307struct PoolMetaCasWriteOutcome {
6308    result: Result<crate::object_api::ObjectInfo>,
6309    source: Option<Arc<Error>>,
6310    may_have_mutated: bool,
6311}
6312
6313impl PoolMetaCasWriteOutcome {
6314    fn not_dispatched(err: Error) -> Self {
6315        Self {
6316            result: Err(err),
6317            source: None,
6318            may_have_mutated: false,
6319        }
6320    }
6321}
6322
6323async fn execute_pool_meta_object_cas<S>(
6324    pool: Arc<S>,
6325    object: &str,
6326    data: Vec<u8>,
6327    opts: ObjectOptions,
6328    fence: &PoolMetaPersistenceFence<'_>,
6329    phase: &'static str,
6330) -> PoolMetaCasWriteOutcome
6331where
6332    S: EcstoreObjectIO,
6333{
6334    // A bounded phase can wait for another replica before this write is polled.
6335    if let Err(err) = fence.ensure_held() {
6336        return PoolMetaCasWriteOutcome::not_dispatched(err);
6337    }
6338    #[cfg(feature = "e2e-test-hooks")]
6339    let observation = std::env::var_os("RUSTFS_E2E_STARTUP_CAS_NONCE").map(|_| {
6340        serde_json::json!({
6341            "kind": "cas", "object": object, "phase": phase,
6342            "pool": STARTUP_CAS_OBSERVATION.try_with(|scope| {
6343                scope.pools.iter().position(|identity| *identity == Arc::as_ptr(&pool) as usize)
6344            }).ok().flatten(),
6345            "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&data)),
6346            "if_match": opts.http_preconditions.as_ref().and_then(|p| p.if_match.as_deref()),
6347            "if_none_match": opts.http_preconditions.as_ref().and_then(|p| p.if_none_match.as_deref()),
6348            "tail_drained": opts.write_completion == crate::object_api::WriteCompletion::TailDrained,
6349            "no_lock": opts.no_lock,
6350        })
6351    });
6352    let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await;
6353    let may_have_mutated = !matches!(&result, Err(Error::PreconditionFailed));
6354    if !may_have_mutated {
6355        record_pool_meta_stale_write_rejection(phase);
6356    }
6357    let mut source = None;
6358    let result = match result {
6359        Ok(object_info) => fence.ensure_held().map(|()| object_info),
6360        Err(err) => {
6361            let original = Arc::new(err);
6362            source = Some(Arc::clone(&original));
6363            if matches!(original.as_ref(), Error::PreconditionFailed) {
6364                Err(Error::PreconditionFailed)
6365            } else {
6366                Err(Error::other(pool_metadata_error(
6367                    crate::error::PoolMetadataFailure::TransactionUnknown,
6368                    phase,
6369                    Some(original),
6370                )))
6371            }
6372        }
6373    };
6374    #[cfg(feature = "e2e-test-hooks")]
6375    if let Some(mut observation) = observation {
6376        observation["ok"] = serde_json::json!(result.is_ok());
6377        observation["etag"] = serde_json::json!(result.as_ref().ok().and_then(|info| info.etag.as_deref()));
6378        observation["mod_time"] = serde_json::json!(
6379            result
6380                .as_ref()
6381                .ok()
6382                .and_then(|info| info.mod_time)
6383                .map(|time| time.unix_timestamp_nanos().to_string())
6384        );
6385        observation["error"] = serde_json::json!(result.as_ref().err().map(ToString::to_string));
6386        startup_cas_test_observe(observation);
6387    }
6388    PoolMetaCasWriteOutcome {
6389        result,
6390        source,
6391        may_have_mutated,
6392    }
6393}
6394
6395async fn save_pool_meta_object_cas<S>(
6396    pool: Arc<S>,
6397    object: &str,
6398    data: Vec<u8>,
6399    token: &PoolMetaCasToken,
6400    fence: &PoolMetaPersistenceFence<'_>,
6401    phase: &'static str,
6402    transaction_arm: &mut PoolMetaTransactionArm,
6403) -> Result<crate::object_api::ObjectInfo>
6404where
6405    S: EcstoreObjectIO,
6406{
6407    let opts = pool_meta_cas_options(object, token, fence)?;
6408    // Cancellation can happen at the very first poll of the storage future.
6409    // Arm before dispatch, but not during read/encode/fence preflight.
6410    let previous_phase = transaction_arm.phase;
6411    transaction_arm.phase = Some(phase);
6412    let outcome = execute_pool_meta_object_cas(pool, object, data, opts, fence, phase).await;
6413    if !outcome.may_have_mutated {
6414        transaction_arm.phase = previous_phase;
6415    }
6416    if let Some(source) = outcome.source
6417        && (!matches!(source.as_ref(), Error::PreconditionFailed)
6418            || transaction_arm
6419                .source
6420                .as_ref()
6421                .is_none_or(|previous| matches!(previous.as_ref(), Error::PreconditionFailed)))
6422    {
6423        transaction_arm.source = Some(source);
6424    }
6425    outcome.result
6426}
6427
6428async fn save_pool_meta_phase<S>(
6429    pools: &[Arc<S>],
6430    data: &[u8],
6431    tokens: &[PoolMetaCasToken],
6432    fence: &PoolMetaPersistenceFence<'_>,
6433    phase: &'static str,
6434    transaction_arm: &mut PoolMetaTransactionArm,
6435    #[cfg(feature = "e2e-test-hooks")] barrier: Option<&PoolMetaPhaseBarrier>,
6436) -> Vec<Result<crate::object_api::ObjectInfo>>
6437where
6438    S: EcstoreObjectIO,
6439{
6440    if pools.len() != tokens.len() {
6441        return vec![Err(Error::other("pool metadata phase has inconsistent replica revisions"))];
6442    }
6443    let options = tokens
6444        .iter()
6445        .map(|token| pool_meta_cas_options(POOL_META_NAME, token, fence))
6446        .collect::<Vec<_>>();
6447    let previous_phase = transaction_arm.phase;
6448    if options.iter().any(Result::is_ok) {
6449        transaction_arm.phase = Some(phase);
6450    }
6451    // The caller owns one arm and the save/namespace guards for the whole phase.
6452    // No replica may restore that arm while another write is still in flight.
6453    let writes = futures::stream::iter(pools.iter().cloned().zip(options).enumerate().map(
6454        |(pool_index, (pool, opts))| async move {
6455            #[cfg(feature = "e2e-test-hooks")]
6456            if opts.is_ok()
6457                && let Some(barrier) = barrier
6458                && let Err(err) = barrier.before_write(phase, pool_index, data).await
6459            {
6460                return (pool_index, PoolMetaCasWriteOutcome::not_dispatched(err));
6461            }
6462            let outcome = match opts {
6463                Ok(opts) => execute_pool_meta_object_cas(pool, POOL_META_NAME, data.to_vec(), opts, fence, phase).await,
6464                Err(err) => PoolMetaCasWriteOutcome::not_dispatched(err),
6465            };
6466            #[cfg(feature = "e2e-test-hooks")]
6467            let outcome = {
6468                let mut outcome = outcome;
6469                if let Some(barrier) = barrier
6470                    && let Err(err) = barrier.after_write(phase, pool_index, data, &outcome).await
6471                    && outcome.result.is_ok()
6472                {
6473                    outcome.result = Err(err);
6474                }
6475                outcome
6476            };
6477            (pool_index, outcome)
6478        },
6479    ))
6480    .buffer_unordered(4);
6481    futures::pin_mut!(writes);
6482    let mut results = Vec::with_capacity(pools.len());
6483    let mut may_have_mutated = false;
6484    let mut source_pool_index: Option<(bool, usize)> = None;
6485    while let Some((pool_index, outcome)) = writes.next().await {
6486        may_have_mutated |= outcome.may_have_mutated;
6487        if let Some(source) = outcome.source
6488            && (outcome.may_have_mutated
6489                || transaction_arm
6490                    .source
6491                    .as_ref()
6492                    .is_none_or(|previous| matches!(previous.as_ref(), Error::PreconditionFailed)))
6493            && source_pool_index.is_none_or(|(previous_mutated, previous_index)| {
6494                (outcome.may_have_mutated && !previous_mutated)
6495                    || (outcome.may_have_mutated == previous_mutated && pool_index < previous_index)
6496            })
6497        {
6498            // A later CAS rejection must not erase an I/O failure if draining is cancelled.
6499            transaction_arm.source = Some(source);
6500            source_pool_index = Some((outcome.may_have_mutated, pool_index));
6501        }
6502        results.push((pool_index, outcome.result));
6503    }
6504    if !may_have_mutated {
6505        transaction_arm.phase = previous_phase;
6506    }
6507    results.sort_unstable_by_key(|(pool_index, _)| *pool_index);
6508    results.into_iter().map(|(_, result)| result).collect()
6509}
6510
6511/// Which pool replicas a cluster-identity write may touch.
6512#[derive(Debug, Clone, Copy)]
6513enum PoolMetaIdentityWriteScope<'a> {
6514    /// Every pool. Creating a pending identity here requires deployment-wide
6515    /// fresh-bootstrap or legacy-adoption proof.
6516    All,
6517    /// Only the listed pools, each of which this process formatted or adopted
6518    /// first-hand. Multi-pool bootstraps whose pools have distinct format
6519    /// creators use this scope: the first pool's creator mints the deployment
6520    /// nonce and every other creator copies it to its own pool, so the elected
6521    /// writer can verify a complete, agreeing pending set instead of trusting
6522    /// an in-process flag it cannot observe on another node.
6523    Pools(&'a [usize]),
6524}
6525
6526fn identity_write_satisfied(
6527    selection: &PoolMetaIdentitySelection,
6528    identity: PersistedPoolMetaIdentity,
6529    scope: PoolMetaIdentityWriteScope<'_>,
6530    targets: &[usize],
6531) -> bool {
6532    if selection.identity != Some(identity) {
6533        return false;
6534    }
6535    match scope {
6536        PoolMetaIdentityWriteScope::All => !selection.needs_repair,
6537        PoolMetaIdentityWriteScope::Pools(_) => targets
6538            .iter()
6539            .all(|pool_idx| selection.valid_replicas.get(*pool_idx).copied().unwrap_or(false)),
6540    }
6541}
6542
6543async fn persist_pool_meta_identity<S>(
6544    pools: Vec<Arc<S>>,
6545    write_state: &mut PoolMetaWriteState,
6546    initialized: bool,
6547    scope: PoolMetaIdentityWriteScope<'_>,
6548    fence: &PoolMetaPersistenceFence<'_>,
6549    transaction_arm: &mut PoolMetaTransactionArm,
6550) -> Result<()>
6551where
6552    S: EcstoreObjectIO,
6553{
6554    let Some(cluster_id) = write_state.expected_cluster_id else {
6555        return Ok(());
6556    };
6557    let targets: Vec<usize> = match scope {
6558        PoolMetaIdentityWriteScope::All => (0..pools.len()).collect(),
6559        PoolMetaIdentityWriteScope::Pools(indices) => {
6560            if initialized {
6561                return Err(Error::other("pool metadata identity commit must address every pool"));
6562            }
6563            if indices
6564                .iter()
6565                .any(|pool_idx| *pool_idx >= pools.len() || !write_state.pool_bootstrap_authority_proven(*pool_idx))
6566            {
6567                return Err(Error::other(
6568                    "pool metadata recovery required: a pending cluster identity can only be attested for pools this startup formatted or adopted first-hand",
6569                ));
6570            }
6571            indices.to_vec()
6572        }
6573    };
6574    for attempt in 0..POOL_META_CAS_MAX_ATTEMPTS {
6575        let selection = load_pool_meta_identity_selection_observing(pools.clone(), write_state, cluster_id).await?;
6576        if !selection.repair_write_safe {
6577            write_state.block_writes();
6578            return Err(Error::other(
6579                "pool metadata recovery required: cluster identity has an unreadable replica",
6580            ));
6581        }
6582        let identity = match (selection.identity, scope) {
6583            // An initialized deployment (for example a pool expansion) never
6584            // reopens bootstrap: first-hand proof for a new pool is not a
6585            // reason to publish a pending identity.
6586            (Some(identity), PoolMetaIdentityWriteScope::Pools(_)) if identity.initialized => return Ok(()),
6587            (Some(identity), _) if identity.initialized || initialized => PersistedPoolMetaIdentity {
6588                initialized: true,
6589                fresh_bootstrap_nonce: None,
6590                ..identity
6591            },
6592            (Some(identity), _) => identity,
6593            // Only the first pool's creator mints the deployment nonce; every
6594            // other creator waits until it is durable and copies it, so two
6595            // concurrent creators can never publish disagreeing replicas.
6596            (None, PoolMetaIdentityWriteScope::Pools(indices)) => {
6597                if !indices.contains(&0) {
6598                    return Ok(());
6599                }
6600                PersistedPoolMetaIdentity {
6601                    version: POOL_META_IDENTITY_VERSION,
6602                    cluster_id,
6603                    epoch: write_state.cluster_epoch.unwrap_or(POOL_META_INITIAL_EPOCH),
6604                    initialized: false,
6605                    fresh_bootstrap_nonce: Some(uuid::Uuid::new_v4()),
6606                }
6607            }
6608            (None, PoolMetaIdentityWriteScope::All) if !initialized && !write_state.bootstrap_identity_proven() => {
6609                write_state.block_writes();
6610                return Err(Error::other(
6611                    "pool metadata recovery required: cannot create a pending cluster identity without verified fresh-bootstrap proof or legacy-adoption proof",
6612                ));
6613            }
6614            (None, PoolMetaIdentityWriteScope::All) => PersistedPoolMetaIdentity {
6615                version: POOL_META_IDENTITY_VERSION,
6616                cluster_id,
6617                epoch: write_state.cluster_epoch.unwrap_or(POOL_META_INITIAL_EPOCH),
6618                initialized,
6619                fresh_bootstrap_nonce: (!initialized).then(uuid::Uuid::new_v4),
6620            },
6621        };
6622        if identity_write_satisfied(&selection, identity, scope, &targets) {
6623            return Ok(());
6624        }
6625        let data = encode_pool_meta_identity(identity)?;
6626        let mut conflict = false;
6627        for (pool_idx, (pool, token)) in pools.iter().cloned().zip(&selection.cas_tokens).enumerate() {
6628            if !targets.contains(&pool_idx) {
6629                continue;
6630            }
6631            match save_pool_meta_object_cas(
6632                pool,
6633                POOL_META_IDENTITY_NAME,
6634                data.clone(),
6635                token,
6636                fence,
6637                "identity_cas",
6638                transaction_arm,
6639            )
6640            .await
6641            {
6642                Ok(_) => {}
6643                Err(Error::PreconditionFailed) => {
6644                    conflict = true;
6645                    break;
6646                }
6647                Err(err) => return Err(err),
6648            }
6649        }
6650        if conflict {
6651            if attempt + 1 < POOL_META_CAS_MAX_ATTEMPTS {
6652                continue;
6653            }
6654            return Err(Error::PreconditionFailed);
6655        }
6656        let confirmed = load_pool_meta_identity_selection_observing(pools.clone(), write_state, cluster_id).await?;
6657        if identity_write_satisfied(&confirmed, identity, scope, &targets) {
6658            return Ok(());
6659        }
6660    }
6661    Err(Error::PreconditionFailed)
6662}
6663
6664pub(crate) async fn load_pool_meta_identity_observing<S>(pools: Vec<Arc<S>>, write_state: &mut PoolMetaWriteState) -> Result<()>
6665where
6666    S: EcstoreObjectIO,
6667{
6668    let Some(cluster_id) = write_state.expected_cluster_id else {
6669        return Ok(());
6670    };
6671    load_pool_meta_identity_selection_observing(pools, write_state, cluster_id)
6672        .await
6673        .map(|_| ())
6674}
6675
6676pub(crate) async fn persist_pool_meta_identity_for_startup<S>(
6677    pools: Vec<Arc<S>>,
6678    write_state: &mut PoolMetaWriteState,
6679    initialized: bool,
6680) -> Result<()>
6681where
6682    S: EcstoreObjectIO,
6683{
6684    let mut transaction_arm = write_state.arm_transaction();
6685    persist_pool_meta_identity(
6686        pools,
6687        write_state,
6688        initialized,
6689        PoolMetaIdentityWriteScope::All,
6690        &PoolMetaPersistenceFence::Distributed(None),
6691        &mut transaction_arm,
6692    )
6693    .await?;
6694    transaction_arm.disarm();
6695    Ok(())
6696}
6697
6698/// Attest, during startup, the pending cluster identity for the pools this
6699/// process formatted or adopted first-hand. The first pool's creator mints the
6700/// deployment nonce; every other creator copies it once it is durable. Nothing
6701/// is written while the deployment is already initialized or while the nonce
6702/// is not yet durable, so callers simply retry through the startup loop.
6703pub(crate) async fn persist_pool_meta_identity_for_attested_pools<S>(
6704    pools: Vec<Arc<S>>,
6705    write_state: &mut PoolMetaWriteState,
6706    pool_indices: &[usize],
6707) -> Result<()>
6708where
6709    S: EcstoreObjectIO,
6710{
6711    let mut transaction_arm = write_state.arm_transaction();
6712    persist_pool_meta_identity(
6713        pools,
6714        write_state,
6715        false,
6716        PoolMetaIdentityWriteScope::Pools(pool_indices),
6717        &PoolMetaPersistenceFence::Distributed(None),
6718        &mut transaction_arm,
6719    )
6720    .await?;
6721    transaction_arm.disarm();
6722    Ok(())
6723}
6724
6725#[derive(Debug, Clone, Serialize, Deserialize)]
6726#[serde(deny_unknown_fields)]
6727struct PersistedPoolMeta {
6728    pub version: u16,
6729    pub pools: Vec<PersistedPoolStatus>,
6730}
6731
6732#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
6733#[serde(deny_unknown_fields)]
6734struct PersistedPoolStatus {
6735    #[serde(rename = "id")]
6736    pub id: usize,
6737    #[serde(rename = "cmdline")]
6738    pub cmd_line: String,
6739    #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
6740    pub last_update: OffsetDateTime,
6741    #[serde(rename = "decommissionInfo")]
6742    pub decommission: Option<PersistedPoolDecommissionInfo>,
6743}
6744
6745#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
6746#[serde(deny_unknown_fields)]
6747struct PersistedPoolDecommissionInfo {
6748    #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
6749    pub start_time: Option<OffsetDateTime>,
6750    #[serde(rename = "startSize")]
6751    pub start_size: usize,
6752    #[serde(rename = "totalSize")]
6753    pub total_size: usize,
6754    #[serde(rename = "currentSize")]
6755    pub current_size: usize,
6756    #[serde(rename = "complete")]
6757    pub complete: bool,
6758    #[serde(rename = "failed")]
6759    pub failed: bool,
6760    #[serde(rename = "canceled")]
6761    pub canceled: bool,
6762    #[serde(rename = "queued", default)]
6763    pub queued: bool,
6764    #[serde(rename = "queuedBuckets", default)]
6765    pub queued_buckets: Vec<String>,
6766    #[serde(rename = "decommissionedBuckets", default)]
6767    pub decommissioned_buckets: Vec<String>,
6768    #[serde(rename = "bucket", default)]
6769    pub bucket: String,
6770    #[serde(rename = "prefix", default)]
6771    pub prefix: String,
6772    #[serde(rename = "object", default)]
6773    pub object: String,
6774    #[serde(rename = "objectsDecommissioned")]
6775    pub items_decommissioned: usize,
6776    #[serde(rename = "objectsDecommissionedFailed")]
6777    pub items_decommission_failed: usize,
6778    #[serde(rename = "bytesDecommissioned")]
6779    pub bytes_done: usize,
6780    #[serde(rename = "bytesDecommissionedFailed")]
6781    pub bytes_failed: usize,
6782    #[serde(rename = "terminalReloadAttemptAt", with = "time::serde::rfc3339::option", default)]
6783    pub terminal_reload_attempt_at: Option<OffsetDateTime>,
6784    #[serde(rename = "terminalReloadFailures", default)]
6785    pub terminal_reload_failures: Vec<String>,
6786    #[serde(rename = "unresolvedEntries", default)]
6787    pub unresolved_entries: Vec<DecommissionUnresolvedEntry>,
6788    #[serde(rename = "capacityReservation", default)]
6789    pub capacity_reservation: Option<DecommissionCapacityReservation>,
6790    #[serde(rename = "capacityBlockedReason", default)]
6791    pub capacity_blocked_reason: Option<String>,
6792}
6793
6794#[derive(Debug, Clone, Serialize, Deserialize)]
6795#[serde(deny_unknown_fields)]
6796struct PersistedPoolMetaV1 {
6797    pub version: u16,
6798    pub pools: Vec<PersistedPoolStatusV1>,
6799}
6800
6801#[derive(Debug, Clone, Serialize, Deserialize)]
6802#[serde(deny_unknown_fields)]
6803struct PersistedPoolStatusV1 {
6804    #[serde(rename = "id")]
6805    pub id: usize,
6806    #[serde(rename = "cmdline")]
6807    pub cmd_line: String,
6808    #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
6809    pub last_update: OffsetDateTime,
6810    #[serde(rename = "decommissionInfo")]
6811    pub decommission: Option<PersistedPoolDecommissionInfoV1>,
6812}
6813
6814#[derive(Debug, Clone, Default, Serialize, Deserialize)]
6815#[serde(deny_unknown_fields)]
6816struct PersistedPoolDecommissionInfoV1 {
6817    #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
6818    pub start_time: Option<OffsetDateTime>,
6819    #[serde(rename = "startSize")]
6820    pub start_size: usize,
6821    #[serde(rename = "totalSize")]
6822    pub total_size: usize,
6823    #[serde(rename = "currentSize")]
6824    pub current_size: usize,
6825    #[serde(rename = "complete")]
6826    pub complete: bool,
6827    #[serde(rename = "failed")]
6828    pub failed: bool,
6829    #[serde(rename = "canceled")]
6830    pub canceled: bool,
6831    #[serde(rename = "queued", default)]
6832    pub queued: bool,
6833    #[serde(rename = "queuedBuckets", default)]
6834    pub queued_buckets: Vec<String>,
6835    #[serde(rename = "decommissionedBuckets", default)]
6836    pub decommissioned_buckets: Vec<String>,
6837    #[serde(rename = "bucket", default)]
6838    pub bucket: String,
6839    #[serde(rename = "prefix", default)]
6840    pub prefix: String,
6841    #[serde(rename = "object", default)]
6842    pub object: String,
6843    #[serde(rename = "objectsDecommissioned")]
6844    pub items_decommissioned: usize,
6845    #[serde(rename = "objectsDecommissionedFailed")]
6846    pub items_decommission_failed: usize,
6847    #[serde(rename = "bytesDecommissioned")]
6848    pub bytes_done: usize,
6849    #[serde(rename = "bytesDecommissionedFailed")]
6850    pub bytes_failed: usize,
6851    #[serde(rename = "terminalReloadAttemptAt", with = "time::serde::rfc3339::option", default)]
6852    pub terminal_reload_attempt_at: Option<OffsetDateTime>,
6853    #[serde(rename = "terminalReloadFailures", default)]
6854    pub terminal_reload_failures: Vec<String>,
6855}
6856
6857#[derive(Debug, Clone, Serialize, Deserialize)]
6858#[serde(deny_unknown_fields)]
6859struct LegacyPoolMeta {
6860    pub version: u16,
6861    pub pools: Vec<LegacyPoolStatus>,
6862    pub dont_save: bool,
6863}
6864
6865#[derive(Debug, Clone, Serialize, Deserialize)]
6866#[serde(deny_unknown_fields)]
6867struct LegacyPoolStatus {
6868    #[serde(rename = "id")]
6869    pub id: usize,
6870    #[serde(rename = "cmdline")]
6871    pub cmd_line: String,
6872    #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
6873    pub last_update: OffsetDateTime,
6874    #[serde(rename = "decommissionInfo")]
6875    pub decommission: Option<LegacyPoolDecommissionInfo>,
6876}
6877
6878#[derive(Debug, Clone, Default, Serialize, Deserialize)]
6879#[serde(deny_unknown_fields)]
6880struct LegacyPoolDecommissionInfo {
6881    #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
6882    pub start_time: Option<OffsetDateTime>,
6883    #[serde(rename = "startSize")]
6884    pub start_size: usize,
6885    #[serde(rename = "totalSize")]
6886    pub total_size: usize,
6887    #[serde(rename = "currentSize")]
6888    pub current_size: usize,
6889    #[serde(rename = "complete")]
6890    pub complete: bool,
6891    #[serde(rename = "failed")]
6892    pub failed: bool,
6893    #[serde(rename = "canceled")]
6894    pub canceled: bool,
6895    #[serde(rename = "objectsDecommissioned")]
6896    pub items_decommissioned: usize,
6897    #[serde(rename = "objectsDecommissionedFailed")]
6898    pub items_decommission_failed: usize,
6899    #[serde(rename = "bytesDecommissioned")]
6900    pub bytes_done: usize,
6901    #[serde(rename = "bytesDecommissionedFailed")]
6902    pub bytes_failed: usize,
6903}
6904
6905fn ensure_pool_meta_payload_version(actual: u16, expected: u16, kind: &str) -> Result<()> {
6906    if actual == expected {
6907        return Ok(());
6908    }
6909    Err(Error::other(format!(
6910        "pool metadata {kind} payload has version {actual}, expected {expected}"
6911    )))
6912}
6913
6914impl TryFrom<PersistedPoolMeta> for PoolMeta {
6915    type Error = Error;
6916
6917    fn try_from(value: PersistedPoolMeta) -> Result<Self> {
6918        ensure_pool_meta_payload_version(value.version, POOL_META_VERSION, "current")?;
6919        let meta = Self {
6920            version: POOL_META_VERSION,
6921            pools: value.pools.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?,
6922            dont_save: false,
6923        };
6924        validate_decommission_capacity_model_cohort(&meta)?;
6925        Ok(meta)
6926    }
6927}
6928
6929impl TryFrom<PersistedPoolMetaV1> for PoolMeta {
6930    type Error = Error;
6931
6932    fn try_from(value: PersistedPoolMetaV1) -> Result<Self> {
6933        ensure_pool_meta_payload_version(value.version, POOL_META_V1_VERSION, "v1")?;
6934        Ok(Self {
6935            version: POOL_META_V1_VERSION,
6936            pools: value.pools.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?,
6937            dont_save: false,
6938        })
6939    }
6940}
6941
6942impl TryFrom<LegacyPoolMeta> for PoolMeta {
6943    type Error = Error;
6944
6945    fn try_from(value: LegacyPoolMeta) -> Result<Self> {
6946        let LegacyPoolMeta {
6947            version,
6948            pools,
6949            dont_save: _,
6950        } = value;
6951        ensure_pool_meta_payload_version(version, POOL_META_V1_VERSION, "legacy v1")?;
6952        Ok(Self {
6953            version: POOL_META_V1_VERSION,
6954            pools: pools.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?,
6955            dont_save: false,
6956        })
6957    }
6958}
6959
6960impl TryFrom<PersistedPoolStatus> for PoolStatus {
6961    type Error = Error;
6962
6963    fn try_from(value: PersistedPoolStatus) -> Result<Self> {
6964        let decommission = value.decommission.map(TryInto::try_into).transpose()?;
6965        if decommission
6966            .as_ref()
6967            .and_then(|info: &PoolDecommissionInfo| info.capacity_reservation.as_ref())
6968            .is_some_and(|reservation| reservation.source_pool_index != value.id)
6969        {
6970            return Err(Error::other(
6971                "pool metadata load failed: decommission capacity reservation source pool does not match its owner",
6972            ));
6973        }
6974        Ok(Self {
6975            id: value.id,
6976            cmd_line: value.cmd_line,
6977            last_update: value.last_update,
6978            decommission,
6979        })
6980    }
6981}
6982
6983impl TryFrom<PersistedPoolStatusV1> for PoolStatus {
6984    type Error = Error;
6985
6986    fn try_from(value: PersistedPoolStatusV1) -> Result<Self> {
6987        Ok(Self {
6988            id: value.id,
6989            cmd_line: value.cmd_line,
6990            last_update: value.last_update,
6991            decommission: value.decommission.map(TryInto::try_into).transpose()?,
6992        })
6993    }
6994}
6995
6996impl TryFrom<LegacyPoolStatus> for PoolStatus {
6997    type Error = Error;
6998
6999    fn try_from(value: LegacyPoolStatus) -> Result<Self> {
7000        Ok(Self {
7001            id: value.id,
7002            cmd_line: value.cmd_line,
7003            last_update: value.last_update,
7004            decommission: value.decommission.map(TryInto::try_into).transpose()?,
7005        })
7006    }
7007}
7008
7009impl TryFrom<PersistedPoolDecommissionInfo> for PoolDecommissionInfo {
7010    type Error = Error;
7011
7012    fn try_from(value: PersistedPoolDecommissionInfo) -> Result<Self> {
7013        validate_decommission_terminal_state(value.complete, value.failed, value.canceled)?;
7014        validate_decommission_capacity_reservation(value.capacity_reservation.as_ref())?;
7015        Ok(Self {
7016            start_time: value.start_time,
7017            start_size: value.start_size,
7018            total_size: value.total_size,
7019            current_size: value.current_size,
7020            complete: value.complete,
7021            failed: value.failed,
7022            canceled: value.canceled,
7023            queued: value.queued,
7024            queued_buckets: value.queued_buckets,
7025            decommissioned_buckets: value.decommissioned_buckets,
7026            bucket: value.bucket,
7027            prefix: value.prefix,
7028            object: value.object,
7029            stage: String::new(),
7030            items_decommissioned: value.items_decommissioned,
7031            items_decommission_failed: value.items_decommission_failed,
7032            bytes_done: value.bytes_done,
7033            bytes_failed: value.bytes_failed,
7034            terminal_reload_attempt_at: value.terminal_reload_attempt_at,
7035            terminal_reload_failures: value.terminal_reload_failures,
7036            capacity_reservation: value.capacity_reservation,
7037            capacity_blocked_reason: value.capacity_blocked_reason,
7038            unresolved_entries: value.unresolved_entries,
7039            progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed),
7040            progress_save_retry_after: None,
7041            progress_save_last_at: None,
7042        })
7043    }
7044}
7045
7046impl TryFrom<PersistedPoolDecommissionInfoV1> for PoolDecommissionInfo {
7047    type Error = Error;
7048
7049    fn try_from(value: PersistedPoolDecommissionInfoV1) -> Result<Self> {
7050        validate_decommission_terminal_state(value.complete, value.failed, value.canceled)?;
7051        Ok(Self {
7052            start_time: value.start_time,
7053            start_size: value.start_size,
7054            total_size: value.total_size,
7055            current_size: value.current_size,
7056            complete: value.complete,
7057            failed: value.failed,
7058            canceled: value.canceled,
7059            queued: value.queued,
7060            queued_buckets: value.queued_buckets,
7061            decommissioned_buckets: value.decommissioned_buckets,
7062            bucket: value.bucket,
7063            prefix: value.prefix,
7064            object: value.object,
7065            stage: String::new(),
7066            items_decommissioned: value.items_decommissioned,
7067            items_decommission_failed: value.items_decommission_failed,
7068            bytes_done: value.bytes_done,
7069            bytes_failed: value.bytes_failed,
7070            terminal_reload_attempt_at: value.terminal_reload_attempt_at,
7071            terminal_reload_failures: value.terminal_reload_failures,
7072            capacity_reservation: None,
7073            capacity_blocked_reason: None,
7074            unresolved_entries: Vec::new(),
7075            progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed),
7076            progress_save_retry_after: None,
7077            progress_save_last_at: None,
7078        })
7079    }
7080}
7081
7082impl TryFrom<LegacyPoolDecommissionInfo> for PoolDecommissionInfo {
7083    type Error = Error;
7084
7085    fn try_from(value: LegacyPoolDecommissionInfo) -> Result<Self> {
7086        validate_decommission_terminal_state(value.complete, value.failed, value.canceled)?;
7087        Ok(Self {
7088            start_time: value.start_time,
7089            start_size: value.start_size,
7090            total_size: value.total_size,
7091            current_size: value.current_size,
7092            complete: value.complete,
7093            failed: value.failed,
7094            canceled: value.canceled,
7095            queued: false,
7096            queued_buckets: Vec::new(),
7097            decommissioned_buckets: Vec::new(),
7098            bucket: String::new(),
7099            prefix: String::new(),
7100            object: String::new(),
7101            stage: String::new(),
7102            items_decommissioned: value.items_decommissioned,
7103            items_decommission_failed: value.items_decommission_failed,
7104            bytes_done: value.bytes_done,
7105            bytes_failed: value.bytes_failed,
7106            terminal_reload_attempt_at: None,
7107            terminal_reload_failures: Vec::new(),
7108            capacity_reservation: None,
7109            capacity_blocked_reason: None,
7110            unresolved_entries: Vec::new(),
7111            progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed),
7112            progress_save_retry_after: None,
7113            progress_save_last_at: None,
7114        })
7115    }
7116}
7117
7118impl From<&PoolMeta> for PersistedPoolMeta {
7119    fn from(value: &PoolMeta) -> Self {
7120        Self {
7121            version: POOL_META_VERSION,
7122            pools: value.pools.iter().map(Into::into).collect(),
7123        }
7124    }
7125}
7126
7127impl From<&PoolMeta> for PersistedPoolMetaV1 {
7128    fn from(value: &PoolMeta) -> Self {
7129        Self {
7130            version: POOL_META_V1_VERSION,
7131            pools: value.pools.iter().map(Into::into).collect(),
7132        }
7133    }
7134}
7135
7136impl From<&PoolStatus> for PersistedPoolStatus {
7137    fn from(value: &PoolStatus) -> Self {
7138        Self {
7139            id: value.id,
7140            cmd_line: value.cmd_line.clone(),
7141            last_update: value.last_update,
7142            decommission: value.decommission.as_ref().map(Into::into),
7143        }
7144    }
7145}
7146
7147impl From<&PoolStatus> for PersistedPoolStatusV1 {
7148    fn from(value: &PoolStatus) -> Self {
7149        Self {
7150            id: value.id,
7151            cmd_line: value.cmd_line.clone(),
7152            last_update: value.last_update,
7153            decommission: value.decommission.as_ref().map(Into::into),
7154        }
7155    }
7156}
7157
7158impl From<&PoolDecommissionInfo> for PersistedPoolDecommissionInfo {
7159    fn from(value: &PoolDecommissionInfo) -> Self {
7160        Self {
7161            start_time: value.start_time,
7162            start_size: value.start_size,
7163            total_size: value.total_size,
7164            current_size: value.current_size,
7165            complete: value.complete,
7166            failed: value.failed,
7167            canceled: value.canceled,
7168            queued: value.queued,
7169            queued_buckets: value.queued_buckets.clone(),
7170            decommissioned_buckets: value.decommissioned_buckets.clone(),
7171            bucket: value.bucket.clone(),
7172            prefix: value.prefix.clone(),
7173            object: value.object.clone(),
7174            items_decommissioned: value.items_decommissioned,
7175            items_decommission_failed: value.items_decommission_failed,
7176            bytes_done: value.bytes_done,
7177            bytes_failed: value.bytes_failed,
7178            terminal_reload_attempt_at: value.terminal_reload_attempt_at,
7179            terminal_reload_failures: value.terminal_reload_failures.clone(),
7180            capacity_reservation: value.capacity_reservation.clone(),
7181            capacity_blocked_reason: value.capacity_blocked_reason.clone(),
7182            unresolved_entries: value.unresolved_entries.clone(),
7183        }
7184    }
7185}
7186
7187impl From<&PoolDecommissionInfo> for PersistedPoolDecommissionInfoV1 {
7188    fn from(value: &PoolDecommissionInfo) -> Self {
7189        Self {
7190            start_time: value.start_time,
7191            start_size: value.start_size,
7192            total_size: value.total_size,
7193            current_size: value.current_size,
7194            complete: value.complete,
7195            failed: value.failed,
7196            canceled: value.canceled,
7197            queued: value.queued,
7198            queued_buckets: value.queued_buckets.clone(),
7199            decommissioned_buckets: value.decommissioned_buckets.clone(),
7200            bucket: value.bucket.clone(),
7201            prefix: value.prefix.clone(),
7202            object: value.object.clone(),
7203            items_decommissioned: value.items_decommissioned,
7204            items_decommission_failed: value.items_decommission_failed,
7205            bytes_done: value.bytes_done,
7206            bytes_failed: value.bytes_failed,
7207            terminal_reload_attempt_at: value.terminal_reload_attempt_at,
7208            terminal_reload_failures: value.terminal_reload_failures.clone(),
7209        }
7210    }
7211}
7212
7213#[derive(Debug)]
7214struct PoolMetaSaveOutcome {
7215    transaction_arm: PoolMetaTransactionArm,
7216    committed: PoolMeta,
7217}
7218
7219/// Reconcile only a known, interrupted pool.bin transaction. The caller keeps
7220/// the original block installed and holds the namespace writer through runtime
7221/// publication. Never feed speculative in-memory progress into this repair.
7222async fn load_pool_meta_for_transaction_recovery<S>(
7223    pools: Vec<Arc<S>>,
7224    write_state: &mut PoolMetaWriteState,
7225) -> Result<PoolMetaSelection>
7226where
7227    S: EcstoreObjectIO,
7228{
7229    let result = async {
7230        if let Some(cluster_id) = write_state.expected_cluster_id {
7231            let reads = join_all(pools.iter().cloned().map(read_pool_meta_identity_replica)).await;
7232            for read in &reads {
7233                match &read.replica {
7234                    PoolMetaIdentityReplica::Unreadable(source) => {
7235                        return Err(Error::other(pool_metadata_error(
7236                            crate::error::PoolMetadataFailure::ReadUnavailable,
7237                            "recovery_identity_read",
7238                            Some(Arc::clone(source)),
7239                        )));
7240                    }
7241                    PoolMetaIdentityReplica::Corrupt(_) | PoolMetaIdentityReplica::Incompatible(_) => {
7242                        return Err(Error::other("pool metadata recovery requires operator repair of cluster identity"));
7243                    }
7244                    _ => {}
7245                }
7246            }
7247            let identity = select_pool_meta_identity(reads, cluster_id)?;
7248            if !identity.identity.is_some_and(|identity| identity.initialized) {
7249                return Err(Error::other("pool metadata recovery requires an initialized durable cluster identity"));
7250            }
7251            write_state.observe_identity(&identity)?;
7252        }
7253        let reads = read_pool_meta_replicas(pools, true).await;
7254        if let Some(err) = pool_meta_replica_read_failure(&reads, "pool metadata transaction recovery") {
7255            return Err(err);
7256        }
7257        if reads
7258            .iter()
7259            .any(|read| matches!(&read.replica, PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)))
7260        {
7261            return Err(Error::other(
7262                "pool metadata recovery requires operator repair of corrupt or incompatible replicas",
7263            ));
7264        }
7265        let selection = select_pool_meta_replica_reads(reads)?;
7266        write_state.observe_selection(&selection)?;
7267        if selection.absent || selection.meta.pools.is_empty() {
7268            return Err(Error::other("pool metadata recovery cannot initialize missing metadata"));
7269        }
7270        selection
7271            .replica_state
7272            .ensure_write_safe("pool metadata transaction recovery")?;
7273        Ok(selection)
7274    }
7275    .await;
7276    result.map_err(|err: Error| {
7277        if err.pool_metadata_failure().is_some() {
7278            err
7279        } else {
7280            Error::other(pool_metadata_error(
7281                crate::error::PoolMetadataFailure::RecoveryRequired,
7282                "recovery_validation",
7283                Some(Arc::new(err)),
7284            ))
7285        }
7286    })
7287}
7288
7289async fn repair_pool_meta_transaction<S>(
7290    pools: Vec<Arc<S>>,
7291    write_state: &mut PoolMetaWriteState,
7292    selection: PoolMetaSelection,
7293    fence: &PoolMetaPersistenceFence<'_>,
7294) -> Result<PoolMetaSaveOutcome>
7295where
7296    S: EcstoreObjectIO,
7297{
7298    let mut transaction_arm = write_state.arm_transaction();
7299    let mut expected_meta = selection.meta.clone();
7300    let mut expected_revision = selection.revision;
7301    // A failed first V3 prepare still establishes the format floor. Commit the
7302    // previous authoritative state as V3 instead of erasing that observation.
7303    if selection.generation_protocol_observed && !expected_revision.is_generation_protocol() {
7304        let (cluster_id, epoch) = selection
7305            .generation_identity
7306            .ok_or_else(|| Error::other("pool metadata recovery has no V3 cluster identity"))?;
7307        expected_meta.version = POOL_META_GENERATION_VERSION;
7308        expected_revision = PoolMetaRevision {
7309            version: POOL_META_GENERATION_VERSION,
7310            cluster_id: Some(cluster_id),
7311            epoch,
7312            generation: 1,
7313            transaction_id: Some(uuid::Uuid::new_v4()),
7314        };
7315    }
7316    let expected_canonical = if expected_revision.is_generation_protocol() {
7317        encode_pool_meta_v3_envelope(&expected_meta, expected_revision, true, None)?
7318    } else {
7319        expected_meta.encode_config_data_for_v2_gate(true)?
7320    };
7321    if selection.replica_state.needs_repair {
7322        let data = if expected_revision.is_generation_protocol() {
7323            expected_canonical.clone()
7324        } else {
7325            expected_meta.encode_config_data_for_v2_gate(expected_revision.version == POOL_META_VERSION)?
7326        };
7327        for (pool, token) in pools.iter().cloned().zip(&selection.cas_tokens) {
7328            save_pool_meta_object_cas(pool, POOL_META_NAME, data.clone(), token, fence, "recovery_cas", &mut transaction_arm)
7329                .await?;
7330        }
7331    }
7332    persist_pool_meta_identity(
7333        pools.clone(),
7334        write_state,
7335        true,
7336        PoolMetaIdentityWriteScope::All,
7337        fence,
7338        &mut transaction_arm,
7339    )
7340    .await?;
7341    let confirmed = load_pool_meta_for_transaction_recovery(pools, write_state).await?;
7342    if confirmed.revision != expected_revision
7343        || confirmed.canonical.as_ref() != Some(&expected_canonical)
7344        || confirmed.replica_state.needs_repair
7345        || write_state.identity_requires_repair()
7346    {
7347        return Err(Error::PreconditionFailed);
7348    }
7349    fence.ensure_held()?;
7350    Ok(PoolMetaSaveOutcome {
7351        transaction_arm,
7352        committed: confirmed.meta,
7353    })
7354}
7355
7356impl PoolMetaSaveOutcome {
7357    fn disarm(mut self) {
7358        self.transaction_arm.disarm();
7359    }
7360
7361    fn into_committed(mut self) -> PoolMeta {
7362        self.transaction_arm.disarm();
7363        self.committed
7364    }
7365}
7366
7367impl PoolMeta {
7368    fn current_decommission_movement_update(&self) -> Option<OffsetDateTime> {
7369        self.pools
7370            .iter()
7371            .filter(|pool| pool.decommission.is_some())
7372            .map(|pool| pool.last_update)
7373            .max()
7374    }
7375
7376    fn current_rebalance_movement_update(rebalance_meta: Option<&RebalanceMeta>) -> Option<OffsetDateTime> {
7377        rebalance_meta
7378            .into_iter()
7379            .flat_map(|meta| {
7380                meta.stopped_at.into_iter().chain(
7381                    meta.pool_stats
7382                        .iter()
7383                        .flat_map(|pool| [pool.info.start_time, pool.info.end_time])
7384                        .flatten(),
7385                )
7386            })
7387            .max()
7388    }
7389
7390    pub(crate) fn next_scanner_data_movement_update(
7391        &self,
7392        now: OffsetDateTime,
7393        rebalance_meta: Option<&RebalanceMeta>,
7394    ) -> OffsetDateTime {
7395        let max_seen = self
7396            .current_decommission_movement_update()
7397            .max(Self::current_rebalance_movement_update(rebalance_meta));
7398        match max_seen {
7399            Some(max_seen) => max_seen
7400                .checked_add(Duration::nanoseconds(1))
7401                .map(|next| now.max(next))
7402                .unwrap_or(max_seen),
7403            None => now,
7404        }
7405    }
7406
7407    fn decode_pool_meta_payload(version: u16, payload: &[u8]) -> Result<Self> {
7408        match version {
7409            POOL_META_VERSION => rmp_serde::from_slice::<PersistedPoolMeta>(payload)
7410                .map_err(|err| Error::other(format!("PoolMeta v{POOL_META_VERSION} decode failed: {err}")))?
7411                .try_into(),
7412            POOL_META_V1_VERSION => match rmp_serde::from_slice::<PersistedPoolMetaV1>(payload) {
7413                Ok(meta) => meta.try_into(),
7414                Err(persisted_err) => {
7415                    let legacy: LegacyPoolMeta = rmp_serde::from_slice(payload).map_err(|legacy_err| {
7416                        Error::other(format!(
7417                            "PoolMeta v1 decode failed for both persisted and legacy formats: persisted={persisted_err}; legacy={legacy_err}"
7418                        ))
7419                    })?;
7420                    legacy.try_into()
7421                }
7422            },
7423            _ => Err(Error::other(format!("pool metadata load failed: unknown version {version}"))),
7424        }
7425    }
7426
7427    pub fn new(pools: &[Arc<Sets>], prev_meta: &PoolMeta) -> Self {
7428        let mut new_meta = Self {
7429            version: if prev_meta.version == POOL_META_GENERATION_VERSION || pool_meta_v3_writer_enabled() {
7430                POOL_META_GENERATION_VERSION
7431            } else if prev_meta.version == POOL_META_VERSION || pool_meta_v2_writer_enabled() {
7432                POOL_META_VERSION
7433            } else {
7434                POOL_META_V1_VERSION
7435            },
7436            pools: Vec::new(),
7437            ..Default::default()
7438        };
7439
7440        for (idx, pool) in pools.iter().enumerate() {
7441            let mut skip = false;
7442
7443            for current_pool in prev_meta.pools.iter() {
7444                if current_pool.cmd_line == pool.endpoints.cmd_line {
7445                    new_meta.pools.push(current_pool.clone());
7446                    skip = true;
7447                    break;
7448                }
7449            }
7450
7451            if skip {
7452                continue;
7453            }
7454
7455            new_meta.pools.push(PoolStatus {
7456                cmd_line: pool.endpoints.cmd_line.clone(),
7457                id: idx,
7458                last_update: OffsetDateTime::now_utc(),
7459                decommission: None,
7460            });
7461        }
7462
7463        new_meta
7464    }
7465
7466    pub fn is_suspended(&self, idx: usize) -> bool {
7467        self.pools
7468            .get(idx)
7469            .and_then(|pool| pool.decommission.as_ref())
7470            .is_some_and(is_decommission_suspended)
7471    }
7472
7473    pub(crate) fn has_active_decommission_capacity_reservation(&self, idx: usize) -> bool {
7474        self.pools
7475            .get(idx)
7476            .and_then(|pool| pool.decommission.as_ref())
7477            .is_some_and(|info| {
7478                info.has_decommission_state()
7479                    && is_decommission_active(info.complete, info.failed, info.canceled)
7480                    && info
7481                        .capacity_reservation
7482                        .as_ref()
7483                        .is_some_and(DecommissionCapacityReservation::active)
7484            })
7485    }
7486
7487    pub(crate) fn scanner_pause_backlog_pool_writable(&self, idx: usize) -> bool {
7488        self.pools.get(idx).is_some_and(|pool| {
7489            !pool
7490                .decommission
7491                .as_ref()
7492                .is_some_and(|info| info.has_decommission_state() && !info.failed && !info.canceled)
7493        })
7494    }
7495
7496    fn mark_decommission_progress_saved(&mut self) {
7497        for pool in &mut self.pools {
7498            if let Some(info) = pool.decommission.as_mut() {
7499                info.mark_progress_saved();
7500                info.progress_save_last_at = Some(pool.last_update);
7501            }
7502        }
7503    }
7504
7505    fn decommission_progress_checkpoint(
7506        &self,
7507        idx: usize,
7508        duration: Duration,
7509        now: OffsetDateTime,
7510        rebalance_meta: Option<&RebalanceMeta>,
7511    ) -> Result<Option<DecommissionProgressCheckpoint>> {
7512        let pool_count = self.pools.len();
7513        ensure_valid_decommission_pool_index(pool_count, idx)?;
7514
7515        let Some(pool) = self.pools.get(idx) else {
7516            return Err(invalid_decommission_pool_index_error(pool_count, idx));
7517        };
7518        let Some(info) = pool.decommission.as_ref() else {
7519            return Err(decommission_metadata_not_initialized_error("update decommission metadata timestamp"));
7520        };
7521
7522        if info.progress_save_retry_after.is_some_and(|retry_after| now < retry_after) {
7523            return Ok(None);
7524        }
7525
7526        let last_progress_save = info.progress_save_last_at.unwrap_or(pool.last_update);
7527        let time_threshold_reached = now.unix_timestamp() - last_progress_save.unix_timestamp() >= duration.whole_seconds();
7528        let item_threshold_reached = info.items_since_last_progress_save() >= DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD;
7529        if !time_threshold_reached && !item_threshold_reached {
7530            return Ok(None);
7531        }
7532
7533        Ok(Some(DecommissionProgressCheckpoint {
7534            start_time: info.start_time,
7535            queued: info.queued,
7536            counted_items: info.counted_items(),
7537            checkpoint_at: self.next_scanner_data_movement_update(now, rebalance_meta),
7538            capacity_operation_id: info
7539                .capacity_reservation
7540                .as_ref()
7541                .filter(|reservation| reservation.lease_active_at(now))
7542                .map(|reservation| reservation.operation_id),
7543            capacity_owner_nonce: info
7544                .capacity_reservation
7545                .as_ref()
7546                .filter(|reservation| reservation.lease_active_at(now))
7547                .map(|reservation| reservation.owner_nonce),
7548            capacity_lease_expires_at: info
7549                .capacity_reservation
7550                .as_ref()
7551                .filter(|reservation| reservation.lease_active_at(now))
7552                .map(|_| now + DECOMMISSION_CAPACITY_RESERVATION_TTL),
7553        }))
7554    }
7555
7556    fn commit_decommission_progress_checkpoint(&mut self, idx: usize, checkpoint: DecommissionProgressCheckpoint) -> bool {
7557        let Some(pool) = self.pools.get_mut(idx) else {
7558            return false;
7559        };
7560        let Some(info) = pool.decommission.as_mut() else {
7561            return false;
7562        };
7563
7564        if info.start_time != checkpoint.start_time
7565            || info.queued != checkpoint.queued
7566            || !is_decommission_active(info.complete, info.failed, info.canceled)
7567        {
7568            return false;
7569        }
7570
7571        info.progress_save_item_baseline = info.progress_save_item_baseline.max(checkpoint.counted_items);
7572        info.progress_save_retry_after = None;
7573        info.progress_save_last_at = Some(
7574            info.progress_save_last_at
7575                .map_or(checkpoint.checkpoint_at, |at| at.max(checkpoint.checkpoint_at)),
7576        );
7577        if let (Some(operation_id), Some(owner_nonce), Some(expires_at), Some(reservation)) = (
7578            checkpoint.capacity_operation_id,
7579            checkpoint.capacity_owner_nonce,
7580            checkpoint.capacity_lease_expires_at,
7581            info.capacity_reservation.as_mut(),
7582        ) && reservation.operation_id == operation_id
7583            && reservation.owner_nonce == owner_nonce
7584            && reservation.active()
7585        {
7586            reservation.renewed_at = checkpoint.checkpoint_at;
7587            reservation.expires_at = expires_at;
7588        }
7589        pool.last_update = pool.last_update.max(checkpoint.checkpoint_at);
7590        true
7591    }
7592
7593    fn defer_decommission_progress_checkpoint(
7594        &mut self,
7595        idx: usize,
7596        checkpoint: DecommissionProgressCheckpoint,
7597        retry_after: OffsetDateTime,
7598    ) {
7599        let Some(pool) = self.pools.get_mut(idx) else {
7600            return;
7601        };
7602        let Some(info) = pool.decommission.as_mut() else {
7603            return;
7604        };
7605
7606        if info.start_time == checkpoint.start_time
7607            && info.queued == checkpoint.queued
7608            && is_decommission_active(info.complete, info.failed, info.canceled)
7609        {
7610            info.progress_save_retry_after = Some(retry_after);
7611        }
7612    }
7613
7614    fn load_from_config_data(&mut self, data: Vec<u8>) -> Result<()> {
7615        if data.is_empty() {
7616            return Ok(());
7617        } else if data.len() <= 4 {
7618            return Err(Error::other("pool metadata load failed: metadata payload is too short"));
7619        }
7620
7621        let format = LittleEndian::read_u16(&data[0..2]);
7622        if format != POOL_META_FORMAT {
7623            return Err(Error::other(format!("pool metadata load failed: unknown format {format}")));
7624        }
7625        let version = LittleEndian::read_u16(&data[2..4]);
7626        if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) {
7627            return Err(Error::other(format!("pool metadata load failed: unknown version {version}")));
7628        }
7629
7630        if version == POOL_META_GENERATION_VERSION {
7631            let PoolMetaReplica::Valid {
7632                meta, committed: true, ..
7633            } = decode_pool_meta_replica(data)
7634            else {
7635                return Err(Error::other(
7636                    "pool metadata load failed: V3 payload is corrupt, incompatible, or not committed",
7637                ));
7638            };
7639            *self = meta;
7640            return Ok(());
7641        }
7642
7643        *self = Self::decode_pool_meta_payload(version, &data[4..])?;
7644
7645        if !matches!(self.version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) {
7646            return Err(Error::other(format!(
7647                "pool metadata load failed: unexpected decoded version {}",
7648                self.version
7649            )));
7650        }
7651        Ok(())
7652    }
7653
7654    pub async fn load(&mut self, pool: Arc<Sets>, pools: Vec<Arc<Sets>>) -> Result<()> {
7655        let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
7656        let _pool_meta_guard = pool_meta_lock.get_read_lock(get_lock_acquire_timeout()).await?;
7657        let replica_state = self.load_no_lock_from_replicas(pools).await?;
7658        replica_state.ensure_write_safe("pool metadata load failed")?;
7659        Ok(())
7660    }
7661
7662    /// Loads every pool metadata replica while the caller owns the metadata fence
7663    /// or before the namespace-lock RPC surface is ready during startup.
7664    pub(crate) async fn load_no_lock_from_replicas<S>(&mut self, pools: Vec<Arc<S>>) -> Result<PoolMetaReplicaState>
7665    where
7666        S: EcstoreObjectIO,
7667    {
7668        let selection = load_pool_meta_replicas(pools, true).await?;
7669        *self = selection.meta;
7670        Ok(selection.replica_state)
7671    }
7672
7673    pub(crate) async fn load_no_lock_from_replicas_observing<S>(
7674        &mut self,
7675        pools: Vec<Arc<S>>,
7676        write_state: &mut PoolMetaWriteState,
7677    ) -> Result<PoolMetaReplicaState>
7678    where
7679        S: EcstoreObjectIO,
7680    {
7681        let selection = load_pool_meta_replicas_observing(pools, true, write_state).await?;
7682        *self = selection.meta;
7683        Ok(selection.replica_state)
7684    }
7685
7686    pub(crate) async fn load_for_startup_observing<S>(
7687        &mut self,
7688        pools: Vec<Arc<S>>,
7689        write_state: &mut PoolMetaWriteState,
7690    ) -> Result<PoolMetaReplicaState>
7691    where
7692        S: EcstoreObjectIO,
7693    {
7694        // Startup may install a readable replica without authorizing writes.
7695        // Runtime preflight retries must not inherit this bootstrap policy.
7696        if let Some(cluster_id) = write_state.expected_cluster_id {
7697            let reads = join_all(pools.iter().cloned().map(read_pool_meta_identity_replica)).await;
7698            if let Some(source) = reads.iter().find_map(|read| match &read.replica {
7699                PoolMetaIdentityReplica::Unreadable(source) => Some(Arc::clone(source)),
7700                _ => None,
7701            }) {
7702                write_state.block_with_context(pool_metadata_error(
7703                    crate::error::PoolMetadataFailure::RecoveryRequired,
7704                    "startup_identity_read",
7705                    Some(source),
7706                ));
7707            }
7708            let identity = select_pool_meta_identity(reads, cluster_id)
7709                .map_err(|err| block_pool_meta_validation(write_state, err, "startup_identity"))?;
7710            write_state.observe_identity(&identity)?;
7711        }
7712        let reads = read_pool_meta_replicas(pools, true).await;
7713        if let Some(err) = pool_meta_replica_read_failure(&reads, "pool metadata load") {
7714            write_state.block_with_context(pool_metadata_error(
7715                crate::error::PoolMetadataFailure::RecoveryRequired,
7716                "startup_read",
7717                Some(Arc::new(err)),
7718            ));
7719        }
7720        let selection = select_pool_meta_replica_reads(reads)
7721            .map_err(|err| block_pool_meta_validation(write_state, err, "startup_selection"))?;
7722        write_state
7723            .observe_selection(&selection)
7724            .map_err(|err| block_pool_meta_validation(write_state, err, "startup_selection"))?;
7725        *self = selection.meta;
7726        Ok(selection.replica_state)
7727    }
7728
7729    #[cfg(test)]
7730    fn encode_config_data(&self) -> Result<Vec<u8>> {
7731        self.encode_config_data_for_v2_gate(pool_meta_v2_writer_enabled())
7732    }
7733
7734    fn encode_config_data_for_v2_gate(&self, v2_enabled: bool) -> Result<Vec<u8>> {
7735        if self.dont_save {
7736            return Ok(Vec::new());
7737        }
7738        if self.version == POOL_META_GENERATION_VERSION {
7739            return Err(Error::other(
7740                "pool metadata V3 save requires cluster identity and generation transaction context",
7741            ));
7742        }
7743        if !matches!(self.version, 0 | POOL_META_V1_VERSION | POOL_META_VERSION) {
7744            return Err(Error::other(format!(
7745                "pool metadata save failed: unexpected runtime version {}",
7746                self.version
7747            )));
7748        }
7749        let version = if self.version == POOL_META_VERSION || v2_enabled {
7750            POOL_META_VERSION
7751        } else {
7752            POOL_META_V1_VERSION
7753        };
7754        if version == POOL_META_V1_VERSION {
7755            let mut decommission_infos = self.pools.iter().filter_map(|pool| pool.decommission.as_ref());
7756            if decommission_infos.clone().any(|info| !info.unresolved_entries.is_empty()) {
7757                return Err(Error::other(format!(
7758                    "pool metadata V2 is required to persist unresolved decommission entries; enable both {} and {} only after every reader and writer supports V2",
7759                    rustfs_config::ENV_POOL_META_V2_WRITE,
7760                    rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
7761                )));
7762            }
7763            if decommission_infos.any(|info| info.capacity_reservation.is_some()) {
7764                return Err(Error::DecommissionCapacity(format!(
7765                    "pool metadata V2 is required to persist decommission capacity reservations; enable both {} and {} only after every reader and writer supports V2",
7766                    rustfs_config::ENV_POOL_META_V2_WRITE,
7767                    rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
7768                )));
7769            }
7770        }
7771        let mut data = Vec::new();
7772        data.write_u16::<LittleEndian>(POOL_META_FORMAT)?;
7773        data.write_u16::<LittleEndian>(version)?;
7774        let mut buf = Vec::new();
7775        match version {
7776            POOL_META_V1_VERSION => PersistedPoolMetaV1::from(self).serialize(&mut Serializer::new(&mut buf))?,
7777            POOL_META_VERSION => PersistedPoolMeta::from(self).serialize(&mut Serializer::new(&mut buf))?,
7778            _ => unreachable!("pool metadata writer selected an unsupported version"),
7779        }
7780        data.write_all(&buf)?;
7781        Ok(data)
7782    }
7783
7784    #[cfg(test)]
7785    pub(crate) fn encode_config_data_for_test(&self) -> Result<Vec<u8>> {
7786        self.encode_config_data_for_v2_gate(true)
7787    }
7788
7789    pub async fn save(&self, pools: Vec<Arc<Sets>>) -> Result<()> {
7790        let pool = pools
7791            .first()
7792            .cloned()
7793            .ok_or_else(|| Error::other("pool metadata save failed: no storage pools available"))?;
7794        let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
7795        let pool_meta_guard = pool_meta_lock.get_write_lock(get_lock_acquire_timeout()).await?;
7796        let mut write_state = PoolMetaWriteState::default();
7797        let indices = (0..self.pools.len()).collect::<Vec<_>>();
7798        let outcome = self
7799            .save_no_lock_armed_scoped(pools, &mut write_state, pool_meta_guard.lock_lost_signal(), Some(&indices))
7800            .await?;
7801        outcome.disarm();
7802        Ok(())
7803    }
7804
7805    /// Startup has a single elected local writer, so it must not depend on namespace locks here.
7806    #[cfg(any(test, feature = "test-util"))]
7807    pub(crate) async fn save_for_startup<S>(&self, pools: Vec<Arc<S>>) -> Result<()>
7808    where
7809        S: EcstoreObjectIO,
7810    {
7811        let mut write_state = PoolMetaWriteState::for_test_bootstrap();
7812        self.save_for_startup_observing(pools, &mut write_state).await.map(|_| ())
7813    }
7814
7815    pub(crate) async fn save_for_startup_observing<S>(
7816        &self,
7817        pools: Vec<Arc<S>>,
7818        write_state: &mut PoolMetaWriteState,
7819    ) -> Result<PoolMeta>
7820    where
7821        S: EcstoreObjectIO,
7822    {
7823        let outcome = self.save_no_lock_armed_scoped(pools, write_state, None, None).await?;
7824        Ok(outcome.into_committed())
7825    }
7826
7827    async fn save_no_lock_with_fence<S>(
7828        &self,
7829        pools: Vec<Arc<S>>,
7830        lock_lost: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
7831        indices: &[usize],
7832    ) -> Result<()>
7833    where
7834        S: EcstoreObjectIO,
7835    {
7836        let mut write_state = PoolMetaWriteState::default();
7837        let outcome = self
7838            .save_no_lock_armed_scoped(pools, &mut write_state, lock_lost, Some(indices))
7839            .await?;
7840        outcome.disarm();
7841        Ok(())
7842    }
7843
7844    async fn save_no_lock_with_activation_fence<S>(
7845        &self,
7846        pools: Vec<Arc<S>>,
7847        write_state: &mut PoolMetaWriteState,
7848        activation_fence: &PoolRebalanceActivationFence,
7849        indices: &[usize],
7850    ) -> Result<PoolMetaSaveOutcome>
7851    where
7852        S: EcstoreObjectIO,
7853    {
7854        write_state.ensure_write_safe("pool metadata activation save failed")?;
7855        let mut transaction_arm = write_state.arm_transaction();
7856        let fence = PoolMetaPersistenceFence::Activation(activation_fence);
7857        let committed = self
7858            .save_no_lock_transaction(pools, write_state, &fence, Some(indices), &mut transaction_arm)
7859            .await
7860            .map_err(|err| transaction_arm.failure(err))?;
7861        if transaction_arm.phase.is_some() {
7862            transaction_arm.phase = Some("publication");
7863        }
7864        Ok(PoolMetaSaveOutcome {
7865            transaction_arm,
7866            committed,
7867        })
7868    }
7869
7870    async fn save_no_lock_armed<S>(
7871        &self,
7872        pools: Vec<Arc<S>>,
7873        write_state: &mut PoolMetaWriteState,
7874        lock_lost: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
7875        indices: &[usize],
7876    ) -> Result<PoolMetaSaveOutcome>
7877    where
7878        S: EcstoreObjectIO,
7879    {
7880        self.save_no_lock_armed_scoped(pools, write_state, lock_lost, Some(indices))
7881            .await
7882    }
7883
7884    async fn save_no_lock_armed_scoped<S>(
7885        &self,
7886        pools: Vec<Arc<S>>,
7887        write_state: &mut PoolMetaWriteState,
7888        lock_lost: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
7889        indices: Option<&[usize]>,
7890    ) -> Result<PoolMetaSaveOutcome>
7891    where
7892        S: EcstoreObjectIO,
7893    {
7894        write_state.ensure_write_safe("pool metadata save failed")?;
7895        // The arm does not block its own transaction. If this future or its
7896        // returned outcome is dropped before publication, Drop latches the
7897        // sticky recovery gate.
7898        let mut transaction_arm = write_state.arm_transaction();
7899        let fence = PoolMetaPersistenceFence::Distributed(lock_lost);
7900        let committed = self
7901            .save_no_lock_transaction(pools, write_state, &fence, indices, &mut transaction_arm)
7902            .await
7903            .map_err(|err| transaction_arm.failure(err))?;
7904        if transaction_arm.phase.is_some() {
7905            transaction_arm.phase = Some("publication");
7906        }
7907        Ok(PoolMetaSaveOutcome {
7908            transaction_arm,
7909            committed,
7910        })
7911    }
7912
7913    async fn save_no_lock_transaction<S>(
7914        &self,
7915        pools: Vec<Arc<S>>,
7916        write_state: &mut PoolMetaWriteState,
7917        fence: &PoolMetaPersistenceFence<'_>,
7918        indices: Option<&[usize]>,
7919        transaction_arm: &mut PoolMetaTransactionArm,
7920    ) -> Result<PoolMeta>
7921    where
7922        S: EcstoreObjectIO,
7923    {
7924        if pools.is_empty() {
7925            return Err(Error::other("pool metadata save failed: no storage pools available"));
7926        }
7927        if self.dont_save {
7928            return Ok(self.clone());
7929        }
7930        for attempt in 0..POOL_META_CAS_MAX_ATTEMPTS {
7931            match self
7932                .save_no_lock_transaction_once(pools.clone(), write_state, fence, indices, transaction_arm)
7933                .await
7934            {
7935                Ok(committed) => return Ok(committed),
7936                Err(Error::PreconditionFailed) if attempt + 1 < POOL_META_CAS_MAX_ATTEMPTS => continue,
7937                Err(err) => return Err(err),
7938            }
7939        }
7940        Err(Error::PreconditionFailed)
7941    }
7942
7943    async fn save_no_lock_transaction_once<S>(
7944        &self,
7945        pools: Vec<Arc<S>>,
7946        write_state: &mut PoolMetaWriteState,
7947        fence: &PoolMetaPersistenceFence<'_>,
7948        indices: Option<&[usize]>,
7949        transaction_arm: &mut PoolMetaTransactionArm,
7950    ) -> Result<PoolMeta>
7951    where
7952        S: EcstoreObjectIO,
7953    {
7954        fence.ensure_held()?;
7955        let selection = load_pool_meta_replicas_observing(pools.clone(), true, write_state).await?;
7956        write_state.observe_replicas(selection.replica_state);
7957        write_state.ensure_write_safe("pool metadata save failed")?;
7958        let mut bootstrap_generation_required = false;
7959        if let Some(cluster_id) = write_state.expected_cluster_id {
7960            let identity = load_pool_meta_identity_selection_observing(pools.clone(), write_state, cluster_id).await?;
7961            bootstrap_generation_required = selection.absent || write_state.identity_initialized == Some(false);
7962            if !identity.repair_write_safe {
7963                write_state.block_writes();
7964                return Err(Error::other(
7965                    "pool metadata recovery required: cluster identity has an unreadable replica",
7966                ));
7967            }
7968            if let Some(identity) = identity.identity
7969                && selection.revision.is_generation_protocol()
7970                && identity.epoch != selection.revision.epoch
7971            {
7972                write_state.block_writes();
7973                return Err(Error::other(format!(
7974                    "pool metadata recovery required: committed epoch {} does not match cluster identity epoch {}",
7975                    selection.revision.epoch, identity.epoch
7976                )));
7977            }
7978            if !selection.absent && write_state.identity_requires_repair() {
7979                let initialized = write_state.identity_initialized != Some(false) || selection.revision.is_generation_protocol();
7980                persist_pool_meta_identity(
7981                    pools.clone(),
7982                    write_state,
7983                    initialized,
7984                    PoolMetaIdentityWriteScope::All,
7985                    fence,
7986                    transaction_arm,
7987                )
7988                .await?;
7989            }
7990        }
7991        // Startup is the only path allowed to create an all-missing metadata
7992        // set. Runtime callers without an identity context must fail closed.
7993        write_state.ensure_missing_metadata_can_initialize()?;
7994
7995        let mut committed = if let Some(indices) = indices {
7996            if selection.meta.pools.is_empty() {
7997                self.clone()
7998            } else {
7999                let mut requested = self.clone();
8000                requested.version = selection.meta.version;
8001                let mut latest = selection.meta.clone();
8002                merge_pool_meta_updates_for_save(&mut latest, &requested, indices, "pool metadata save failed")?;
8003                latest
8004            }
8005        } else {
8006            self.clone()
8007        };
8008        let target_version = if selection.generation_protocol_observed
8009            || selection.revision.is_generation_protocol()
8010            || pool_meta_v3_writer_enabled()
8011            || bootstrap_generation_required
8012        {
8013            POOL_META_GENERATION_VERSION
8014        } else if selection.meta.version == POOL_META_VERSION
8015            || self.version == POOL_META_VERSION
8016            || pool_meta_v2_writer_enabled()
8017        {
8018            POOL_META_VERSION
8019        } else {
8020            POOL_META_V1_VERSION
8021        };
8022        committed.version = target_version;
8023
8024        if target_version != POOL_META_GENERATION_VERSION {
8025            let data = committed.encode_config_data_for_v2_gate(target_version == POOL_META_VERSION)?;
8026            let mut canonical_saved = false;
8027            for (pool_index, (pool, token)) in pools.iter().cloned().zip(&selection.cas_tokens).enumerate() {
8028                let result = save_pool_meta_object_cas(
8029                    pool.clone(),
8030                    POOL_META_NAME,
8031                    data.clone(),
8032                    token,
8033                    fence,
8034                    "legacy_cas",
8035                    transaction_arm,
8036                )
8037                .await;
8038                if result.is_ok() && pool_index == 0 {
8039                    canonical_saved = true;
8040                    #[cfg(test)]
8041                    if let PoolMetaPersistenceFence::Activation(activation_fence) = fence {
8042                        pause_pool_activation_after_durable_save(&pool, activation_fence).await;
8043                    }
8044                }
8045                if let Err(err) = result {
8046                    if canonical_saved && fence.is_activation() {
8047                        warn!(
8048                            event = EVENT_DECOMMISSION_STATE,
8049                            component = LOG_COMPONENT_ECSTORE,
8050                            subsystem = LOG_SUBSYSTEM_POOLS,
8051                            pool_index,
8052                            state = "activation_replica_repair_pending",
8053                            error = %err,
8054                            "Decommission activation replica repair pending"
8055                        );
8056                        continue;
8057                    }
8058                    return Err(err);
8059                }
8060            }
8061            let confirmed = if fence.is_activation() {
8062                load_pool_meta_replicas(pools, true).await?
8063            } else {
8064                let confirmed = load_pool_meta_replicas_observing(pools, true, write_state).await?;
8065                write_state.observe_replicas(confirmed.replica_state);
8066                confirmed
8067            };
8068            let expected = committed.encode_config_data_for_v2_gate(true)?;
8069            if confirmed.canonical.as_ref() != Some(&expected) {
8070                record_pool_meta_stale_write_rejection("legacy_verify");
8071                return Err(Error::PreconditionFailed);
8072            }
8073            return Ok(confirmed.meta);
8074        }
8075
8076        let cluster_id = selection
8077            .revision
8078            .cluster_id
8079            .or(selection.generation_identity.map(|(cluster_id, _)| cluster_id))
8080            .or(write_state.expected_cluster_id)
8081            .ok_or_else(|| Error::other("pool metadata V3 save failed: cluster identity is not initialized"))?;
8082        let epoch = if selection.revision.is_generation_protocol() {
8083            selection.revision.epoch
8084        } else {
8085            selection
8086                .generation_identity
8087                .map(|(_, epoch)| epoch)
8088                .or(write_state.cluster_epoch)
8089                .unwrap_or(POOL_META_INITIAL_EPOCH)
8090        };
8091        let generation = if selection.revision.is_generation_protocol() {
8092            selection
8093                .revision
8094                .generation
8095                .checked_add(1)
8096                .ok_or_else(|| Error::other("pool metadata V3 generation exhausted"))?
8097        } else {
8098            1
8099        };
8100        let revision = PoolMetaRevision {
8101            version: POOL_META_GENERATION_VERSION,
8102            cluster_id: Some(cluster_id),
8103            epoch,
8104            generation,
8105            transaction_id: Some(uuid::Uuid::new_v4()),
8106        };
8107        let previous = if let Some(canonical) = selection.canonical.clone() {
8108            PoolMetaCommittedCandidate {
8109                canonical,
8110                meta: selection.meta.clone(),
8111                revision: selection.revision,
8112            }
8113        } else {
8114            let empty = PoolMeta {
8115                version: POOL_META_V1_VERSION,
8116                ..Default::default()
8117            };
8118            PoolMetaCommittedCandidate {
8119                canonical: empty.encode_config_data_for_v2_gate(true)?,
8120                meta: empty,
8121                revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
8122            }
8123        };
8124        let pending = encode_pool_meta_v3_envelope(&committed, revision, false, Some(&previous))?;
8125        let durable = encode_pool_meta_v3_envelope(&committed, revision, true, None)?;
8126        let concurrent_phases = pools.len() > 1
8127            && selection.revision.is_generation_protocol()
8128            && !selection.replica_state.needs_repair
8129            && !bootstrap_generation_required
8130            && matches!(fence, PoolMetaPersistenceFence::Distributed(Some(_)));
8131        #[cfg(feature = "e2e-test-hooks")]
8132        let phase_barrier = if concurrent_phases && pools.len() == 4 {
8133            PoolMetaPhaseBarrier::bind(&previous, &committed, revision, &durable).await?
8134        } else {
8135            None
8136        };
8137        let mut pending_tokens = Vec::with_capacity(pools.len());
8138        if concurrent_phases {
8139            // Drain the entire prepare phase before validating ETags or returning an error.
8140            for result in save_pool_meta_phase(
8141                pools.as_slice(),
8142                &pending,
8143                &selection.cas_tokens,
8144                fence,
8145                "prepare_cas",
8146                transaction_arm,
8147                #[cfg(feature = "e2e-test-hooks")]
8148                phase_barrier.as_ref(),
8149            )
8150            .await
8151            {
8152                let etag = result?
8153                    .etag
8154                    .filter(|etag| !etag.trim().is_empty())
8155                    .ok_or_else(|| Error::other("pool metadata V3 prepare succeeded without a conditional-write revision"))?;
8156                pending_tokens.push(PoolMetaCasToken::Existing(etag));
8157            }
8158        } else {
8159            for (pool, token) in pools.iter().cloned().zip(&selection.cas_tokens) {
8160                let object_info = save_pool_meta_object_cas(
8161                    pool,
8162                    POOL_META_NAME,
8163                    pending.clone(),
8164                    token,
8165                    fence,
8166                    "prepare_cas",
8167                    transaction_arm,
8168                )
8169                .await?;
8170                let etag = object_info
8171                    .etag
8172                    .filter(|etag| !etag.trim().is_empty())
8173                    .ok_or_else(|| Error::other("pool metadata V3 prepare succeeded without a conditional-write revision"))?;
8174                pending_tokens.push(PoolMetaCasToken::Existing(etag));
8175            }
8176        }
8177
8178        #[cfg(feature = "e2e-test-hooks")]
8179        if let Some(barrier) = &phase_barrier
8180            && barrier.arm.case == PoolMetaPhaseBarrierCase::PreparedAll
8181        {
8182            barrier.ready().await?;
8183        }
8184        let mut commit_error = None;
8185        let mut commit_succeeded = false;
8186        if concurrent_phases {
8187            for result in save_pool_meta_phase(
8188                pools.as_slice(),
8189                &durable,
8190                &pending_tokens,
8191                fence,
8192                "commit_cas",
8193                transaction_arm,
8194                #[cfg(feature = "e2e-test-hooks")]
8195                phase_barrier.as_ref(),
8196            )
8197            .await
8198            {
8199                match result {
8200                    Ok(_) => commit_succeeded = true,
8201                    Err(err) => {
8202                        commit_error.get_or_insert(err);
8203                    }
8204                }
8205            }
8206        } else {
8207            #[cfg(test)]
8208            let mut first_pool = true;
8209            for (pool, token) in pools.iter().cloned().zip(&pending_tokens) {
8210                match save_pool_meta_object_cas(
8211                    pool.clone(),
8212                    POOL_META_NAME,
8213                    durable.clone(),
8214                    token,
8215                    fence,
8216                    "commit_cas",
8217                    transaction_arm,
8218                )
8219                .await
8220                {
8221                    Ok(_) => {
8222                        commit_succeeded = true;
8223                        #[cfg(test)]
8224                        if first_pool && let PoolMetaPersistenceFence::Activation(activation_fence) = fence {
8225                            pause_pool_activation_after_durable_save(&pool, activation_fence).await;
8226                        }
8227                    }
8228                    Err(err) => {
8229                        commit_error.get_or_insert(err);
8230                    }
8231                }
8232                #[cfg(test)]
8233                {
8234                    first_pool = false;
8235                }
8236            }
8237        }
8238        let confirmed = if fence.is_activation() {
8239            load_pool_meta_replicas(pools.clone(), true).await?
8240        } else {
8241            let confirmed = load_pool_meta_replicas_observing(pools.clone(), true, write_state).await?;
8242            write_state.observe_replicas(confirmed.replica_state);
8243            confirmed
8244        };
8245        if confirmed.revision == revision && confirmed.canonical.as_ref() == Some(&durable) {
8246            persist_pool_meta_identity(pools, write_state, true, PoolMetaIdentityWriteScope::All, fence, transaction_arm).await?;
8247            #[cfg(feature = "e2e-test-hooks")]
8248            startup_cas_test_observe(serde_json::json!({
8249                "kind": "confirmed", "object": POOL_META_NAME,
8250                "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&durable)),
8251                "generation": confirmed.revision.generation,
8252                "transaction_id": confirmed.revision.transaction_id,
8253            }));
8254            #[cfg(feature = "e2e-test-hooks")]
8255            if let Some(barrier) = &phase_barrier
8256                && barrier.arm.case == PoolMetaPhaseBarrierCase::BeforePublish
8257            {
8258                barrier.ready().await?;
8259            }
8260            return Ok(confirmed.meta);
8261        }
8262        if !commit_succeeded {
8263            return Err(commit_error.unwrap_or(Error::PreconditionFailed));
8264        }
8265        Err(commit_error.unwrap_or_else(|| {
8266            Error::other("pool metadata recovery required: committed V3 transaction was not selected after write")
8267        }))
8268    }
8269
8270    #[cfg(test)]
8271    async fn save_no_lock_observing<S>(&self, pools: Vec<Arc<S>>, write_state: &mut PoolMetaWriteState) -> Result<()>
8272    where
8273        S: EcstoreObjectIO,
8274    {
8275        let indices = (0..self.pools.len()).collect::<Vec<_>>();
8276        let outcome = self.save_no_lock_armed(pools, write_state, None, &indices).await?;
8277        outcome.disarm();
8278        Ok(())
8279    }
8280
8281    pub fn decommission_cancel(&mut self, idx: usize) -> bool {
8282        self.decommission_cancel_at(idx, OffsetDateTime::now_utc(), None)
8283    }
8284
8285    #[cfg(test)]
8286    pub(crate) fn decommission_cancel_at_for_test(
8287        &mut self,
8288        idx: usize,
8289        now: OffsetDateTime,
8290        rebalance_meta: Option<&RebalanceMeta>,
8291    ) -> bool {
8292        self.decommission_cancel_at(idx, now, rebalance_meta)
8293    }
8294
8295    fn decommission_cancel_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool {
8296        let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else {
8297            return false;
8298        };
8299        if d.canceled {
8300            return false;
8301        }
8302
8303        let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8304        let mut pd = d.clone();
8305        pd.canceled = true;
8306        pd.failed = false;
8307        pd.complete = false;
8308        pd.start_time = None;
8309        pd.terminal_reload_attempt_at = None;
8310        pd.terminal_reload_failures.clear();
8311        pd.capacity_blocked_reason = None;
8312        release_decommission_capacity_reservation(&mut pd, DECOMMISSION_CAPACITY_RELEASE_CANCELED, last_update);
8313
8314        let Some(stats) = self.pools.get_mut(idx) else {
8315            return false;
8316        };
8317        stats.last_update = last_update;
8318        stats.decommission = Some(pd);
8319        true
8320    }
8321    pub fn decommission_failed(&mut self, idx: usize) -> bool {
8322        self.decommission_failed_at(idx, OffsetDateTime::now_utc(), None)
8323    }
8324
8325    #[cfg(test)]
8326    pub(crate) fn decommission_failed_at_for_test(
8327        &mut self,
8328        idx: usize,
8329        now: OffsetDateTime,
8330        rebalance_meta: Option<&RebalanceMeta>,
8331    ) -> bool {
8332        self.decommission_failed_at(idx, now, rebalance_meta)
8333    }
8334
8335    fn decommission_failed_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool {
8336        let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else {
8337            return false;
8338        };
8339        if !is_decommission_active(d.complete, d.failed, d.canceled) {
8340            return false;
8341        }
8342
8343        let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8344        let mut pd = d.clone();
8345        pd.canceled = false;
8346        pd.failed = true;
8347        pd.complete = false;
8348        pd.start_time = None;
8349        pd.terminal_reload_attempt_at = None;
8350        pd.terminal_reload_failures.clear();
8351        pd.capacity_blocked_reason = None;
8352        release_decommission_capacity_reservation(&mut pd, DECOMMISSION_CAPACITY_RELEASE_FAILED, last_update);
8353
8354        let Some(stats) = self.pools.get_mut(idx) else {
8355            return false;
8356        };
8357        stats.last_update = last_update;
8358        stats.decommission = Some(pd);
8359        true
8360    }
8361
8362    pub fn clear_decommission(&mut self, idx: usize) -> Result<bool> {
8363        self.clear_decommission_at(idx, OffsetDateTime::now_utc(), None)
8364    }
8365
8366    #[cfg(test)]
8367    pub(crate) fn clear_decommission_at_for_test(
8368        &mut self,
8369        idx: usize,
8370        now: OffsetDateTime,
8371        rebalance_meta: Option<&RebalanceMeta>,
8372    ) -> Result<bool> {
8373        self.clear_decommission_at(idx, now, rebalance_meta)
8374    }
8375
8376    fn clear_decommission_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> Result<bool> {
8377        let pool_count = self.pools.len();
8378        ensure_valid_decommission_pool_index(pool_count, idx)?;
8379
8380        let Some(pool) = self.pools.get(idx) else {
8381            return Err(invalid_decommission_pool_index_error(pool_count, idx));
8382        };
8383
8384        let (decommission_present, complete, failed, canceled, unresolved_entries) = pool
8385            .decommission
8386            .as_ref()
8387            .map(|info| {
8388                (
8389                    info.has_decommission_state(),
8390                    info.complete,
8391                    info.failed,
8392                    info.canceled,
8393                    info.unresolved_entries.len(),
8394                )
8395            })
8396            .unwrap_or((false, false, false, false, 0));
8397
8398        ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled, unresolved_entries)?;
8399
8400        let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8401        let Some(pool) = self.pools.get_mut(idx) else {
8402            return Err(invalid_decommission_pool_index_error(pool_count, idx));
8403        };
8404        pool.last_update = last_update;
8405        // Preserve a state-empty tombstone so scanner catch-up can recover the
8406        // durable movement generation after a clear followed by a restart.
8407        pool.decommission = Some(PoolDecommissionInfo::default());
8408        Ok(true)
8409    }
8410
8411    pub fn decommission_complete(&mut self, idx: usize) -> bool {
8412        self.decommission_complete_at(idx, OffsetDateTime::now_utc(), None)
8413    }
8414
8415    #[cfg(test)]
8416    pub(crate) fn decommission_complete_at_for_test(
8417        &mut self,
8418        idx: usize,
8419        now: OffsetDateTime,
8420        rebalance_meta: Option<&RebalanceMeta>,
8421    ) -> bool {
8422        self.decommission_complete_at(idx, now, rebalance_meta)
8423    }
8424
8425    fn decommission_complete_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool {
8426        let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else {
8427            return false;
8428        };
8429        if !is_decommission_active(d.complete, d.failed, d.canceled) {
8430            return false;
8431        }
8432
8433        let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8434        let mut pd = d.clone();
8435        pd.canceled = false;
8436        pd.failed = false;
8437        pd.complete = true;
8438        pd.terminal_reload_attempt_at = None;
8439        pd.terminal_reload_failures.clear();
8440        pd.capacity_blocked_reason = None;
8441        release_decommission_capacity_reservation(&mut pd, DECOMMISSION_CAPACITY_RELEASE_COMPLETED, last_update);
8442
8443        let Some(stats) = self.pools.get_mut(idx) else {
8444            return false;
8445        };
8446        stats.last_update = last_update;
8447        stats.decommission = Some(pd);
8448        true
8449    }
8450    fn set_decommission_state_at(
8451        &mut self,
8452        idx: usize,
8453        pi: PoolSpaceInfo,
8454        queued: bool,
8455        now: OffsetDateTime,
8456        rebalance_meta: Option<&RebalanceMeta>,
8457    ) -> Result<()> {
8458        let pool_count = self.pools.len();
8459        ensure_valid_decommission_pool_index(pool_count, idx)?;
8460
8461        let Some(pool) = self.pools.get(idx) else {
8462            return Err(invalid_decommission_pool_index_error(pool_count, idx));
8463        };
8464
8465        ensure_decommission_start_allowed(decommission_start_pool_state(Some(pool)))?;
8466
8467        let generation = self.next_scanner_data_movement_update(now, rebalance_meta);
8468        let Some(pool) = self.pools.get_mut(idx) else {
8469            return Err(invalid_decommission_pool_index_error(pool_count, idx));
8470        };
8471        let previous = pool.decommission.as_ref();
8472        pool.last_update = generation;
8473        pool.decommission = Some(build_decommission_start_state(pi, queued, generation, previous));
8474
8475        Ok(())
8476    }
8477
8478    fn set_decommission_state(&mut self, idx: usize, pi: PoolSpaceInfo, queued: bool) -> Result<()> {
8479        self.set_decommission_state_at(idx, pi, queued, OffsetDateTime::now_utc(), None)
8480    }
8481
8482    #[cfg(test)]
8483    pub(crate) fn decommission_at_for_test(
8484        &mut self,
8485        idx: usize,
8486        pi: PoolSpaceInfo,
8487        now: OffsetDateTime,
8488        rebalance_meta: Option<&RebalanceMeta>,
8489    ) -> Result<()> {
8490        self.set_decommission_state_at(idx, pi, false, now, rebalance_meta)
8491    }
8492
8493    pub fn decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> {
8494        self.set_decommission_state(idx, pi, false)
8495    }
8496
8497    pub fn queue_decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> {
8498        self.set_decommission_state(idx, pi, true)
8499    }
8500
8501    pub fn record_decommission_terminal_reload_failure(&mut self, idx: usize, stage: &str, message: String) -> Result<bool> {
8502        self.record_decommission_terminal_reload_failure_at(idx, stage, message, OffsetDateTime::now_utc(), None)
8503    }
8504
8505    #[cfg(test)]
8506    pub(crate) fn record_decommission_terminal_reload_failure_at_for_test(
8507        &mut self,
8508        idx: usize,
8509        stage: &str,
8510        message: String,
8511        now: OffsetDateTime,
8512        rebalance_meta: Option<&RebalanceMeta>,
8513    ) -> Result<bool> {
8514        self.record_decommission_terminal_reload_failure_at(idx, stage, message, now, rebalance_meta)
8515    }
8516
8517    fn record_decommission_terminal_reload_failure_at(
8518        &mut self,
8519        idx: usize,
8520        stage: &str,
8521        message: String,
8522        now: OffsetDateTime,
8523        rebalance_meta: Option<&RebalanceMeta>,
8524    ) -> Result<bool> {
8525        let pool_count = self.pools.len();
8526        ensure_valid_decommission_pool_index(pool_count, idx)?;
8527
8528        let Some(pool) = self.pools.get(idx) else {
8529            return Err(invalid_decommission_pool_index_error(pool_count, idx));
8530        };
8531        let Some(info) = pool.decommission.as_ref() else {
8532            return Err(decommission_metadata_not_initialized_error("record decommission terminal reload failure"));
8533        };
8534
8535        let failure = format!("{stage}: {message}");
8536        if info.terminal_reload_failures.last().is_some_and(|last| last == &failure) {
8537            return Ok(false);
8538        }
8539
8540        let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8541        let Some(pool) = self.pools.get_mut(idx) else {
8542            return Err(invalid_decommission_pool_index_error(pool_count, idx));
8543        };
8544        let Some(info) = pool.decommission.as_mut() else {
8545            return Err(decommission_metadata_not_initialized_error("record decommission terminal reload failure"));
8546        };
8547        pool.last_update = last_update;
8548        info.terminal_reload_attempt_at = Some(last_update);
8549        info.terminal_reload_failures.push(failure);
8550        Ok(true)
8551    }
8552
8553    fn mark_decommission_capacity_blocked(&mut self, idx: usize, reason: String, now: OffsetDateTime) -> Result<bool> {
8554        let pool_count = self.pools.len();
8555        let pool = self
8556            .pools
8557            .get_mut(idx)
8558            .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, idx))?;
8559        let info = pool
8560            .decommission
8561            .as_mut()
8562            .ok_or_else(|| decommission_metadata_not_initialized_error("pause decommission for target capacity"))?;
8563        if !is_decommission_active(info.complete, info.failed, info.canceled) {
8564            return Ok(false);
8565        }
8566        if let Some(reservation) = info.capacity_reservation.as_mut().filter(|reservation| reservation.active()) {
8567            renew_decommission_capacity_reservation(reservation, now, true);
8568        }
8569        let changed = info.capacity_blocked_reason.as_deref() != Some(reason.as_str());
8570        info.capacity_blocked_reason = Some(reason);
8571        pool.last_update = now;
8572        Ok(changed)
8573    }
8574
8575    pub fn promote_queued_decommission(&mut self, idx: usize) -> bool {
8576        self.promote_queued_decommission_at(idx, OffsetDateTime::now_utc(), None)
8577    }
8578
8579    #[cfg(test)]
8580    pub(crate) fn promote_queued_decommission_at_for_test(
8581        &mut self,
8582        idx: usize,
8583        now: OffsetDateTime,
8584        rebalance_meta: Option<&RebalanceMeta>,
8585    ) -> bool {
8586        self.promote_queued_decommission_at(idx, now, rebalance_meta)
8587    }
8588
8589    fn promote_queued_decommission_at(
8590        &mut self,
8591        idx: usize,
8592        now: OffsetDateTime,
8593        rebalance_meta: Option<&RebalanceMeta>,
8594    ) -> bool {
8595        let Some(info) = self.pools.get(idx).and_then(|pool| pool.decommission.as_ref()) else {
8596            return false;
8597        };
8598        if !info.queued || !is_decommission_active(info.complete, info.failed, info.canceled) {
8599            return false;
8600        }
8601
8602        let generation = self.next_scanner_data_movement_update(now, rebalance_meta);
8603        let Some(pool) = self.pools.get_mut(idx) else {
8604            return false;
8605        };
8606        let Some(info) = pool.decommission.as_mut() else {
8607            return false;
8608        };
8609        pool.last_update = generation;
8610        info.queued = false;
8611        info.start_time = Some(generation);
8612        for entry in &mut info.unresolved_entries {
8613            entry.source_generation = generation;
8614        }
8615        true
8616    }
8617    pub fn queue_buckets(&mut self, idx: usize, bks: Vec<DecomBucketInfo>) {
8618        if let Some(pool) = self.pools.get_mut(idx)
8619            && let Some(dec) = pool.decommission.as_mut()
8620        {
8621            for bk in bks.iter() {
8622                dec.bucket_push(bk);
8623            }
8624        }
8625    }
8626    pub fn pending_buckets(&self, idx: usize) -> Vec<DecomBucketInfo> {
8627        let mut list = Vec::new();
8628
8629        if let Some(pool) = self.pools.get(idx)
8630            && let Some(ref info) = pool.decommission
8631        {
8632            for bk in info.queued_buckets.iter() {
8633                let (name, prefix) = path2_bucket_object(bk);
8634                list.push(DecomBucketInfo { name, prefix });
8635            }
8636        }
8637
8638        list
8639    }
8640
8641    pub fn is_bucket_decommissioned(&self, idx: usize, bucket: String) -> bool {
8642        self.pools
8643            .get(idx)
8644            .and_then(|pool| pool.decommission.as_ref())
8645            .is_some_and(|info| info.is_bucket_decommissioned(&bucket))
8646    }
8647
8648    pub fn bucket_done(&mut self, idx: usize, bucket: String) -> bool {
8649        if let Some(pool) = self.pools.get_mut(idx) {
8650            if let Some(info) = pool.decommission.as_mut() {
8651                info.bucket_pop(&bucket)
8652            } else {
8653                false
8654            }
8655        } else {
8656            false
8657        }
8658    }
8659
8660    pub fn count_item(&mut self, idx: usize, size: usize, failed: bool) {
8661        if let Some(pool) = self.pools.get_mut(idx)
8662            && let Some(info) = pool.decommission.as_mut()
8663        {
8664            if failed {
8665                info.items_decommission_failed += 1;
8666                info.bytes_failed += size;
8667            } else {
8668                info.items_decommissioned += 1;
8669                info.bytes_done += size;
8670            }
8671        }
8672    }
8673
8674    pub fn track_current_bucket_object(&mut self, idx: usize, bucket: String, object: String) {
8675        self.track_current_bucket_object_stage(idx, bucket, object, String::new());
8676    }
8677
8678    pub fn track_current_bucket_object_stage(&mut self, idx: usize, bucket: String, object: String, stage: String) {
8679        if self.pools.get(idx).is_none_or(|v| v.decommission.is_none()) {
8680            return;
8681        }
8682
8683        if let Some(pool) = self.pools.get_mut(idx)
8684            && let Some(info) = pool.decommission.as_mut()
8685        {
8686            info.object = object;
8687            info.bucket = bucket;
8688            info.stage = stage;
8689        }
8690    }
8691
8692    pub fn update_after(&mut self, idx: usize, duration: Duration) -> Result<bool> {
8693        Ok(self
8694            .decommission_progress_checkpoint(idx, duration, OffsetDateTime::now_utc(), None)?
8695            .is_some())
8696    }
8697
8698    pub fn validate(&self, pools: Vec<Arc<Sets>>) -> Result<bool> {
8699        struct PoolInfo {
8700            position: usize,
8701            completed: bool,
8702            #[allow(dead_code, reason = "written but never read back (backlog#1823)")]
8703            decom_started: bool,
8704        }
8705
8706        let mut remembered_pools = HashMap::new();
8707        for (idx, pool) in self.pools.iter().enumerate() {
8708            let mut complete = false;
8709            let mut decom_started = false;
8710            if let Some(decommission) = &pool.decommission {
8711                if decommission.complete {
8712                    complete = true;
8713                }
8714                decom_started = true;
8715            }
8716            remembered_pools.insert(
8717                pool.cmd_line.clone(),
8718                PoolInfo {
8719                    position: idx,
8720                    completed: complete,
8721                    decom_started,
8722                },
8723            );
8724        }
8725
8726        let mut specified_pools = HashMap::new();
8727        for (idx, pool) in pools.iter().enumerate() {
8728            specified_pools.insert(pool.endpoints.cmd_line.clone(), idx);
8729        }
8730
8731        let mut update = false;
8732
8733        // Determine whether the selected pool should be removed from the retired list.
8734        for k in specified_pools.keys() {
8735            if let Some(pi) = remembered_pools.get(k) {
8736                ensure_pool_not_left_in_cmdline_after_decommission(pi.position, k, pi.completed)?;
8737            } else {
8738                // If the previous pool no longer exists, allow updates because a new pool may have been added.
8739                update = true;
8740            }
8741        }
8742
8743        if specified_pools.len() == remembered_pools.len() {
8744            for (k, pi) in remembered_pools.iter() {
8745                if let Some(pos) = specified_pools.get(k)
8746                    && *pos != pi.position
8747                {
8748                    update = true; // Pool order changed, allow the update.
8749                }
8750            }
8751        }
8752
8753        if !update {
8754            update = specified_pools.len() != remembered_pools.len();
8755        }
8756
8757        Ok(update)
8758    }
8759
8760    pub fn return_resumable_pools(&self) -> Vec<PoolStatus> {
8761        resumable_decommission_queue_indices(self)
8762            .into_iter()
8763            .map(|idx| self.pools[idx].clone())
8764            .collect()
8765    }
8766}
8767
8768pub fn path2_bucket_object(name: &str) -> (String, String) {
8769    path_to_bucket_object(name)
8770}
8771
8772pub fn path2_bucket_object_with_base_path(base_path: &str, path: &str) -> (String, String) {
8773    path_to_bucket_object_with_base_path(base_path, path)
8774}
8775
8776#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
8777#[serde(deny_unknown_fields)]
8778pub struct DecommissionUnresolvedEntry {
8779    pub bucket: String,
8780    pub object: String,
8781    #[serde(rename = "poolIndex")]
8782    pub pool_index: usize,
8783    #[serde(rename = "setIndex")]
8784    pub set_index: usize,
8785    #[serde(rename = "sourceGeneration", with = "time::serde::rfc3339")]
8786    pub source_generation: OffsetDateTime,
8787    #[serde(rename = "candidateCount")]
8788    pub candidate_count: usize,
8789    #[serde(rename = "diskErrorCount")]
8790    pub disk_error_count: usize,
8791    #[serde(rename = "observedAt", with = "time::serde::rfc3339")]
8792    pub observed_at: OffsetDateTime,
8793    pub reason: String,
8794}
8795
8796#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
8797#[serde(rename_all = "camelCase", deny_unknown_fields)]
8798pub struct DecommissionErasureLayout {
8799    pub data: usize,
8800    pub parity: usize,
8801}
8802
8803impl DecommissionErasureLayout {
8804    fn width(self) -> usize {
8805        self.data.saturating_add(self.parity)
8806    }
8807
8808    fn is_valid(self) -> bool {
8809        self.data > 0 && self.data.checked_add(self.parity).is_some()
8810    }
8811}
8812
8813#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
8814#[serde(rename_all = "camelCase", deny_unknown_fields)]
8815pub struct DecommissionCapacityTarget {
8816    pub pool_index: usize,
8817    pub layout: DecommissionErasureLayout,
8818    pub physical_total_at_reservation: usize,
8819    pub physical_free_at_reservation: usize,
8820    pub reserved_physical_bytes: usize,
8821    #[serde(default)]
8822    pub consumed_physical_bytes: usize,
8823    #[serde(default)]
8824    pub observed_physical_bytes: usize,
8825    #[serde(default)]
8826    pub inflight_physical_bytes: usize,
8827    #[serde(default)]
8828    pub pending_physical_bytes: usize,
8829    #[serde(default)]
8830    pub pending_mutation_id: Option<uuid::Uuid>,
8831    #[serde(default, skip_serializing_if = "Vec::is_empty")]
8832    pub temporary_mutations: Vec<DecommissionCapacityTemporaryMutation>,
8833}
8834
8835#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
8836#[serde(rename_all = "camelCase", deny_unknown_fields)]
8837pub struct DecommissionCapacityTemporaryMutation {
8838    pub mutation_id: uuid::Uuid,
8839    pub physical_bytes: usize,
8840}
8841
8842#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
8843#[serde(rename_all = "camelCase", deny_unknown_fields)]
8844pub struct DecommissionCapacityReservation {
8845    pub model_version: u16,
8846    pub operation_id: uuid::Uuid,
8847    pub generation: u64,
8848    pub owner_nonce: uuid::Uuid,
8849    pub source_pool_index: usize,
8850    pub source_layout: DecommissionErasureLayout,
8851    pub source_physical_total_bytes: usize,
8852    pub source_physical_bytes: usize,
8853    pub source_data_equivalent_bytes: usize,
8854    pub predicted_physical_bytes: usize,
8855    pub temporary_copies: usize,
8856    pub temporary_physical_bytes: usize,
8857    pub peak_physical_bytes: usize,
8858    pub committed_data_bytes: usize,
8859    #[serde(default)]
8860    pub consumed_target_physical_bytes: usize,
8861    #[serde(default)]
8862    pub observed_target_physical_bytes: usize,
8863    #[serde(default)]
8864    pub inflight_target_physical_bytes: usize,
8865    #[serde(default)]
8866    pub pending_target_physical_bytes: usize,
8867    pub prediction_error_bytes: i64,
8868    pub targets: Vec<DecommissionCapacityTarget>,
8869    #[serde(with = "time::serde::rfc3339")]
8870    pub created_at: OffsetDateTime,
8871    #[serde(with = "time::serde::rfc3339")]
8872    pub renewed_at: OffsetDateTime,
8873    #[serde(with = "time::serde::rfc3339")]
8874    pub expires_at: OffsetDateTime,
8875    #[serde(with = "time::serde::rfc3339::option", default)]
8876    pub recovered_at: Option<OffsetDateTime>,
8877    #[serde(with = "time::serde::rfc3339::option", default)]
8878    pub released_at: Option<OffsetDateTime>,
8879    #[serde(default, skip_serializing_if = "Option::is_none")]
8880    pub release_reason: Option<String>,
8881}
8882
8883#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8884enum DecommissionCapacityMutationMode {
8885    Durable,
8886    Temporary,
8887    NonGrowingReplacement,
8888    TemporaryRelease,
8889}
8890
8891#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8892struct DecommissionCapacityReleaseProof {
8893    confirmed_absent: bool,
8894    clear_pending: bool,
8895}
8896
8897impl DecommissionCapacityReleaseProof {
8898    const fn confirmed_absence(clear_pending: bool) -> Self {
8899        Self {
8900            confirmed_absent: true,
8901            clear_pending,
8902        }
8903    }
8904}
8905
8906#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8907pub(crate) struct DecommissionCapacityOwner {
8908    pub(crate) source_pool_index: usize,
8909    pub(crate) operation_id: uuid::Uuid,
8910    pub(crate) generation: u64,
8911    pub(crate) owner_nonce: uuid::Uuid,
8912    pub(crate) mutation_id: Option<uuid::Uuid>,
8913}
8914
8915#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
8916struct DecommissionCapacityTargetPermitKey {
8917    store_id: uuid::Uuid,
8918    target_pool_index: usize,
8919    source_pool_index: usize,
8920    operation_id: uuid::Uuid,
8921    generation: u64,
8922    owner_nonce: uuid::Uuid,
8923    mutation_id: uuid::Uuid,
8924}
8925
8926impl DecommissionCapacityTargetPermitKey {
8927    fn new(store_id: uuid::Uuid, target_pool_index: usize, owner: DecommissionCapacityOwner) -> Option<Self> {
8928        Some(Self {
8929            store_id,
8930            target_pool_index,
8931            source_pool_index: owner.source_pool_index,
8932            operation_id: owner.operation_id,
8933            generation: owner.generation,
8934            owner_nonce: owner.owner_nonce,
8935            mutation_id: owner.mutation_id?,
8936        })
8937    }
8938
8939    fn owns_same_mutation(&self, store_id: uuid::Uuid, owner: DecommissionCapacityOwner) -> bool {
8940        owner.mutation_id.is_some_and(|mutation_id| {
8941            self.store_id == store_id
8942                && self.source_pool_index == owner.source_pool_index
8943                && self.operation_id == owner.operation_id
8944                && self.generation == owner.generation
8945                && self.owner_nonce == owner.owner_nonce
8946                && self.mutation_id == mutation_id
8947        })
8948    }
8949}
8950
8951struct DecommissionCapacityTargetPermit {
8952    key: DecommissionCapacityTargetPermitKey,
8953}
8954
8955// Busy recovery crosses option rebuilding and a spawned migration task. Keep
8956// the distributed guard in one exact-mutation side table so target selection
8957// can pin that target and the formal capacity mutation can consume the guard.
8958static DECOMMISSION_CAPACITY_TARGET_PERMITS: std::sync::OnceLock<
8959    std::sync::Mutex<HashMap<DecommissionCapacityTargetPermitKey, rustfs_lock::NamespaceLockGuard>>,
8960> = std::sync::OnceLock::new();
8961
8962fn decommission_capacity_target_permits()
8963-> &'static std::sync::Mutex<HashMap<DecommissionCapacityTargetPermitKey, rustfs_lock::NamespaceLockGuard>> {
8964    DECOMMISSION_CAPACITY_TARGET_PERMITS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
8965}
8966
8967fn install_decommission_capacity_target_permit(
8968    store_id: uuid::Uuid,
8969    target_pool_index: usize,
8970    owner: DecommissionCapacityOwner,
8971    guard: rustfs_lock::NamespaceLockGuard,
8972) -> Result<DecommissionCapacityTargetPermit> {
8973    let key = DecommissionCapacityTargetPermitKey::new(store_id, target_pool_index, owner)
8974        .ok_or_else(|| Error::other("decommission target permit is missing its mutation identity"))?;
8975    let mut permits = decommission_capacity_target_permits()
8976        .lock()
8977        .unwrap_or_else(std::sync::PoisonError::into_inner);
8978    if permits.keys().any(|candidate| candidate.owns_same_mutation(store_id, owner)) {
8979        return Err(Error::other("decommission target permit already exists for this mutation"));
8980    }
8981    permits.insert(key, guard);
8982    Ok(DecommissionCapacityTargetPermit { key })
8983}
8984
8985fn take_decommission_capacity_target_permit(
8986    store_id: uuid::Uuid,
8987    target_pool_index: usize,
8988    owner: DecommissionCapacityOwner,
8989) -> Option<rustfs_lock::NamespaceLockGuard> {
8990    let key = DecommissionCapacityTargetPermitKey::new(store_id, target_pool_index, owner)?;
8991    decommission_capacity_target_permits()
8992        .lock()
8993        .unwrap_or_else(std::sync::PoisonError::into_inner)
8994        .remove(&key)
8995}
8996
8997fn decommission_capacity_target_permit_index(store_id: uuid::Uuid, owner: DecommissionCapacityOwner) -> Option<usize> {
8998    decommission_capacity_target_permits()
8999        .lock()
9000        .unwrap_or_else(std::sync::PoisonError::into_inner)
9001        .keys()
9002        .find(|key| key.owns_same_mutation(store_id, owner))
9003        .map(|key| key.target_pool_index)
9004}
9005
9006fn discard_decommission_capacity_target_permit_except(
9007    store_id: uuid::Uuid,
9008    owner: DecommissionCapacityOwner,
9009    retained_target_pool_index: Option<usize>,
9010) {
9011    let guard = {
9012        let mut permits = decommission_capacity_target_permits()
9013            .lock()
9014            .unwrap_or_else(std::sync::PoisonError::into_inner);
9015        let key = permits
9016            .keys()
9017            .find(|key| key.owns_same_mutation(store_id, owner) && retained_target_pool_index != Some(key.target_pool_index))
9018            .copied();
9019        key.and_then(|key| permits.remove(&key))
9020    };
9021    drop(guard);
9022}
9023
9024impl Drop for DecommissionCapacityTargetPermit {
9025    fn drop(&mut self) {
9026        let guard = decommission_capacity_target_permits()
9027            .lock()
9028            .unwrap_or_else(std::sync::PoisonError::into_inner)
9029            .remove(&self.key);
9030        drop(guard);
9031    }
9032}
9033
9034#[derive(Debug, Clone, PartialEq, Eq)]
9035struct DecommissionCapacityTerminalFencePlan {
9036    model_version: u16,
9037    operation_id: uuid::Uuid,
9038    generation: u64,
9039    owner_nonce: uuid::Uuid,
9040    target_pool_indices: Vec<usize>,
9041}
9042
9043fn decommission_capacity_terminal_fence_plan(
9044    meta: &PoolMeta,
9045    source_pool_index: usize,
9046) -> Result<Option<DecommissionCapacityTerminalFencePlan>> {
9047    let Some(reservation) = meta
9048        .pools
9049        .get(source_pool_index)
9050        .and_then(|pool| pool.decommission.as_ref())
9051        .and_then(|info| info.capacity_reservation.as_ref())
9052        .filter(|reservation| reservation.active())
9053    else {
9054        return Ok(None);
9055    };
9056    if reservation.source_pool_index != source_pool_index {
9057        return Err(Error::DecommissionCapacity(
9058            "decommission terminal transition found a mismatched capacity source".to_string(),
9059        ));
9060    }
9061    if !matches!(
9062        reservation.model_version,
9063        DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
9064    ) {
9065        return Err(Error::DecommissionCapacity(format!(
9066            "decommission terminal transition found unsupported capacity lock model {}",
9067            reservation.model_version
9068        )));
9069    }
9070    let mut target_pool_indices = reservation.targets.iter().map(|target| target.pool_index).collect::<Vec<_>>();
9071    target_pool_indices.sort_unstable();
9072    Ok(Some(DecommissionCapacityTerminalFencePlan {
9073        model_version: reservation.model_version,
9074        operation_id: reservation.operation_id,
9075        generation: reservation.generation,
9076        owner_nonce: reservation.owner_nonce,
9077        target_pool_indices,
9078    }))
9079}
9080
9081#[derive(Debug, Clone, PartialEq, Eq)]
9082pub(crate) struct DecommissionDurableIlmCheckpointTarget {
9083    pub(crate) source_pool_index: usize,
9084    pub(crate) target_pool_index: usize,
9085    pub(crate) capacity_owner: DecommissionCapacityOwner,
9086    pub(crate) already_committed: bool,
9087    pub(crate) target_etag: Option<String>,
9088}
9089
9090impl DecommissionCapacityOwner {
9091    pub(crate) fn apply_to(self, opts: &mut ObjectOptions) {
9092        opts.src_pool_idx = self.source_pool_index;
9093        let admission = opts
9094            .decommission_capacity
9095            .get_or_insert_with(|| Box::new(DecommissionCapacityOptions::default()));
9096        admission.operation_id = Some(self.operation_id);
9097        admission.generation = Some(self.generation);
9098        admission.owner_nonce = Some(self.owner_nonce);
9099        admission.mutation_id = self.mutation_id;
9100    }
9101
9102    pub(crate) fn from_options(opts: &ObjectOptions) -> Option<Self> {
9103        let capacity = opts.decommission_capacity.as_deref()?;
9104        Some(Self {
9105            source_pool_index: opts.src_pool_idx,
9106            operation_id: capacity.operation_id?,
9107            generation: capacity.generation?,
9108            owner_nonce: capacity.owner_nonce?,
9109            mutation_id: capacity.mutation_id,
9110        })
9111    }
9112
9113    pub(crate) fn with_mutation_id(self, mutation_id: uuid::Uuid) -> Self {
9114        Self {
9115            mutation_id: Some(mutation_id),
9116            ..self
9117        }
9118    }
9119}
9120
9121pub(crate) fn decommission_capacity_mutation_id(
9122    owner: DecommissionCapacityOwner,
9123    bucket: &str,
9124    object: &str,
9125    version_id: Option<&str>,
9126    delete_marker: bool,
9127    mod_time: Option<OffsetDateTime>,
9128) -> uuid::Uuid {
9129    let mut hasher = Sha256::new();
9130    hasher.update(owner.operation_id.as_bytes());
9131    hasher.update(owner.generation.to_le_bytes());
9132    hasher.update(owner.source_pool_index.to_le_bytes());
9133    for value in [bucket, object] {
9134        hasher.update((value.len() as u64).to_le_bytes());
9135        hasher.update(value.as_bytes());
9136    }
9137    hasher.update([u8::from(delete_marker)]);
9138    if let Some(version_id) = version_id {
9139        hasher.update([1]);
9140        hasher.update((version_id.len() as u64).to_le_bytes());
9141        hasher.update(version_id.as_bytes());
9142    } else {
9143        hasher.update([0]);
9144    }
9145    if let Some(mod_time) = mod_time {
9146        hasher.update([1]);
9147        hasher.update(mod_time.unix_timestamp_nanos().to_le_bytes());
9148    } else {
9149        hasher.update([0]);
9150    }
9151    let digest = hasher.finalize();
9152    let mut bytes = [0; 16];
9153    bytes.copy_from_slice(&digest[..16]);
9154    uuid::Uuid::from_bytes(bytes)
9155}
9156
9157#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9158struct ExactDeleteCapacityReconciliation {
9159    source_pool_index: usize,
9160    target_pool_index: usize,
9161    mutation_id: uuid::Uuid,
9162    expected_data_bytes: usize,
9163    expected_target_physical_bytes: usize,
9164}
9165
9166fn plan_exact_delete_capacity_reconciliations(
9167    meta: &PoolMeta,
9168    object: &str,
9169    exact: &ObjectInfo,
9170) -> Result<Vec<ExactDeleteCapacityReconciliation>> {
9171    let version_id = exact.version_id.map(|version_id| version_id.to_string());
9172    let mut matches = Vec::new();
9173
9174    for (source_pool_index, pool) in meta.pools.iter().enumerate() {
9175        let Some(reservation) = pool
9176            .decommission
9177            .as_ref()
9178            .and_then(|info| info.capacity_reservation.as_ref())
9179            .filter(|reservation| reservation.active())
9180        else {
9181            continue;
9182        };
9183        let owner = DecommissionCapacityOwner {
9184            source_pool_index,
9185            operation_id: reservation.operation_id,
9186            generation: reservation.generation,
9187            owner_nonce: reservation.owner_nonce,
9188            mutation_id: None,
9189        };
9190        let logical_mutation_id = decommission_capacity_mutation_id(
9191            owner,
9192            &exact.bucket,
9193            &exact.name,
9194            version_id.as_deref(),
9195            exact.delete_marker,
9196            exact.mod_time,
9197        );
9198        // Existing data-movement producers persist directory-key intents using
9199        // either the logical name or its internal `__XLDIR__` representation.
9200        // Accept both while retaining the exact persisted identity for CAS.
9201        let internal_mutation_id = if object == exact.name {
9202            logical_mutation_id
9203        } else {
9204            decommission_capacity_mutation_id(
9205                owner,
9206                &exact.bucket,
9207                object,
9208                version_id.as_deref(),
9209                exact.delete_marker,
9210                exact.mod_time,
9211            )
9212        };
9213        let mut source_match = None;
9214
9215        for target in &reservation.targets {
9216            if target.pending_physical_bytes == 0 {
9217                continue;
9218            }
9219            let Some(pending_mutation_id) = target.pending_mutation_id else {
9220                return Err(decommission_capacity_blocked_error(format!(
9221                    "source pool {source_pool_index} target pool {} has pending capacity without an object identity",
9222                    target.pool_index
9223                )));
9224            };
9225            if pending_mutation_id != logical_mutation_id && pending_mutation_id != internal_mutation_id {
9226                continue;
9227            }
9228            if source_match.is_some() {
9229                return Err(decommission_capacity_blocked_error(format!(
9230                    "source pool {source_pool_index} has the same exact-delete capacity intent on multiple targets"
9231                )));
9232            }
9233            source_match = Some((target.pool_index, target.layout, target.pending_physical_bytes, pending_mutation_id));
9234        }
9235
9236        let Some((target_pool_index, target_layout, pending_physical_bytes, mutation_id)) = source_match else {
9237            continue;
9238        };
9239        if exact.version_id.is_none() && exact.mod_time.is_none() {
9240            return Err(decommission_capacity_blocked_error(
9241                "unversioned exact delete cannot identify pending capacity without a modification time",
9242            ));
9243        }
9244        let expected_data_bytes = if exact.delete_marker {
9245            0
9246        } else {
9247            usize::try_from(exact.size).map_err(|_| {
9248                decommission_capacity_blocked_error("exact delete cannot reconcile a negative or overflowing object size")
9249            })?
9250        };
9251        let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
9252        if pending_physical_bytes != expected_target_physical_bytes {
9253            return Err(decommission_capacity_blocked_error(format!(
9254                "source pool {source_pool_index} target pool {target_pool_index} pending capacity does not match the exact object size"
9255            )));
9256        }
9257        let remaining_target_physical_bytes = reservation
9258            .targets
9259            .iter()
9260            .find(|target| target.pool_index == target_pool_index)
9261            .map(|target| {
9262                target.remaining_reserved_physical_bytes(reservation.temporary_copies)
9263                    / 1usize.saturating_add(reservation.temporary_copies)
9264            })
9265            .unwrap_or_default();
9266        let remaining_total_physical_bytes = reservation
9267            .predicted_physical_bytes
9268            .saturating_sub(reservation.consumed_target_physical_bytes);
9269        let remaining_data_bytes = reservation
9270            .source_data_equivalent_bytes
9271            .saturating_sub(reservation.committed_data_bytes);
9272        if expected_target_physical_bytes > remaining_target_physical_bytes
9273            || expected_target_physical_bytes > remaining_total_physical_bytes
9274            || expected_data_bytes > remaining_data_bytes
9275        {
9276            return Err(decommission_capacity_blocked_error(format!(
9277                "source pool {source_pool_index} target pool {target_pool_index} lacks reservation capacity for the exact object"
9278            )));
9279        }
9280        matches.push(ExactDeleteCapacityReconciliation {
9281            source_pool_index,
9282            target_pool_index,
9283            mutation_id,
9284            expected_data_bytes,
9285            expected_target_physical_bytes,
9286        });
9287    }
9288
9289    Ok(matches)
9290}
9291
9292fn ensure_exact_delete_capacity_namespace_fences(opts: &ObjectOptions, bucket: &str, object: &str) -> Result<()> {
9293    let object_fence = opts.namespace_lock_fence.as_ref().ok_or_else(|| {
9294        decommission_capacity_blocked_error("exact delete capacity reconciliation requires an object namespace fence")
9295    })?;
9296    if object_fence.is_lock_lost() {
9297        return Err(StorageError::NamespaceLockQuorumUnavailable {
9298            mode: "exact_delete_capacity_reconciliation",
9299            bucket: bucket.to_string(),
9300            object: decode_dir_object(object),
9301            required: 1,
9302            achieved: 0,
9303        });
9304    }
9305    if opts
9306        .bucket_lifecycle_lock_fence
9307        .as_ref()
9308        .is_some_and(crate::object_api::NamespaceLockFence::is_lock_lost)
9309    {
9310        return Err(StorageError::NamespaceLockQuorumUnavailable {
9311            mode: "exact_delete_capacity_bucket_generation",
9312            bucket: bucket.to_string(),
9313            object: decode_dir_object(object),
9314            required: 1,
9315            achieved: 0,
9316        });
9317    }
9318    Ok(())
9319}
9320
9321pub(crate) fn ensure_decommission_capacity_mutation_id(bucket: &str, object: &str, opts: &mut ObjectOptions) {
9322    if opts
9323        .decommission_capacity
9324        .as_deref()
9325        .is_none_or(|capacity| capacity.mutation_id.is_some())
9326    {
9327        return;
9328    }
9329    let Some(owner) = DecommissionCapacityOwner::from_options(opts) else {
9330        return;
9331    };
9332    let mutation_id =
9333        decommission_capacity_mutation_id(owner, bucket, object, opts.version_id.as_deref(), opts.delete_marker, opts.mod_time);
9334    if let Some(capacity) = opts.decommission_capacity.as_mut() {
9335        capacity.mutation_id = Some(mutation_id);
9336    }
9337}
9338
9339impl DecommissionCapacityReservation {
9340    fn active(&self) -> bool {
9341        self.released_at.is_none()
9342    }
9343
9344    fn lease_active_at(&self, now: OffsetDateTime) -> bool {
9345        self.active() && self.expires_at > now
9346    }
9347
9348    fn admits_owner(&self, owner: DecommissionCapacityOwner, now: OffsetDateTime) -> bool {
9349        self.lease_active_at(now)
9350            && self.source_pool_index == owner.source_pool_index
9351            && self.operation_id == owner.operation_id
9352            && self.generation == owner.generation
9353            && self.owner_nonce == owner.owner_nonce
9354    }
9355
9356    fn admits_cleanup_owner(&self, owner: DecommissionCapacityOwner) -> bool {
9357        self.active()
9358            && self.source_pool_index == owner.source_pool_index
9359            && self.operation_id == owner.operation_id
9360            && self.generation == owner.generation
9361    }
9362
9363    fn remaining_peak_physical_bytes(&self) -> usize {
9364        self.predicted_physical_bytes
9365            .saturating_sub(self.consumed_target_physical_bytes)
9366            .saturating_mul(1usize.saturating_add(self.temporary_copies))
9367    }
9368}
9369
9370impl DecommissionCapacityTarget {
9371    fn remaining_reserved_physical_bytes(&self, temporary_copies: usize) -> usize {
9372        self.reserved_physical_bytes.saturating_sub(
9373            self.consumed_physical_bytes
9374                .saturating_mul(1usize.saturating_add(temporary_copies)),
9375        )
9376    }
9377}
9378
9379#[derive(Debug, Clone, Serialize, Deserialize, Default)]
9380pub struct PoolDecommissionInfo {
9381    #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
9382    pub start_time: Option<OffsetDateTime>,
9383    #[serde(rename = "startSize")]
9384    pub start_size: usize,
9385    #[serde(rename = "totalSize")]
9386    pub total_size: usize,
9387    #[serde(rename = "currentSize")]
9388    pub current_size: usize,
9389    #[serde(rename = "complete")]
9390    pub complete: bool,
9391    #[serde(rename = "failed")]
9392    pub failed: bool,
9393    #[serde(rename = "canceled")]
9394    pub canceled: bool,
9395    #[serde(skip)]
9396    pub queued: bool,
9397
9398    #[serde(skip)]
9399    pub queued_buckets: Vec<String>,
9400    #[serde(skip)]
9401    pub decommissioned_buckets: Vec<String>,
9402    #[serde(skip)]
9403    pub bucket: String,
9404    #[serde(skip)]
9405    pub prefix: String,
9406    #[serde(skip)]
9407    pub object: String,
9408    #[serde(skip)]
9409    pub stage: String,
9410
9411    #[serde(rename = "objectsDecommissioned")]
9412    pub items_decommissioned: usize,
9413    #[serde(rename = "objectsDecommissionedFailed")]
9414    pub items_decommission_failed: usize,
9415    #[serde(rename = "bytesDecommissioned")]
9416    pub bytes_done: usize,
9417    #[serde(rename = "bytesDecommissionedFailed")]
9418    pub bytes_failed: usize,
9419    #[serde(rename = "terminalReloadAttemptAt", with = "time::serde::rfc3339::option", default)]
9420    pub terminal_reload_attempt_at: Option<OffsetDateTime>,
9421    #[serde(rename = "terminalReloadFailures", default)]
9422    pub terminal_reload_failures: Vec<String>,
9423    #[serde(rename = "capacityReservation", default, skip_serializing_if = "Option::is_none")]
9424    pub capacity_reservation: Option<DecommissionCapacityReservation>,
9425    #[serde(rename = "capacityBlockedReason", default, skip_serializing_if = "Option::is_none")]
9426    pub capacity_blocked_reason: Option<String>,
9427    #[serde(skip)]
9428    pub unresolved_entries: Vec<DecommissionUnresolvedEntry>,
9429    #[serde(skip)]
9430    pub progress_save_item_baseline: usize,
9431    #[serde(skip)]
9432    pub progress_save_retry_after: Option<OffsetDateTime>,
9433    #[serde(skip)]
9434    pub progress_save_last_at: Option<OffsetDateTime>,
9435}
9436
9437#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9438struct DecommissionProgressCheckpoint {
9439    start_time: Option<OffsetDateTime>,
9440    queued: bool,
9441    counted_items: usize,
9442    checkpoint_at: OffsetDateTime,
9443    capacity_operation_id: Option<uuid::Uuid>,
9444    capacity_owner_nonce: Option<uuid::Uuid>,
9445    capacity_lease_expires_at: Option<OffsetDateTime>,
9446}
9447
9448impl PoolDecommissionInfo {
9449    pub fn has_decommission_state(&self) -> bool {
9450        self.complete
9451            || self.failed
9452            || self.canceled
9453            || self.queued
9454            || self.start_time.is_some()
9455            || self.start_size > 0
9456            || !self.queued_buckets.is_empty()
9457            || !self.decommissioned_buckets.is_empty()
9458            || !self.bucket.is_empty()
9459            || !self.prefix.is_empty()
9460            || !self.object.is_empty()
9461            || !self.stage.is_empty()
9462            || self.items_decommissioned > 0
9463            || self.items_decommission_failed > 0
9464            || self.bytes_done > 0
9465            || self.bytes_failed > 0
9466            || self.terminal_reload_attempt_at.is_some()
9467            || !self.terminal_reload_failures.is_empty()
9468            || self.capacity_reservation.is_some()
9469            || self.capacity_blocked_reason.is_some()
9470            || !self.unresolved_entries.is_empty()
9471    }
9472
9473    fn counted_items(&self) -> usize {
9474        self.items_decommissioned.saturating_add(self.items_decommission_failed)
9475    }
9476
9477    fn items_since_last_progress_save(&self) -> usize {
9478        self.counted_items().saturating_sub(self.progress_save_item_baseline)
9479    }
9480
9481    fn mark_progress_saved(&mut self) {
9482        self.progress_save_item_baseline = self.counted_items();
9483        self.progress_save_retry_after = None;
9484    }
9485
9486    pub fn bucket_push(&mut self, bucket: &DecomBucketInfo) {
9487        let bucket_key = bucket.to_string();
9488        if self.is_bucket_decommissioned(&bucket_key) {
9489            return;
9490        }
9491
9492        for b in self.queued_buckets.iter() {
9493            if b == &bucket_key {
9494                return;
9495            }
9496        }
9497
9498        self.queued_buckets.push(bucket_key);
9499
9500        self.bucket = bucket.name.clone();
9501        self.prefix = bucket.prefix.clone();
9502    }
9503    pub fn is_bucket_decommissioned(&self, bucket: &String) -> bool {
9504        for b in self.decommissioned_buckets.iter() {
9505            if b == bucket {
9506                return true;
9507            }
9508        }
9509        false
9510    }
9511    pub fn bucket_pop(&mut self, bucket: &String) -> bool {
9512        let mut found = None;
9513        for (i, b) in self.queued_buckets.iter().enumerate() {
9514            if b == bucket {
9515                found = Some(i);
9516                break;
9517            }
9518        }
9519
9520        if let Some(i) = found {
9521            if !self.is_bucket_decommissioned(bucket) {
9522                self.decommissioned_buckets.push(bucket.clone());
9523            }
9524            self.queued_buckets.remove(i);
9525            if &self.bucket == bucket {
9526                self.bucket = "".to_owned();
9527                self.prefix = "".to_owned();
9528                self.object = "".to_owned();
9529            }
9530
9531            return true;
9532        }
9533        false
9534    }
9535}
9536
9537#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9538pub struct PoolSpaceInfo {
9539    pub free: usize,
9540    pub total: usize,
9541    pub used: usize,
9542}
9543
9544#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9545pub(crate) struct DecommissionPoolCapacityInfo {
9546    pool_index: usize,
9547    space: PoolSpaceInfo,
9548    layout: DecommissionErasureLayout,
9549    physical_free: usize,
9550    physical_total: usize,
9551    physical_used: usize,
9552}
9553
9554impl DecommissionPoolCapacityInfo {
9555    #[cfg(any(test, feature = "test-util"))]
9556    pub(crate) fn for_test(
9557        pool_index: usize,
9558        layout: DecommissionErasureLayout,
9559        physical_free: usize,
9560        physical_total: usize,
9561        physical_used: usize,
9562    ) -> Self {
9563        Self {
9564            pool_index,
9565            space: PoolSpaceInfo {
9566                free: physical_free,
9567                total: physical_total,
9568                used: physical_used,
9569            },
9570            layout,
9571            physical_free,
9572            physical_total,
9573            physical_used,
9574        }
9575    }
9576}
9577
9578#[cfg(any(test, feature = "test-util"))]
9579type DecommissionCapacityInfoOverrides =
9580    std::sync::Mutex<HashMap<uuid::Uuid, std::collections::VecDeque<Vec<DecommissionPoolCapacityInfo>>>>;
9581
9582#[cfg(any(test, feature = "test-util"))]
9583static DECOMMISSION_CAPACITY_INFO_OVERRIDES: std::sync::OnceLock<DecommissionCapacityInfoOverrides> = std::sync::OnceLock::new();
9584
9585/// Queues capacity snapshots consumed in order by `get_decommission_all_pool_capacity_infos`;
9586/// the final snapshot is retained and replayed for every subsequent sample, so tests never
9587/// fall back to the host's real disk statistics once an override is installed.
9588#[cfg(any(test, feature = "test-util"))]
9589pub(crate) fn set_decommission_capacity_info_overrides_for_test(
9590    store_id: uuid::Uuid,
9591    snapshots: Vec<Vec<DecommissionPoolCapacityInfo>>,
9592) {
9593    DECOMMISSION_CAPACITY_INFO_OVERRIDES
9594        .get_or_init(|| std::sync::Mutex::new(HashMap::new()))
9595        .lock()
9596        .expect("decommission capacity info override should not be poisoned")
9597        .insert(store_id, snapshots.into());
9598}
9599
9600#[cfg(any(test, feature = "test-util"))]
9601fn take_decommission_capacity_info_override_for_test(store_id: uuid::Uuid) -> Option<Vec<DecommissionPoolCapacityInfo>> {
9602    let mut overrides = DECOMMISSION_CAPACITY_INFO_OVERRIDES
9603        .get_or_init(|| std::sync::Mutex::new(HashMap::new()))
9604        .lock()
9605        .expect("decommission capacity info override should not be poisoned");
9606    let queue = overrides.get_mut(&store_id)?;
9607    if queue.len() > 1 {
9608        queue.pop_front()
9609    } else {
9610        // The final snapshot is replayed forever: extra sampling points added to
9611        // the decommission paths must keep observing injected capacity instead of
9612        // silently falling back to the host's real statfs numbers (see #6989).
9613        queue.front().cloned()
9614    }
9615}
9616
9617#[cfg(test)]
9618struct DecommissionCapacityLockOrderBarrierState {
9619    owner_store_id: uuid::Uuid,
9620    external_store_id: uuid::Uuid,
9621    owner_arrived: tokio::sync::Notify,
9622    owner_release: tokio::sync::Notify,
9623    owner_pause_enabled: AtomicBool,
9624    external_capacity_released: tokio::sync::Notify,
9625    external_object_capacity_probe_acquired: tokio::sync::Notify,
9626    external_object_capacity_probe_release: tokio::sync::Notify,
9627    external_object_capacity_probe_paused: AtomicBool,
9628    external_object_commit_phase_started: tokio::sync::Notify,
9629    external_object_commit_phase_release: tokio::sync::Notify,
9630    external_object_commit_phase_paused: AtomicBool,
9631    external_heal_operation_started: tokio::sync::Notify,
9632    external_heal_target_lock_attempted: tokio::sync::Notify,
9633    target_gate_retry_entered: tokio::sync::Notify,
9634    target_gate_retry_entries: AtomicUsize,
9635    target_gate_exact_reloads: AtomicUsize,
9636    target_gate_acquire_pause_target: AtomicUsize,
9637    target_gate_acquire_entered: tokio::sync::Notify,
9638    target_gate_acquire_release: tokio::sync::Notify,
9639    cancel_before_start_entered: tokio::sync::Notify,
9640    cancel_before_start_release: tokio::sync::Notify,
9641    cancel_before_start_paused: AtomicBool,
9642    cancel_target_timeout_entered: tokio::sync::Notify,
9643    cancel_target_timeout_release: tokio::sync::Notify,
9644    cancel_target_timeout_paused: AtomicBool,
9645    cancel_target_timeouts: AtomicUsize,
9646}
9647
9648#[cfg(test)]
9649pub(crate) struct DecommissionCapacityLockOrderBarrier {
9650    state: Arc<DecommissionCapacityLockOrderBarrierState>,
9651}
9652
9653#[cfg(test)]
9654static DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER: std::sync::OnceLock<
9655    std::sync::Mutex<Option<Arc<DecommissionCapacityLockOrderBarrierState>>>,
9656> = std::sync::OnceLock::new();
9657
9658#[cfg(test)]
9659impl DecommissionCapacityLockOrderBarrier {
9660    pub(crate) fn install(owner_store_id: uuid::Uuid, external_store_id: uuid::Uuid) -> Self {
9661        let state = Arc::new(DecommissionCapacityLockOrderBarrierState {
9662            owner_store_id,
9663            external_store_id,
9664            owner_arrived: tokio::sync::Notify::new(),
9665            owner_release: tokio::sync::Notify::new(),
9666            owner_pause_enabled: AtomicBool::new(true),
9667            external_capacity_released: tokio::sync::Notify::new(),
9668            external_object_capacity_probe_acquired: tokio::sync::Notify::new(),
9669            external_object_capacity_probe_release: tokio::sync::Notify::new(),
9670            external_object_capacity_probe_paused: AtomicBool::new(false),
9671            external_object_commit_phase_started: tokio::sync::Notify::new(),
9672            external_object_commit_phase_release: tokio::sync::Notify::new(),
9673            external_object_commit_phase_paused: AtomicBool::new(false),
9674            external_heal_operation_started: tokio::sync::Notify::new(),
9675            external_heal_target_lock_attempted: tokio::sync::Notify::new(),
9676            target_gate_retry_entered: tokio::sync::Notify::new(),
9677            target_gate_retry_entries: AtomicUsize::new(0),
9678            target_gate_exact_reloads: AtomicUsize::new(0),
9679            target_gate_acquire_pause_target: AtomicUsize::new(usize::MAX),
9680            target_gate_acquire_entered: tokio::sync::Notify::new(),
9681            target_gate_acquire_release: tokio::sync::Notify::new(),
9682            cancel_before_start_entered: tokio::sync::Notify::new(),
9683            cancel_before_start_release: tokio::sync::Notify::new(),
9684            cancel_before_start_paused: AtomicBool::new(false),
9685            cancel_target_timeout_entered: tokio::sync::Notify::new(),
9686            cancel_target_timeout_release: tokio::sync::Notify::new(),
9687            cancel_target_timeout_paused: AtomicBool::new(false),
9688            cancel_target_timeouts: AtomicUsize::new(0),
9689        });
9690        let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9691            .get_or_init(|| std::sync::Mutex::new(None))
9692            .lock()
9693            .expect("decommission capacity lock-order barrier should not be poisoned");
9694        assert!(slot.is_none(), "decommission capacity lock-order barrier must be unique");
9695        *slot = Some(Arc::clone(&state));
9696        Self { state }
9697    }
9698
9699    #[cfg(feature = "test-util")]
9700    pub(crate) async fn wait_until_owner_paused(&self) {
9701        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.owner_arrived.notified())
9702            .await
9703            .expect("owned capacity mutation should reach admission before its metadata write");
9704    }
9705
9706    pub(crate) async fn wait_until_external_capacity_released(&self) {
9707        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.external_capacity_released.notified())
9708            .await
9709            .expect("external mutation should release capacity before waiting for the object namespace");
9710    }
9711
9712    #[cfg(feature = "test-util")]
9713    pub(crate) async fn wait_until_external_object_capacity_probe_acquired(&self) {
9714        tokio::time::timeout(
9715            std::time::Duration::from_secs(30),
9716            self.state.external_object_capacity_probe_acquired.notified(),
9717        )
9718        .await
9719        .expect("external object mutation should acquire its no-active capacity probe");
9720    }
9721
9722    #[cfg(feature = "test-util")]
9723    pub(crate) async fn wait_until_external_object_commit_phase_started(&self) {
9724        tokio::time::timeout(
9725            std::time::Duration::from_secs(30),
9726            self.state.external_object_commit_phase_started.notified(),
9727        )
9728        .await
9729        .expect("external object mutation should reach its staged commit phase before waiting for the namespace");
9730    }
9731
9732    pub(crate) async fn wait_until_external_heal_operation_started(&self) {
9733        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.external_heal_operation_started.notified())
9734            .await
9735            .expect("external heal should reach the target operation after capacity admission");
9736    }
9737
9738    pub(crate) async fn wait_until_external_heal_target_lock_attempted(&self) {
9739        tokio::time::timeout(
9740            std::time::Duration::from_secs(30),
9741            self.state.external_heal_target_lock_attempted.notified(),
9742        )
9743        .await
9744        .expect("external heal should attempt the target namespace lock after capacity admission");
9745    }
9746
9747    #[cfg(feature = "test-util")]
9748    pub(crate) async fn wait_until_target_gate_retry(&self) {
9749        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.target_gate_retry_entered.notified())
9750            .await
9751            .expect("decommission entry should observe target gate contention");
9752    }
9753
9754    #[cfg(feature = "test-util")]
9755    pub(crate) async fn wait_until_target_gate_retries(&self, expected: usize) {
9756        tokio::time::timeout(std::time::Duration::from_secs(30), async {
9757            loop {
9758                let notified = self.state.target_gate_retry_entered.notified();
9759                if self.state.target_gate_retry_entries.load(Ordering::Acquire) >= expected {
9760                    return;
9761                }
9762                notified.await;
9763            }
9764        })
9765        .await
9766        .expect("decommission entries should observe target gate contention");
9767    }
9768
9769    #[cfg(feature = "test-util")]
9770    pub(crate) fn target_gate_exact_reloads(&self) -> usize {
9771        self.state.target_gate_exact_reloads.load(Ordering::Acquire)
9772    }
9773
9774    #[cfg(feature = "test-util")]
9775    pub(crate) fn pause_target_gate_acquire(&self, target_pool_index: usize) {
9776        self.state
9777            .target_gate_acquire_pause_target
9778            .store(target_pool_index, Ordering::Release);
9779    }
9780
9781    #[cfg(feature = "test-util")]
9782    pub(crate) async fn wait_until_target_gate_acquire_paused(&self) {
9783        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.target_gate_acquire_entered.notified())
9784            .await
9785            .expect("decommission capacity mutation should pause before acquiring its target gate");
9786    }
9787
9788    #[cfg(feature = "test-util")]
9789    pub(crate) fn release_target_gate_acquire(&self) {
9790        self.state
9791            .target_gate_acquire_pause_target
9792            .store(usize::MAX, Ordering::Release);
9793        self.state.target_gate_acquire_release.notify_one();
9794    }
9795
9796    pub(crate) fn pause_cancel_before_start(&self) {
9797        self.state.cancel_before_start_paused.store(true, Ordering::Release);
9798    }
9799
9800    pub(crate) async fn wait_until_cancel_before_start(&self) {
9801        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.cancel_before_start_entered.notified())
9802            .await
9803            .expect("decommission cancel should pause before acquiring the start gate");
9804    }
9805
9806    pub(crate) fn release_cancel_before_start(&self) {
9807        self.state.cancel_before_start_release.notify_one();
9808    }
9809
9810    fn pause_cancel_target_timeout(&self) {
9811        self.state.cancel_target_timeout_paused.store(true, Ordering::Release);
9812    }
9813
9814    async fn wait_until_cancel_target_timeout(&self) {
9815        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.cancel_target_timeout_entered.notified())
9816            .await
9817            .expect("cancel should observe contention on its target capacity fence");
9818    }
9819
9820    fn release_cancel_target_timeout(&self) {
9821        self.state.cancel_target_timeout_paused.store(false, Ordering::Release);
9822        self.state.cancel_target_timeout_release.notify_one();
9823    }
9824
9825    #[cfg(feature = "test-util")]
9826    pub(crate) fn release_owner(&self) {
9827        self.state.owner_release.notify_one();
9828        self.state.target_gate_acquire_release.notify_one();
9829    }
9830
9831    #[cfg(feature = "test-util")]
9832    pub(crate) fn disable_owner_pause(&self) {
9833        self.state.owner_pause_enabled.store(false, Ordering::Release);
9834        self.state.owner_release.notify_waiters();
9835    }
9836
9837    #[cfg(feature = "test-util")]
9838    pub(crate) fn pause_external_object_commit_phase(&self) {
9839        self.state.external_object_commit_phase_paused.store(true, Ordering::Release);
9840    }
9841
9842    #[cfg(feature = "test-util")]
9843    pub(crate) fn release_external_object_commit_phase(&self) {
9844        self.state.external_object_commit_phase_release.notify_one();
9845    }
9846
9847    #[cfg(feature = "test-util")]
9848    pub(crate) fn pause_external_object_capacity_probe(&self) {
9849        self.state
9850            .external_object_capacity_probe_paused
9851            .store(true, Ordering::Release);
9852    }
9853
9854    #[cfg(feature = "test-util")]
9855    pub(crate) fn release_external_object_capacity_probe(&self) {
9856        self.state.external_object_capacity_probe_release.notify_one();
9857    }
9858}
9859
9860#[cfg(test)]
9861impl Drop for DecommissionCapacityLockOrderBarrier {
9862    fn drop(&mut self) {
9863        self.state.owner_release.notify_one();
9864        self.state.external_object_capacity_probe_release.notify_one();
9865        self.state.external_object_commit_phase_release.notify_one();
9866        self.state.cancel_before_start_release.notify_one();
9867        self.state.cancel_target_timeout_release.notify_one();
9868        let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9869            .get_or_init(|| std::sync::Mutex::new(None))
9870            .lock()
9871            .expect("decommission capacity lock-order barrier should not be poisoned");
9872        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
9873            *slot = None;
9874        }
9875    }
9876}
9877
9878#[cfg(test)]
9879async fn pause_decommission_capacity_before_owner_write(store_id: uuid::Uuid) {
9880    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9881        .get_or_init(|| std::sync::Mutex::new(None))
9882        .lock()
9883        .expect("decommission capacity lock-order barrier should not be poisoned")
9884        .as_ref()
9885        .filter(|state| state.owner_store_id == store_id && state.owner_pause_enabled.load(Ordering::Acquire))
9886        .cloned();
9887    if let Some(barrier) = barrier {
9888        barrier.owner_arrived.notify_one();
9889        barrier.owner_release.notified().await;
9890    }
9891}
9892
9893#[cfg(test)]
9894async fn pause_decommission_capacity_before_target_gate_acquire(store_id: uuid::Uuid, target_pool_index: usize) {
9895    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9896        .get_or_init(|| std::sync::Mutex::new(None))
9897        .lock()
9898        .expect("decommission capacity lock-order barrier should not be poisoned")
9899        .as_ref()
9900        .filter(|state| {
9901            state.owner_store_id == store_id
9902                && state.target_gate_acquire_pause_target.load(Ordering::Acquire) == target_pool_index
9903        })
9904        .cloned();
9905    if let Some(barrier) = barrier {
9906        barrier.target_gate_acquire_entered.notify_one();
9907        barrier.target_gate_acquire_release.notified().await;
9908    }
9909}
9910
9911#[cfg(test)]
9912async fn pause_decommission_cancel_before_start_gate(store_id: uuid::Uuid) {
9913    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9914        .get_or_init(|| std::sync::Mutex::new(None))
9915        .lock()
9916        .expect("decommission capacity lock-order barrier should not be poisoned")
9917        .as_ref()
9918        .filter(|state| state.owner_store_id == store_id && state.cancel_before_start_paused.load(Ordering::Acquire))
9919        .cloned();
9920    if let Some(barrier) = barrier {
9921        barrier.cancel_before_start_entered.notify_one();
9922        barrier.cancel_before_start_release.notified().await;
9923    }
9924}
9925
9926#[cfg(test)]
9927async fn pause_decommission_cancel_target_timeout(store_id: uuid::Uuid) {
9928    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9929        .get_or_init(|| std::sync::Mutex::new(None))
9930        .lock()
9931        .expect("decommission capacity lock-order barrier should not be poisoned")
9932        .as_ref()
9933        .filter(|state| state.owner_store_id == store_id)
9934        .cloned();
9935    if let Some(barrier) = barrier {
9936        barrier.cancel_target_timeouts.fetch_add(1, Ordering::AcqRel);
9937        barrier.cancel_target_timeout_entered.notify_one();
9938        if barrier.cancel_target_timeout_paused.load(Ordering::Acquire) {
9939            barrier.cancel_target_timeout_release.notified().await;
9940        }
9941    }
9942}
9943
9944#[cfg(test)]
9945fn notify_decommission_target_gate_retry(store_id: uuid::Uuid) {
9946    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9947        .get_or_init(|| std::sync::Mutex::new(None))
9948        .lock()
9949        .expect("decommission capacity lock-order barrier should not be poisoned")
9950        .as_ref()
9951        .filter(|state| state.owner_store_id == store_id)
9952        .cloned();
9953    if let Some(barrier) = barrier {
9954        barrier.target_gate_retry_entries.fetch_add(1, Ordering::AcqRel);
9955        barrier.target_gate_retry_entered.notify_one();
9956    }
9957}
9958
9959#[cfg(test)]
9960fn notify_decommission_target_gate_exact_reload(store_id: uuid::Uuid) {
9961    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9962        .get_or_init(|| std::sync::Mutex::new(None))
9963        .lock()
9964        .expect("decommission capacity lock-order barrier should not be poisoned")
9965        .as_ref()
9966        .filter(|state| state.owner_store_id == store_id)
9967        .cloned();
9968    if let Some(barrier) = barrier {
9969        barrier.target_gate_exact_reloads.fetch_add(1, Ordering::AcqRel);
9970    }
9971}
9972
9973#[cfg(test)]
9974pub(crate) fn notify_decommission_external_object_capacity_released(store_id: uuid::Uuid) {
9975    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9976        .get_or_init(|| std::sync::Mutex::new(None))
9977        .lock()
9978        .expect("decommission capacity lock-order barrier should not be poisoned")
9979        .as_ref()
9980        .filter(|state| state.external_store_id == store_id)
9981        .cloned();
9982    if let Some(barrier) = barrier {
9983        barrier.external_capacity_released.notify_one();
9984    }
9985}
9986
9987#[cfg(test)]
9988pub(crate) fn notify_decommission_external_object_capacity_probe_acquired(store_id: uuid::Uuid) {
9989    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
9990        .get_or_init(|| std::sync::Mutex::new(None))
9991        .lock()
9992        .expect("decommission capacity lock-order barrier should not be poisoned")
9993        .as_ref()
9994        .filter(|state| state.external_store_id == store_id)
9995        .cloned();
9996    if let Some(barrier) = barrier {
9997        barrier.external_object_capacity_probe_acquired.notify_one();
9998    }
9999}
10000
10001#[cfg(test)]
10002pub(crate) async fn wait_for_decommission_external_object_capacity_probe_release(store_id: uuid::Uuid) {
10003    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10004        .get_or_init(|| std::sync::Mutex::new(None))
10005        .lock()
10006        .expect("decommission capacity lock-order barrier should not be poisoned")
10007        .as_ref()
10008        .filter(|state| state.external_store_id == store_id)
10009        .cloned();
10010    if let Some(barrier) = barrier
10011        && barrier.external_object_capacity_probe_paused.load(Ordering::Acquire)
10012    {
10013        barrier.external_object_capacity_probe_release.notified().await;
10014    }
10015}
10016
10017#[cfg(test)]
10018pub(crate) fn notify_decommission_external_object_commit_phase_started(store_id: uuid::Uuid) {
10019    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10020        .get_or_init(|| std::sync::Mutex::new(None))
10021        .lock()
10022        .expect("decommission capacity lock-order barrier should not be poisoned")
10023        .as_ref()
10024        .filter(|state| state.external_store_id == store_id)
10025        .cloned();
10026    if let Some(barrier) = barrier {
10027        barrier.external_object_commit_phase_started.notify_one();
10028    }
10029}
10030
10031#[cfg(test)]
10032pub(crate) async fn wait_for_decommission_external_object_commit_phase_release(store_id: uuid::Uuid) {
10033    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10034        .get_or_init(|| std::sync::Mutex::new(None))
10035        .lock()
10036        .expect("decommission capacity lock-order barrier should not be poisoned")
10037        .as_ref()
10038        .filter(|state| state.external_store_id == store_id)
10039        .cloned();
10040    if let Some(barrier) = barrier
10041        && barrier.external_object_commit_phase_paused.load(Ordering::Acquire)
10042    {
10043        barrier.external_object_commit_phase_release.notified().await;
10044    }
10045}
10046
10047#[cfg(test)]
10048pub(crate) fn notify_decommission_external_heal_operation_started(store_id: uuid::Uuid) {
10049    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10050        .get_or_init(|| std::sync::Mutex::new(None))
10051        .lock()
10052        .expect("decommission capacity lock-order barrier should not be poisoned")
10053        .as_ref()
10054        .filter(|state| state.external_store_id == store_id)
10055        .cloned();
10056    if let Some(barrier) = barrier {
10057        barrier.external_heal_operation_started.notify_one();
10058    }
10059}
10060
10061#[cfg(test)]
10062pub(crate) fn notify_decommission_external_heal_target_lock_attempted() {
10063    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10064        .get_or_init(|| std::sync::Mutex::new(None))
10065        .lock()
10066        .expect("decommission capacity lock-order barrier should not be poisoned")
10067        .as_ref()
10068        .cloned();
10069    if let Some(barrier) = barrier {
10070        barrier.external_heal_target_lock_attempted.notify_one();
10071    }
10072}
10073
10074#[derive(Debug, Default, Clone)]
10075pub struct DecomBucketInfo {
10076    pub name: String,
10077    pub prefix: String,
10078}
10079
10080impl Display for DecomBucketInfo {
10081    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10082        write!(
10083            f,
10084            "{}",
10085            path_join(&[PathBuf::from(self.name.clone()), PathBuf::from(self.prefix.clone())]).to_string_lossy()
10086        )
10087    }
10088}
10089
10090#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10091enum DecommissionFinalState {
10092    Complete,
10093    Failed,
10094}
10095
10096fn determine_decommission_final_state(items_failed: usize, was_cancelled: bool) -> DecommissionFinalState {
10097    if items_failed > 0 || was_cancelled {
10098        DecommissionFinalState::Failed
10099    } else {
10100        DecommissionFinalState::Complete
10101    }
10102}
10103
10104fn decommission_remaining_version_count(versions: &[rustfs_filemeta::FileInfo], expired: usize) -> usize {
10105    versions
10106        .iter()
10107        .filter(|version| !version.tier_free_version())
10108        .count()
10109        .saturating_sub(expired)
10110}
10111
10112fn should_skip_decommission_delete_marker(
10113    version: &rustfs_filemeta::FileInfo,
10114    remaining_versions: usize,
10115    replication_configured: bool,
10116) -> bool {
10117    // Match MinIO decommission behavior: an empty delete marker is not moved to
10118    // another pool unless replication is configured and its marker state matters.
10119    version.deleted && remaining_versions == 1 && !replication_configured
10120}
10121
10122fn decommission_delete_marker_opts(
10123    version: &rustfs_filemeta::FileInfo,
10124    version_id: Option<String>,
10125    src_pool_idx: usize,
10126    expected_bucket_incarnation_id: Option<uuid::Uuid>,
10127) -> ObjectOptions {
10128    let version_suspended = version.version_id.is_none() && version_id.is_none();
10129    ObjectOptions {
10130        versioned: !version_suspended,
10131        version_suspended,
10132        version_id: version_id.or_else(|| version_suspended.then(|| uuid::Uuid::nil().to_string())),
10133        mod_time: version.mod_time,
10134        src_pool_idx,
10135        data_movement: true,
10136        delete_marker: true,
10137        skip_decommissioned: true,
10138        expected_bucket_incarnation_id,
10139        delete_replication: version
10140            .replication_state_internal
10141            .as_ref()
10142            .map(replication_state_from_filemeta),
10143        ..Default::default()
10144    }
10145}
10146
10147fn decommission_object_migration_read_opts(version_id: Option<String>) -> ObjectOptions {
10148    ObjectOptions {
10149        version_id,
10150        no_lock: true,
10151        data_movement: true,
10152        raw_data_movement_read: true,
10153        skip_decommissioned: true,
10154        skip_rebalancing: true,
10155        ..Default::default()
10156    }
10157}
10158
10159fn decommission_remote_tiered_opts(
10160    version: &rustfs_filemeta::FileInfo,
10161    version_id: Option<String>,
10162    src_pool_idx: usize,
10163    expected_bucket_incarnation_id: Option<uuid::Uuid>,
10164) -> ObjectOptions {
10165    ObjectOptions {
10166        versioned: version_id.is_some(),
10167        version_id,
10168        mod_time: version.mod_time,
10169        user_defined: version.metadata.clone(),
10170        src_pool_idx,
10171        data_movement: true,
10172        incl_free_versions: version.tier_free_version(),
10173        include_part_checksums: true,
10174        http_preconditions: Some(data_movement::data_movement_target_precondition()),
10175        expected_bucket_incarnation_id,
10176        ..Default::default()
10177    }
10178}
10179
10180fn decommission_capacity_version_mutation_id(
10181    owner: DecommissionCapacityOwner,
10182    bucket: &str,
10183    version: &rustfs_filemeta::FileInfo,
10184) -> uuid::Uuid {
10185    let version_id = if version.deleted && version.version_id.is_none() {
10186        Some(uuid::Uuid::nil().to_string())
10187    } else {
10188        version.version_id.map(|version_id| version_id.to_string())
10189    };
10190    decommission_capacity_mutation_id(owner, bucket, &version.name, version_id.as_deref(), version.deleted, version.mod_time)
10191}
10192
10193fn decommission_capacity_owned_opts(mut opts: ObjectOptions, capacity_owner: Option<DecommissionCapacityOwner>) -> ObjectOptions {
10194    if let Some(capacity_owner) = capacity_owner {
10195        capacity_owner.apply_to(&mut opts);
10196    }
10197    opts
10198}
10199
10200fn lifecycle_action_removes_data_movement_version(action: IlmAction) -> bool {
10201    matches!(
10202        action,
10203        IlmAction::DeleteVersionAction | IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction
10204    )
10205}
10206
10207fn lifecycle_action_skips_heal_version(action: IlmAction) -> bool {
10208    action.delete()
10209}
10210
10211#[cfg(test)]
10212struct LifecycleDataMovementMutationBarrierState {
10213    bucket: String,
10214    object: String,
10215    arrived: tokio::sync::Notify,
10216    release: tokio::sync::Notify,
10217}
10218
10219#[cfg(test)]
10220pub(crate) struct LifecycleDataMovementMutationBarrier {
10221    state: Arc<LifecycleDataMovementMutationBarrierState>,
10222}
10223
10224#[cfg(test)]
10225static LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER: std::sync::OnceLock<
10226    std::sync::Mutex<Option<Arc<LifecycleDataMovementMutationBarrierState>>>,
10227> = std::sync::OnceLock::new();
10228
10229#[cfg(test)]
10230impl LifecycleDataMovementMutationBarrier {
10231    pub(crate) fn install(bucket: &str, object: &str) -> Self {
10232        let state = Arc::new(LifecycleDataMovementMutationBarrierState {
10233            bucket: bucket.to_string(),
10234            object: object.to_string(),
10235            arrived: tokio::sync::Notify::new(),
10236            release: tokio::sync::Notify::new(),
10237        });
10238        let mut slot = LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER
10239            .get_or_init(|| std::sync::Mutex::new(None))
10240            .lock()
10241            .expect("lifecycle data movement mutation barrier should not be poisoned");
10242        assert!(slot.is_none(), "lifecycle data movement mutation barrier must be unique");
10243        *slot = Some(Arc::clone(&state));
10244        Self { state }
10245    }
10246
10247    pub(crate) async fn wait_until_paused(&self) {
10248        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.arrived.notified())
10249            .await
10250            .expect("lifecycle data movement should reach its mutation boundary");
10251    }
10252
10253    pub(crate) fn release(&self) {
10254        self.state.release.notify_one();
10255    }
10256}
10257
10258#[cfg(test)]
10259impl Drop for LifecycleDataMovementMutationBarrier {
10260    fn drop(&mut self) {
10261        self.state.release.notify_one();
10262        let mut slot = LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER
10263            .get_or_init(|| std::sync::Mutex::new(None))
10264            .lock()
10265            .expect("lifecycle data movement mutation barrier should not be poisoned");
10266        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
10267            *slot = None;
10268        }
10269    }
10270}
10271
10272#[cfg(test)]
10273async fn pause_lifecycle_data_movement_mutation(bucket: &str, object: &str, has_run_fence_signal: bool) {
10274    if !has_run_fence_signal {
10275        return;
10276    }
10277    let barrier = LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER
10278        .get_or_init(|| std::sync::Mutex::new(None))
10279        .lock()
10280        .expect("lifecycle data movement mutation barrier should not be poisoned")
10281        .as_ref()
10282        .filter(|barrier| barrier.bucket == bucket && barrier.object == object)
10283        .cloned();
10284    if let Some(barrier) = barrier {
10285        barrier.arrived.notify_one();
10286        barrier.release.notified().await;
10287    }
10288}
10289
10290fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_actions: bool, applied: bool) -> Result<bool> {
10291    if !apply_actions || applied {
10292        return Ok(true);
10293    }
10294
10295    Err(Error::other(format!(
10296        "failed to apply lifecycle expiry action {action:?} during data movement"
10297    )))
10298}
10299
10300#[allow(clippy::too_many_arguments)]
10301pub(crate) async fn should_skip_lifecycle_for_data_movement(
10302    store: Arc<ECStore>,
10303    bucket: &str,
10304    version: &rustfs_filemeta::FileInfo,
10305    lifecycle_config: Option<&BucketLifecycleConfiguration>,
10306    object_lock_config: Option<&ObjectLockConfiguration>,
10307    apply_actions: bool,
10308    event_source: &LcEventSrc,
10309    lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
10310) -> Result<bool> {
10311    let Some(lifecycle_config) = lifecycle_config else {
10312        return Ok(false);
10313    };
10314
10315    let versioned = match BucketVersioningSys::get_in(&store.ctx, bucket).await {
10316        Ok(config) => config.prefix_enabled(&version.name),
10317        Err(err) => {
10318            warn!(
10319                event = EVENT_DECOMMISSION_ENTRY,
10320                component = LOG_COMPONENT_ECSTORE,
10321                subsystem = LOG_SUBSYSTEM_POOLS,
10322                state = "versioning_config_unavailable",
10323                bucket = %bucket,
10324                error = %err,
10325                "Decommission lifecycle versioning config unavailable; treating object as unversioned"
10326            );
10327            false
10328        }
10329    };
10330    let object_info = crate::object_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned);
10331    let event = eval_action_from_lifecycle(lifecycle_config, object_lock_config, &object_info).await;
10332
10333    match event.action {
10334        IlmAction::DeleteRestoredAction | IlmAction::DeleteRestoredVersionAction => {
10335            if apply_actions && object_info.is_remote() {
10336                let Ok(bucket_incarnation_id) = store.bucket_incarnation_id_from_disk(bucket).await else {
10337                    return Ok(false);
10338                };
10339                let _ = match lock_lost_signal {
10340                    Some(signal) => {
10341                        apply_expiry_rule_for_data_movement(store, &event, event_source, &object_info, Some(signal)).await
10342                    }
10343                    None => {
10344                        apply_expiry_on_transitioned_object(store, &object_info, &event, event_source, bucket_incarnation_id)
10345                            .await
10346                    }
10347                };
10348            }
10349            Ok(false)
10350        }
10351        action if lifecycle_action_removes_data_movement_version(action) => {
10352            #[cfg(test)]
10353            pause_lifecycle_data_movement_mutation(bucket, &version.name, lock_lost_signal.is_some()).await;
10354            if lifecycle_delete_all_versions_blocked_by_replication(store.clone(), bucket, &object_info.name, action).await? {
10355                return Ok(false);
10356            }
10357            let applied = !apply_actions
10358                || match lock_lost_signal {
10359                    Some(signal) => {
10360                        apply_expiry_rule_for_data_movement(store, &event, event_source, &object_info, Some(signal)).await
10361                    }
10362                    None => apply_expiry_rule_in(store, &event, event_source, &object_info).await,
10363                };
10364            resolve_data_movement_lifecycle_expiry_result(action, apply_actions, applied)
10365        }
10366        _ => Ok(false),
10367    }
10368}
10369
10370pub struct HealLifecycleExpiryContext {
10371    configs: LifecycleExpiryConfigs,
10372}
10373
10374impl ECStore {
10375    pub async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
10376        if bucket == RUSTFS_META_BUCKET {
10377            return Ok(None);
10378        }
10379
10380        let configs = get_expiry_configs(self, bucket).await?;
10381        if configs.lifecycle.is_none() {
10382            return Ok(None);
10383        }
10384
10385        Ok(Some(HealLifecycleExpiryContext { configs }))
10386    }
10387
10388    pub async fn enqueue_heal_lifecycle_expiry(
10389        self: &Arc<Self>,
10390        context: &HealLifecycleExpiryContext,
10391        bucket: &str,
10392        object: &str,
10393        version_id: Option<&str>,
10394        object_info: Option<&crate::object_api::ObjectInfo>,
10395    ) -> Result<bool> {
10396        let Some(lifecycle_config) = context.configs.lifecycle.as_ref() else {
10397            return Ok(false);
10398        };
10399
10400        let object_info = if let Some(object_info) = object_info {
10401            if object_info.bucket != bucket || object_info.name != object {
10402                return Ok(false);
10403            }
10404            let snapshot_version_id = object_info
10405                .version_id
10406                .filter(|version_id| !version_id.is_nil())
10407                .map(|version_id| version_id.to_string());
10408            if snapshot_version_id.as_deref() != version_id {
10409                return Ok(false);
10410            }
10411            object_info.clone()
10412        } else {
10413            match self
10414                .get_object_info(
10415                    bucket,
10416                    object,
10417                    &ObjectOptions {
10418                        version_id: version_id.map(str::to_string),
10419                        versioned: version_id.is_some(),
10420                        expected_bucket_incarnation_id: Some(context.configs.bucket_incarnation_id),
10421                        ..Default::default()
10422                    },
10423                )
10424                .await
10425            {
10426                Ok(object_info) => object_info,
10427                Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => return Ok(false),
10428                Err(err) => return Err(err),
10429            }
10430        };
10431
10432        let event = eval_action_from_lifecycle(lifecycle_config, context.configs.object_lock.as_deref(), &object_info).await;
10433        if !lifecycle_action_skips_heal_version(event.action) {
10434            return Ok(false);
10435        }
10436
10437        if lifecycle_delete_all_versions_blocked_by_replication(self.clone(), bucket, &object_info.name, event.action).await? {
10438            return Ok(false);
10439        }
10440
10441        Ok(apply_expiry_rule_in(self.clone(), &event, &LcEventSrc::Scanner, &object_info).await)
10442    }
10443
10444    async fn acquire_pool_meta_write_guard(
10445        &self,
10446        write_state: &mut PoolMetaWriteState,
10447        operation: &str,
10448    ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> {
10449        self.acquire_pool_meta_write_guard_with_lock_error(write_state, operation, activation_pool_meta_lock_error)
10450            .await
10451    }
10452
10453    async fn acquire_pool_meta_write_guard_with_lock_error<F>(
10454        &self,
10455        write_state: &mut PoolMetaWriteState,
10456        operation: &str,
10457        map_lock_error: F,
10458    ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)>
10459    where
10460        F: FnOnce(rustfs_lock::LockError) -> Error,
10461    {
10462        write_state.ensure_write_safe(operation)?;
10463        load_pool_meta_identity_observing(self.pools.clone(), write_state).await?;
10464        let pool = self.pools.first().cloned().ok_or_else(|| {
10465            Error::InvalidArgument(
10466                operation.to_string(),
10467                "storage-pools".to_string(),
10468                "no storage pools available".to_string(),
10469            )
10470        })?;
10471        let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
10472        let pool_meta_guard = pool_meta_lock
10473            .get_write_lock(get_lock_acquire_timeout())
10474            .await
10475            .map_err(map_lock_error)?;
10476        let selection = load_pool_meta_replicas_observing(self.pools.clone(), true, write_state).await?;
10477        write_state.observe_replicas(selection.replica_state);
10478        write_state.ensure_write_safe(operation)?;
10479        Ok((pool_meta_guard, selection.meta))
10480    }
10481
10482    async fn acquire_pool_meta_read_guard(
10483        &self,
10484        write_state: &PoolMetaWriteState,
10485        operation: &str,
10486    ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> {
10487        write_state.ensure_write_safe(operation)?;
10488        let pool = self.pools.first().cloned().ok_or_else(|| {
10489            Error::InvalidArgument(
10490                operation.to_string(),
10491                "storage-pools".to_string(),
10492                "no storage pools available".to_string(),
10493            )
10494        })?;
10495        let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
10496        let pool_meta_guard = pool_meta_lock.get_read_lock(get_lock_acquire_timeout()).await?;
10497        let selection = load_pool_meta_replicas_for_read_probe(self.pools.clone(), true, write_state, operation).await?;
10498        Ok((pool_meta_guard, selection.meta))
10499    }
10500
10501    async fn acquire_decommission_capacity_target_guard(
10502        &self,
10503        target_pool_index: usize,
10504    ) -> Result<rustfs_lock::NamespaceLockGuard> {
10505        // Some reconciliation callers already hold an object lock, while a
10506        // target mutation acquires its object lock after this gate. A short,
10507        // retryable acquisition bounds that inverse-order overlap.
10508        let pool = self.pools.first().cloned().ok_or_else(|| {
10509            Error::InvalidArgument(
10510                "decommission-capacity".to_string(),
10511                "storage-pools".to_string(),
10512                "no storage pools available".to_string(),
10513            )
10514        })?;
10515        let object = format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}");
10516        let target_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
10517        #[cfg(test)]
10518        pause_decommission_capacity_before_target_gate_acquire(self.id, target_pool_index).await;
10519        match target_lock
10520            .get_write_lock_quiet(DECOMMISSION_CAPACITY_TARGET_LOCK_TIMEOUT)
10521            .await
10522        {
10523            Ok(guard) => Ok(guard),
10524            Err(rustfs_lock::LockError::Timeout { .. } | rustfs_lock::LockError::AlreadyLocked { .. }) => {
10525                Err(decommission_capacity_blocked_error(format!(
10526                    "{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}{target_pool_index}{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}"
10527                )))
10528            }
10529            Err(rustfs_lock::LockError::QuorumNotReached { required, achieved }) => Err(Error::NamespaceLockQuorumUnavailable {
10530                mode: "write",
10531                bucket: RUSTFS_META_BUCKET.to_string(),
10532                object,
10533                required,
10534                achieved,
10535            }),
10536            Err(err) => Err(Error::Lock(err)),
10537        }
10538    }
10539
10540    async fn acquire_decommission_capacity_owner_target_guard(
10541        &self,
10542        owner: DecommissionCapacityOwner,
10543        target_pool_index: usize,
10544    ) -> Result<Option<rustfs_lock::NamespaceLockGuard>> {
10545        let model_version = self
10546            .pool_meta
10547            .read()
10548            .await
10549            .pools
10550            .get(owner.source_pool_index)
10551            .and_then(|pool| pool.decommission.as_ref())
10552            .and_then(|info| info.capacity_reservation.as_ref())
10553            .filter(|reservation| {
10554                reservation.active()
10555                    && reservation.source_pool_index == owner.source_pool_index
10556                    && reservation.operation_id == owner.operation_id
10557                    && reservation.generation == owner.generation
10558            })
10559            .map(|reservation| reservation.model_version)
10560            .ok_or_else(|| decommission_capacity_blocked_error("decommission capacity owner is stale"))?;
10561        match model_version {
10562            DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION => Ok(None),
10563            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION => {
10564                discard_decommission_capacity_target_permit_except(self.id, owner, Some(target_pool_index));
10565                match take_decommission_capacity_target_permit(self.id, target_pool_index, owner) {
10566                    Some(guard) => Ok(Some(guard)),
10567                    None => self
10568                        .acquire_decommission_capacity_target_guard(target_pool_index)
10569                        .await
10570                        .map(Some),
10571                }
10572            }
10573            version => Err(Error::DecommissionCapacity(format!(
10574                "decommission capacity owner uses unsupported lock model {version}"
10575            ))),
10576        }
10577    }
10578
10579    async fn acquire_decommission_capacity_terminal_guards(
10580        &self,
10581        source_pool_index: usize,
10582        plan: Option<&DecommissionCapacityTerminalFencePlan>,
10583    ) -> Result<Vec<rustfs_lock::NamespaceLockGuard>> {
10584        let Some(plan) = plan else {
10585            return Ok(Vec::new());
10586        };
10587        match plan.model_version {
10588            DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION => return Ok(Vec::new()),
10589            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION => {}
10590            version => {
10591                return Err(Error::DecommissionCapacity(format!(
10592                    "decommission terminal transition found unsupported capacity lock model {version}"
10593                )));
10594            }
10595        }
10596        // Terminal transitions hold no object or pool metadata lock here, so
10597        // they can wait in target-index order without forming a lock cycle.
10598        let pool = self.pools.first().cloned().ok_or_else(|| {
10599            Error::InvalidArgument(
10600                "decommission-capacity".to_string(),
10601                "storage-pools".to_string(),
10602                "no storage pools available".to_string(),
10603            )
10604        })?;
10605        // Retry only target acquisition, never persistence. Keep the original
10606        // owner/cohort pinned so a remote Clear/start cannot retarget a cancel.
10607        let mut attempt = 1;
10608        'acquire_targets: loop {
10609            let mut guards = Vec::with_capacity(plan.target_pool_indices.len());
10610            for &target_pool_index in &plan.target_pool_indices {
10611                let object = format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}");
10612                let target_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
10613                let started = std::time::Instant::now();
10614                match target_lock.get_write_lock(get_lock_acquire_timeout()).await {
10615                    Ok(guard) => guards.push(guard),
10616                    Err(err @ rustfs_lock::LockError::Timeout { .. }) => {
10617                        // Release the entire partial cohort before backoff or
10618                        // metadata reads; workers need these gates to settle I/O.
10619                        drop(guards);
10620                        #[cfg(test)]
10621                        pause_decommission_cancel_target_timeout(self.id).await;
10622                        if attempt >= DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS {
10623                            return Err(Error::Lock(err));
10624                        }
10625                        warn!(
10626                            event = EVENT_DECOMMISSION_STATE,
10627                            component = LOG_COMPONENT_ECSTORE,
10628                            subsystem = LOG_SUBSYSTEM_POOLS,
10629                            state = "cancel_target_fence_retry",
10630                            pool_index = source_pool_index,
10631                            target_pool_index,
10632                            operation_id = %plan.operation_id,
10633                            generation = plan.generation,
10634                            owner_nonce = %plan.owner_nonce,
10635                            attempt,
10636                            max_attempts = DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS,
10637                            wait_ms = %started.elapsed().as_millis(),
10638                            error = %err,
10639                            "Decommission cancel will retry target capacity fencing"
10640                        );
10641                        tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await;
10642                        let save_guard = self.pool_meta_save_gate.lock().await;
10643                        let (_read_guard, snapshot) = self
10644                            .acquire_pool_meta_read_guard(&save_guard, "decommission cancel fence retry failed")
10645                            .await?;
10646                        if decommission_capacity_terminal_fence_plan(&snapshot, source_pool_index)?.as_ref() != Some(plan) {
10647                            return Err(decommission_capacity_blocked_error(
10648                                "decommission capacity owner or target cohort changed while retrying terminal fences",
10649                            ));
10650                        }
10651                        attempt += 1;
10652                        continue 'acquire_targets;
10653                    }
10654                    Err(rustfs_lock::LockError::QuorumNotReached { required, achieved }) => {
10655                        return Err(Error::NamespaceLockQuorumUnavailable {
10656                            mode: "write",
10657                            bucket: RUSTFS_META_BUCKET.to_string(),
10658                            object,
10659                            required,
10660                            achieved,
10661                        });
10662                    }
10663                    Err(err) => return Err(Error::Lock(err)),
10664                }
10665            }
10666            return Ok(guards);
10667        }
10668    }
10669
10670    pub(crate) async fn acquire_external_decommission_capacity_fence(
10671        &self,
10672        target_pool_indices: &[usize],
10673        admission: DecommissionCapacityAdmission,
10674    ) -> Result<rustfs_lock::NamespaceLockGuard> {
10675        Ok(self
10676            .acquire_external_decommission_capacity_fence_with_active_source(target_pool_indices, admission)
10677            .await?
10678            .0)
10679    }
10680
10681    pub(crate) async fn acquire_external_decommission_capacity_fence_with_active_source(
10682        &self,
10683        target_pool_indices: &[usize],
10684        admission: DecommissionCapacityAdmission,
10685    ) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
10686        let save_guard = self.pool_meta_save_gate.lock().await;
10687        let (pool_meta_guard, snapshot) = self
10688            .acquire_pool_meta_read_guard(&save_guard, "target capacity admission failed")
10689            .await?;
10690        for target_pool_index in target_pool_indices.iter().copied() {
10691            ensure_external_decommission_target_admission(&snapshot, target_pool_index, admission)?;
10692        }
10693        let has_active_source = pool_meta_has_active_decommission(&snapshot);
10694        drop(save_guard);
10695        Ok((pool_meta_guard, has_active_source))
10696    }
10697
10698    /// Fence healing of the pool metadata object itself without recursively
10699    /// acquiring its namespace lock through the ordinary capacity probe. The
10700    /// caller must retain the returned write guard through every admitted
10701    /// repair; admission results preserve the input target order.
10702    pub(crate) async fn acquire_pool_meta_object_heal_fence(
10703        &self,
10704        target_pool_indices: &[usize],
10705    ) -> Result<(rustfs_lock::NamespaceLockGuard, Vec<Result<()>>)> {
10706        let mut save_guard = self.pool_meta_save_gate.lock().await;
10707        let (pool_meta_guard, snapshot) = self
10708            .acquire_pool_meta_write_guard_with_lock_error(&mut save_guard, "pool metadata heal admission failed", Error::from)
10709            .await?;
10710        let admissions = target_pool_indices
10711            .iter()
10712            .copied()
10713            .map(|target_pool_index| {
10714                ensure_external_decommission_target_admission(&snapshot, target_pool_index, DecommissionCapacityAdmission::Heal)
10715            })
10716            .collect();
10717        drop(save_guard);
10718        Ok((pool_meta_guard, admissions))
10719    }
10720
10721    pub(crate) async fn acquire_decommission_capacity_release_fence_with_active_source(
10722        &self,
10723    ) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
10724        let save_guard = self.pool_meta_save_gate.lock().await;
10725        let (pool_meta_guard, snapshot) = self
10726            .acquire_pool_meta_read_guard(&save_guard, "capacity release fence failed")
10727            .await?;
10728        let has_active_source = pool_meta_has_active_decommission(&snapshot);
10729        drop(save_guard);
10730        Ok((pool_meta_guard, has_active_source))
10731    }
10732
10733    pub(crate) async fn run_decommission_capacity_admitted_mutation<T, F, Fut>(
10734        &self,
10735        target_pool_index: usize,
10736        capacity_owner: Option<DecommissionCapacityOwner>,
10737        expected_data_bytes: Option<usize>,
10738        operation: F,
10739    ) -> Result<T>
10740    where
10741        F: FnOnce() -> Fut,
10742        Fut: std::future::Future<Output = Result<T>>,
10743    {
10744        self.run_decommission_capacity_mutation(
10745            target_pool_index,
10746            capacity_owner,
10747            expected_data_bytes,
10748            DecommissionCapacityMutationMode::Durable,
10749            |_| false,
10750            |_| operation(),
10751        )
10752        .await
10753    }
10754
10755    pub(crate) async fn run_decommission_capacity_admitted_mutation_with_capacity_lease<T, F, Fut>(
10756        &self,
10757        target_pool_index: usize,
10758        capacity_owner: Option<DecommissionCapacityOwner>,
10759        expected_data_bytes: Option<usize>,
10760        operation: F,
10761    ) -> Result<T>
10762    where
10763        F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
10764        Fut: std::future::Future<Output = Result<T>>,
10765    {
10766        self.run_decommission_capacity_mutation(
10767            target_pool_index,
10768            capacity_owner,
10769            expected_data_bytes,
10770            DecommissionCapacityMutationMode::Durable,
10771            |_| false,
10772            operation,
10773        )
10774        .await
10775    }
10776
10777    pub(crate) async fn reconcile_decommission_capacity_before_exact_delete(
10778        &self,
10779        bucket: &str,
10780        object: &str,
10781        opts: &ObjectOptions,
10782        exact: &ObjectInfo,
10783    ) -> Result<()> {
10784        if exact.bucket != bucket || exact.name != decode_dir_object(object) {
10785            return Err(decommission_capacity_blocked_error(
10786                "exact delete object identity changed before capacity reconciliation",
10787            ));
10788        }
10789
10790        let (reconciliations, model_version) = {
10791            let save_guard = self.pool_meta_save_gate.lock().await;
10792            let (_read_guard, snapshot) = self
10793                .acquire_pool_meta_read_guard(&save_guard, "exact delete capacity reconciliation failed")
10794                .await?;
10795            let reconciliations = plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)?;
10796            let model_version = active_decommission_capacity_model(&snapshot)?;
10797            (reconciliations, model_version)
10798        };
10799        if reconciliations.is_empty() {
10800            return Ok(());
10801        }
10802        ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
10803
10804        let mut target_guards = Vec::new();
10805        let mut legacy_write_fence = None;
10806        match model_version {
10807            Some(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION) => {
10808                let mut save_guard = self.pool_meta_save_gate.lock().await;
10809                let (write_guard, snapshot) = self
10810                    .acquire_pool_meta_write_guard(&mut save_guard, "exact delete capacity reconciliation failed")
10811                    .await?;
10812                if active_decommission_capacity_model(&snapshot)? != model_version
10813                    || plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)? != reconciliations
10814                {
10815                    return Err(decommission_capacity_blocked_error(
10816                        "pending capacity changed before exact target evidence could be verified",
10817                    ));
10818                }
10819                legacy_write_fence = Some((save_guard, write_guard, snapshot));
10820            }
10821            Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) => {
10822                let mut target_pool_indices = reconciliations
10823                    .iter()
10824                    .map(|reconciliation| reconciliation.target_pool_index)
10825                    .collect::<Vec<_>>();
10826                target_pool_indices.sort_unstable();
10827                target_pool_indices.dedup();
10828                target_guards.reserve(target_pool_indices.len());
10829                for target_pool_index in target_pool_indices {
10830                    let guard = self.acquire_decommission_capacity_target_guard(target_pool_index).await?;
10831                    ensure_decommission_capacity_target_fence(&guard, target_pool_index, "exact delete evidence")?;
10832                    target_guards.push((target_pool_index, guard));
10833                }
10834            }
10835            Some(version) => {
10836                return Err(Error::DecommissionCapacity(format!(
10837                    "exact delete capacity reconciliation found unsupported lock model {version}"
10838                )));
10839            }
10840            None => {
10841                return Err(decommission_capacity_blocked_error(
10842                    "exact delete capacity reconciliation has no active reservation lock model",
10843                ));
10844            }
10845        }
10846
10847        let target_lookup_options = ObjectOptions {
10848            versioned: opts.versioned,
10849            version_suspended: opts.version_suspended,
10850            version_id: opts.version_id.clone(),
10851            metadata_chg: opts.version_id.is_some(),
10852            no_lock: true,
10853            ..Default::default()
10854        };
10855        for reconciliation in &reconciliations {
10856            let target_pool = self.pools.get(reconciliation.target_pool_index).ok_or_else(|| {
10857                decommission_capacity_blocked_error(format!(
10858                    "source pool {} exact-delete capacity target pool {} is out of range",
10859                    reconciliation.source_pool_index, reconciliation.target_pool_index
10860                ))
10861            })?;
10862            let target = target_pool
10863                .get_object_info(bucket, object, &target_lookup_options)
10864                .await
10865                .map_err(|err| {
10866                    decommission_capacity_blocked_error(format!(
10867                        "source pool {} target pool {} exact object evidence could not be read: {err}",
10868                        reconciliation.source_pool_index, reconciliation.target_pool_index
10869                    ))
10870                })?;
10871            if !Self::is_equivalent_decommission_capacity_target(exact, &target) {
10872                return Err(decommission_capacity_blocked_error(format!(
10873                    "source pool {} target pool {} does not contain an equivalent exact object for its pending capacity intent",
10874                    reconciliation.source_pool_index, reconciliation.target_pool_index
10875                )));
10876            }
10877        }
10878        ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
10879
10880        let (mut save_guard, write_guard, mut snapshot) = if let Some(fence) = legacy_write_fence {
10881            fence
10882        } else {
10883            let mut save_guard = self.pool_meta_save_gate.lock().await;
10884            let (write_guard, snapshot) = self
10885                .acquire_pool_meta_write_guard(&mut save_guard, "exact delete capacity reconciliation failed")
10886                .await?;
10887            (save_guard, write_guard, snapshot)
10888        };
10889        if active_decommission_capacity_model(&snapshot)? != model_version {
10890            return Err(decommission_capacity_blocked_error(
10891                "capacity lock model changed while exact target evidence was being verified",
10892            ));
10893        }
10894        let current_reconciliations = plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)?;
10895        if current_reconciliations != reconciliations {
10896            return Err(decommission_capacity_blocked_error(
10897                "pending capacity changed while exact target evidence was being verified",
10898            ));
10899        }
10900        ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
10901        for (target_pool_index, target_guard) in &target_guards {
10902            ensure_decommission_capacity_target_fence(target_guard, *target_pool_index, "exact delete capacity finalize")?;
10903        }
10904
10905        let now = OffsetDateTime::now_utc();
10906        let mut source_pool_indices = Vec::with_capacity(current_reconciliations.len());
10907        for reconciliation in current_reconciliations {
10908            resolve_decommission_target_pending(
10909                &mut snapshot,
10910                reconciliation.source_pool_index,
10911                reconciliation.target_pool_index,
10912                reconciliation.expected_target_physical_bytes,
10913                reconciliation.mutation_id,
10914            )?;
10915            record_decommission_target_consumption(
10916                &mut snapshot,
10917                reconciliation.source_pool_index,
10918                reconciliation.target_pool_index,
10919                DecommissionTargetConsumption {
10920                    committed_data_bytes: reconciliation.expected_data_bytes,
10921                    target_physical_bytes: reconciliation.expected_target_physical_bytes,
10922                    observed_physical_bytes: 0,
10923                },
10924                reconciliation.mutation_id,
10925                now,
10926            )?;
10927            source_pool_indices.push(reconciliation.source_pool_index);
10928        }
10929        source_pool_indices.sort_unstable();
10930        source_pool_indices.dedup();
10931        ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
10932
10933        let outcome = snapshot
10934            .save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &source_pool_indices)
10935            .await?;
10936        ensure_pool_meta_write_fence(&write_guard, "exact delete capacity reconciliation save failed")?;
10937        for (target_pool_index, target_guard) in &target_guards {
10938            ensure_decommission_capacity_target_fence(target_guard, *target_pool_index, "exact delete capacity save")?;
10939        }
10940        {
10941            let mut pool_meta = self.pool_meta.write().await;
10942            for source_pool_index in source_pool_indices {
10943                publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, source_pool_index)?;
10944            }
10945        }
10946        ensure_pool_meta_write_fence(&write_guard, "exact delete capacity reconciliation save failed")?;
10947        for (target_pool_index, target_guard) in &target_guards {
10948            ensure_decommission_capacity_target_fence(target_guard, *target_pool_index, "exact delete capacity publication")?;
10949        }
10950        outcome.disarm();
10951        Ok(())
10952    }
10953
10954    pub(crate) async fn reconcile_decommission_capacity_after_equivalent_target(
10955        &self,
10956        owner: DecommissionCapacityOwner,
10957        target_pool_index: usize,
10958        expected_data_bytes: usize,
10959    ) -> Result<()> {
10960        let target_guard = self
10961            .acquire_decommission_capacity_owner_target_guard(owner, target_pool_index)
10962            .await?;
10963        if let Some(target_guard) = target_guard.as_ref() {
10964            ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target reconciliation")?;
10965        }
10966        let mut save_guard = self.pool_meta_save_gate.lock().await;
10967        let (pool_meta_guard, mut snapshot) = self
10968            .acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent target reconciliation failed")
10969            .await?;
10970        let source_pool_index = owner.source_pool_index;
10971        let (target_layout, target_pending_physical_bytes, target_consumed_physical_bytes) = {
10972            let reservation = snapshot
10973                .pools
10974                .get(source_pool_index)
10975                .and_then(|pool| pool.decommission.as_ref())
10976                .and_then(|info| info.capacity_reservation.as_ref())
10977                .filter(|reservation| {
10978                    reservation.active()
10979                        && reservation.source_pool_index == owner.source_pool_index
10980                        && reservation.operation_id == owner.operation_id
10981                        && reservation.generation == owner.generation
10982                        && reservation.owner_nonce == owner.owner_nonce
10983                })
10984                .ok_or_else(|| decommission_capacity_blocked_error("equivalent target reconciliation owner is stale"))?;
10985            let target = reservation
10986                .targets
10987                .iter()
10988                .find(|target| target.pool_index == target_pool_index)
10989                .ok_or_else(|| decommission_capacity_blocked_error("equivalent target reconciliation allocation is missing"))?;
10990            (target.layout, target.pending_physical_bytes, target.consumed_physical_bytes)
10991        };
10992        let mutation_id = owner.mutation_id.ok_or_else(|| {
10993            decommission_capacity_blocked_error("equivalent target reconciliation mutation identity is missing")
10994        })?;
10995        let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
10996        if target_pending_physical_bytes == 0 {
10997            if target_consumed_physical_bytes >= expected_target_physical_bytes {
10998                let mut pool_meta = self.pool_meta.write().await;
10999                ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent target idempotent publication")?;
11000                publish_decommission_capacity_update(&mut pool_meta, &snapshot, source_pool_index)?;
11001                if let Some(target_guard) = target_guard.as_ref() {
11002                    ensure_decommission_capacity_target_fence(
11003                        target_guard,
11004                        target_pool_index,
11005                        "equivalent target idempotent publication",
11006                    )?;
11007                }
11008                return Ok(());
11009            }
11010            return Err(decommission_capacity_blocked_error(
11011                "equivalent target has no pending capacity intent to reconcile",
11012            ));
11013        }
11014        if target_pending_physical_bytes < expected_target_physical_bytes {
11015            return Err(decommission_capacity_blocked_error(
11016                "equivalent target pending capacity is smaller than the committed object",
11017            ));
11018        }
11019
11020        resolve_decommission_target_pending(
11021            &mut snapshot,
11022            source_pool_index,
11023            target_pool_index,
11024            expected_target_physical_bytes,
11025            mutation_id,
11026        )?;
11027        record_decommission_target_consumption(
11028            &mut snapshot,
11029            source_pool_index,
11030            target_pool_index,
11031            DecommissionTargetConsumption {
11032                committed_data_bytes: expected_data_bytes,
11033                target_physical_bytes: expected_target_physical_bytes,
11034                observed_physical_bytes: 0,
11035            },
11036            mutation_id,
11037            OffsetDateTime::now_utc(),
11038        )?;
11039        let outcome = snapshot
11040            .save_no_lock_armed(
11041                self.pools.clone(),
11042                &mut save_guard,
11043                pool_meta_guard.lock_lost_signal(),
11044                &[source_pool_index],
11045            )
11046            .await?;
11047        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission equivalent target reconciliation save failed")?;
11048        if let Some(target_guard) = target_guard.as_ref() {
11049            ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target capacity save")?;
11050        }
11051        {
11052            let mut pool_meta = self.pool_meta.write().await;
11053            publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, source_pool_index)?;
11054        }
11055        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission equivalent target reconciliation save failed")?;
11056        if let Some(target_guard) = target_guard.as_ref() {
11057            ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target capacity publication")?;
11058        }
11059        outcome.disarm();
11060        Ok(())
11061    }
11062
11063    pub(crate) async fn run_decommission_capacity_temporary_mutation<T, F, Fut>(
11064        &self,
11065        target_pool_index: usize,
11066        capacity_owner: Option<DecommissionCapacityOwner>,
11067        expected_data_bytes: Option<usize>,
11068        operation: F,
11069    ) -> Result<T>
11070    where
11071        F: FnOnce() -> Fut,
11072        Fut: std::future::Future<Output = Result<T>>,
11073    {
11074        self.run_decommission_capacity_mutation(
11075            target_pool_index,
11076            capacity_owner,
11077            expected_data_bytes,
11078            DecommissionCapacityMutationMode::Temporary,
11079            |_| false,
11080            |_| operation(),
11081        )
11082        .await
11083    }
11084
11085    pub(crate) async fn run_decommission_capacity_temporary_mutation_with_capacity_lease<T, F, Fut>(
11086        &self,
11087        target_pool_index: usize,
11088        capacity_owner: Option<DecommissionCapacityOwner>,
11089        expected_data_bytes: Option<usize>,
11090        operation: F,
11091    ) -> Result<T>
11092    where
11093        F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11094        Fut: std::future::Future<Output = Result<T>>,
11095    {
11096        self.run_decommission_capacity_mutation(
11097            target_pool_index,
11098            capacity_owner,
11099            expected_data_bytes,
11100            DecommissionCapacityMutationMode::Temporary,
11101            |_| false,
11102            operation,
11103        )
11104        .await
11105    }
11106
11107    /// Run an identity-preserving replacement that the caller has already
11108    /// proven cannot grow the target object. A failed or ambiguous write keeps
11109    /// the ordinary temporary-mutation recovery marker, while a successful
11110    /// write resolves the capacity intent without retaining MPU cleanup state.
11111    pub(crate) async fn run_decommission_capacity_non_growing_replacement_with_capacity_lease<T, F, Fut>(
11112        &self,
11113        target_pool_index: usize,
11114        capacity_owner: Option<DecommissionCapacityOwner>,
11115        expected_data_bytes: Option<usize>,
11116        operation: F,
11117    ) -> Result<T>
11118    where
11119        F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11120        Fut: std::future::Future<Output = Result<T>>,
11121    {
11122        self.run_decommission_capacity_mutation(
11123            target_pool_index,
11124            capacity_owner,
11125            expected_data_bytes,
11126            DecommissionCapacityMutationMode::NonGrowingReplacement,
11127            |_| false,
11128            operation,
11129        )
11130        .await
11131    }
11132
11133    /// Finish the capacity transaction for an identity-preserving temporary
11134    /// replacement whose target bytes are already durably present. This is
11135    /// the crash-recovery half of a non-growing replacement: it never writes
11136    /// the target again and only settles state owned by the exact deterministic
11137    /// mutation id.
11138    pub(crate) async fn reconcile_decommission_capacity_after_equivalent_temporary_target(
11139        &self,
11140        owner: DecommissionCapacityOwner,
11141        target_pool_index: usize,
11142        expected_data_bytes: usize,
11143    ) -> Result<()> {
11144        let target_guard = self
11145            .acquire_decommission_capacity_owner_target_guard(owner, target_pool_index)
11146            .await?;
11147        if let Some(target_guard) = target_guard.as_ref() {
11148            ensure_decommission_capacity_target_fence(
11149                target_guard,
11150                target_pool_index,
11151                "equivalent temporary target reconciliation",
11152            )?;
11153        }
11154        let mut save_guard = self.pool_meta_save_gate.lock().await;
11155        let (pool_meta_guard, mut snapshot) = self
11156            .acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent temporary target reconciliation failed")
11157            .await?;
11158        let source_pool_index = owner.source_pool_index;
11159        let mutation_id = owner
11160            .mutation_id
11161            .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target mutation identity is missing"))?;
11162        let (target_layout, pending_physical_bytes, pending_mutation_id, has_temporary_mutation) = {
11163            let reservation = snapshot
11164                .pools
11165                .get(source_pool_index)
11166                .and_then(|pool| pool.decommission.as_ref())
11167                .and_then(|info| info.capacity_reservation.as_ref())
11168                .filter(|reservation| reservation.admits_cleanup_owner(owner))
11169                .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target owner is stale"))?;
11170            let target = reservation
11171                .targets
11172                .iter()
11173                .find(|target| target.pool_index == target_pool_index)
11174                .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target allocation is missing"))?;
11175            (
11176                target.layout,
11177                target.pending_physical_bytes,
11178                target.pending_mutation_id,
11179                target
11180                    .temporary_mutations
11181                    .iter()
11182                    .any(|mutation| mutation.mutation_id == mutation_id),
11183            )
11184        };
11185        if pending_physical_bytes == 0 && !has_temporary_mutation {
11186            // A prior successful attempt already settled both durable halves.
11187            ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation fence failed")?;
11188            if let Some(target_guard) = target_guard.as_ref() {
11189                ensure_decommission_capacity_target_fence(
11190                    target_guard,
11191                    target_pool_index,
11192                    "equivalent temporary target idempotent reconciliation",
11193                )?;
11194            }
11195            return Ok(());
11196        }
11197        if pending_physical_bytes > 0 && pending_mutation_id != Some(mutation_id) {
11198            return Err(decommission_capacity_blocked_error(
11199                "equivalent temporary target pending intent belongs to another mutation",
11200            ));
11201        }
11202        let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
11203        if pending_physical_bytes > 0 && pending_physical_bytes != expected_target_physical_bytes {
11204            return Err(decommission_capacity_blocked_error(
11205                "equivalent temporary target pending capacity does not match the committed checkpoint",
11206            ));
11207        }
11208        if pending_physical_bytes > 0 {
11209            resolve_decommission_target_pending(
11210                &mut snapshot,
11211                source_pool_index,
11212                target_pool_index,
11213                expected_target_physical_bytes,
11214                mutation_id,
11215            )?;
11216        }
11217        // Exact byte equivalence proves that this identity-preserving,
11218        // byte-non-growing replacement committed. Settle both crash windows:
11219        // the pending intent before finalize and the temporary marker written
11220        // when the operation returned an error after committing its target.
11221        settle_decommission_target_non_growing_replacement(
11222            &mut snapshot,
11223            source_pool_index,
11224            target_pool_index,
11225            mutation_id,
11226            OffsetDateTime::now_utc(),
11227        )?;
11228        let outcome = snapshot
11229            .save_no_lock_armed(
11230                self.pools.clone(),
11231                &mut save_guard,
11232                pool_meta_guard.lock_lost_signal(),
11233                &[source_pool_index],
11234            )
11235            .await?;
11236        ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?;
11237        if let Some(target_guard) = target_guard.as_ref() {
11238            ensure_decommission_capacity_target_fence(
11239                target_guard,
11240                target_pool_index,
11241                "equivalent temporary target capacity save",
11242            )?;
11243        }
11244        {
11245            let mut pool_meta = self.pool_meta.write().await;
11246            publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, source_pool_index)?;
11247        }
11248        ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?;
11249        if let Some(target_guard) = target_guard.as_ref() {
11250            ensure_decommission_capacity_target_fence(
11251                target_guard,
11252                target_pool_index,
11253                "equivalent temporary target capacity publication",
11254            )?;
11255        }
11256        outcome.disarm();
11257        Ok(())
11258    }
11259
11260    pub(crate) async fn has_decommission_capacity_temporary_mutation_state(
11261        &self,
11262        target_pool_index: usize,
11263        owner: DecommissionCapacityOwner,
11264    ) -> bool {
11265        let Some(mutation_id) = owner.mutation_id else {
11266            return false;
11267        };
11268        self.pool_meta
11269            .read()
11270            .await
11271            .pools
11272            .get(owner.source_pool_index)
11273            .and_then(|pool| pool.decommission.as_ref())
11274            .and_then(|info| info.capacity_reservation.as_ref())
11275            .filter(|reservation| reservation.admits_cleanup_owner(owner))
11276            .and_then(|reservation| {
11277                reservation
11278                    .targets
11279                    .iter()
11280                    .find(|target| target.pool_index == target_pool_index)
11281                    .map(|target| (reservation.model_version, target))
11282            })
11283            .is_some_and(|(model_version, target)| {
11284                if model_version == DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION {
11285                    // V1 cannot persist an exact zero-byte staging marker, so
11286                    // retain its compatibility scan for admitted cleanup targets.
11287                    true
11288                } else {
11289                    (target.pending_physical_bytes > 0 && target.pending_mutation_id == Some(mutation_id))
11290                        || target
11291                            .temporary_mutations
11292                            .iter()
11293                            .any(|mutation| mutation.mutation_id == mutation_id)
11294                }
11295            })
11296    }
11297
11298    pub(crate) async fn decommission_capacity_cleanup_target_indices(
11299        &self,
11300        owner: DecommissionCapacityOwner,
11301    ) -> Result<Vec<usize>> {
11302        self.pool_meta
11303            .read()
11304            .await
11305            .pools
11306            .get(owner.source_pool_index)
11307            .and_then(|pool| pool.decommission.as_ref())
11308            .and_then(|info| info.capacity_reservation.as_ref())
11309            .filter(|reservation| reservation.admits_cleanup_owner(owner))
11310            .map(|reservation| reservation.targets.iter().map(|target| target.pool_index).collect())
11311            .ok_or_else(|| decommission_capacity_blocked_error("decommission multipart cleanup reservation is stale"))
11312    }
11313
11314    pub(crate) async fn run_decommission_capacity_temporary_release_with_capacity_lease<T, F, Fut>(
11315        &self,
11316        target_pool_index: usize,
11317        capacity_owner: Option<DecommissionCapacityOwner>,
11318        operation: F,
11319    ) -> Result<T>
11320    where
11321        F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11322        Fut: std::future::Future<Output = Result<(T, bool)>>,
11323    {
11324        self.run_decommission_capacity_mutation(
11325            target_pool_index,
11326            capacity_owner,
11327            None,
11328            DecommissionCapacityMutationMode::TemporaryRelease,
11329            |result: &(T, bool)| result.1,
11330            operation,
11331        )
11332        .await
11333        .map(|(result, _)| result)
11334    }
11335
11336    async fn run_decommission_capacity_mutation<T, F, Fut, P>(
11337        &self,
11338        target_pool_index: usize,
11339        capacity_owner: Option<DecommissionCapacityOwner>,
11340        expected_data_bytes: Option<usize>,
11341        mode: DecommissionCapacityMutationMode,
11342        clear_pending_on_temporary_release: P,
11343        operation: F,
11344    ) -> Result<T>
11345    where
11346        F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11347        Fut: std::future::Future<Output = Result<T>>,
11348        P: Fn(&T) -> bool,
11349    {
11350        let temporary = matches!(
11351            mode,
11352            DecommissionCapacityMutationMode::Temporary | DecommissionCapacityMutationMode::NonGrowingReplacement
11353        );
11354        let non_growing_replacement = matches!(mode, DecommissionCapacityMutationMode::NonGrowingReplacement);
11355        let temporary_release = matches!(mode, DecommissionCapacityMutationMode::TemporaryRelease);
11356        let mut operation = Some(operation);
11357        let save_guard = self.pool_meta_save_gate.lock().await;
11358        let (read_guard, snapshot) = self
11359            .acquire_pool_meta_read_guard(&save_guard, "target capacity admission failed")
11360            .await?;
11361        let admission_now = OffsetDateTime::now_utc();
11362        let admitted_owner = capacity_owner.and_then(|owner| {
11363            snapshot
11364                .pools
11365                .get(owner.source_pool_index)
11366                .and_then(|pool| pool.decommission.as_ref())
11367                .and_then(|info| info.capacity_reservation.as_ref())
11368                .filter(|reservation| {
11369                    if temporary_release {
11370                        reservation.admits_cleanup_owner(owner)
11371                    } else {
11372                        reservation.admits_owner(owner, admission_now)
11373                    }
11374                })
11375                .filter(|reservation| {
11376                    reservation
11377                        .targets
11378                        .iter()
11379                        .any(|target| target.pool_index == target_pool_index)
11380                })
11381                .map(|reservation| (owner, reservation.model_version))
11382        });
11383        if capacity_owner.is_some() && admitted_owner.is_none() {
11384            return Err(decommission_capacity_blocked_error(
11385                "decommission target mutation reservation identity is stale",
11386            ));
11387        }
11388        let Some((owner, model_version)) = admitted_owner else {
11389            ensure_external_decommission_target_admission(&snapshot, target_pool_index, DecommissionCapacityAdmission::Mutation)?;
11390            drop(save_guard);
11391            let capacity_lease = read_guard.lock_lost_signal();
11392            return operation.take().expect("capacity-admitted operation should run once")(capacity_lease).await;
11393        };
11394
11395        // Per-target reservations always acquire target -> pool metadata.
11396        // Legacy reservations keep the pool metadata write guard through I/O.
11397        drop(read_guard);
11398        drop(save_guard);
11399        if model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION {
11400            discard_decommission_capacity_target_permit_except(self.id, owner, Some(target_pool_index));
11401        }
11402        let target_guard = match model_version {
11403            DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION => None,
11404            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION => {
11405                Some(match take_decommission_capacity_target_permit(self.id, target_pool_index, owner) {
11406                    Some(guard) => guard,
11407                    None => self.acquire_decommission_capacity_target_guard(target_pool_index).await?,
11408                })
11409            }
11410            version => {
11411                return Err(Error::DecommissionCapacity(format!(
11412                    "decommission target mutation found unsupported capacity lock model {version}"
11413                )));
11414            }
11415        };
11416
11417        #[cfg(test)]
11418        pause_decommission_capacity_before_owner_write(self.id).await;
11419        let mut save_guard = self.pool_meta_save_gate.lock().await;
11420        let (write_guard, mut snapshot) = self
11421            .acquire_pool_meta_write_guard(&mut save_guard, "decommission target capacity admission failed")
11422            .await?;
11423        let mutation_id = owner
11424            .mutation_id
11425            .ok_or_else(|| decommission_capacity_blocked_error("decommission mutation identity is missing"))?;
11426        let source_pool_index = owner.source_pool_index;
11427        let owner_current = snapshot
11428            .pools
11429            .get(source_pool_index)
11430            .and_then(|pool| pool.decommission.as_ref())
11431            .and_then(|info| info.capacity_reservation.as_ref())
11432            .filter(|reservation| {
11433                (if temporary_release {
11434                    reservation.admits_cleanup_owner(owner)
11435                } else {
11436                    reservation.admits_owner(owner, OffsetDateTime::now_utc())
11437                }) && reservation.model_version == model_version
11438                    && reservation
11439                        .targets
11440                        .iter()
11441                        .any(|target| target.pool_index == target_pool_index)
11442            })
11443            .is_some();
11444        if !owner_current {
11445            return Err(decommission_capacity_blocked_error(
11446                "decommission target mutation reservation identity changed before commit",
11447            ));
11448        }
11449        let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
11450        if !temporary_release {
11451            ensure_decommission_capacity_reservations_available(&snapshot, &capacity_infos, "mutation")?;
11452        }
11453        let target_layout = snapshot
11454            .pools
11455            .get(source_pool_index)
11456            .and_then(|pool| pool.decommission.as_ref())
11457            .and_then(|info| info.capacity_reservation.as_ref())
11458            .and_then(|reservation| {
11459                reservation
11460                    .targets
11461                    .iter()
11462                    .find(|target| target.pool_index == target_pool_index)
11463            })
11464            .map(|target| target.layout)
11465            .ok_or_else(|| Error::SlowDown)?;
11466        let expected_target_physical_bytes = if temporary_release {
11467            0
11468        } else {
11469            capacity_target_physical_bytes(expected_data_bytes.unwrap_or(1).max(1), target_layout)?
11470        };
11471        if temporary {
11472            let (remaining, inflight) = snapshot
11473                .pools
11474                .get(source_pool_index)
11475                .and_then(|pool| pool.decommission.as_ref())
11476                .and_then(|info| info.capacity_reservation.as_ref())
11477                .and_then(|reservation| {
11478                    reservation
11479                        .targets
11480                        .iter()
11481                        .find(|target| target.pool_index == target_pool_index)
11482                        .map(|target| {
11483                            (
11484                                target.remaining_reserved_physical_bytes(reservation.temporary_copies)
11485                                    / 1usize.saturating_add(reservation.temporary_copies),
11486                                target.inflight_physical_bytes,
11487                            )
11488                        })
11489                })
11490                .unwrap_or_default();
11491            let available = remaining.saturating_sub(inflight);
11492            if expected_target_physical_bytes > available {
11493                return Err(decommission_capacity_blocked_error(format!(
11494                    "source pool {source_pool_index} target pool {target_pool_index} temporary operation requires {expected_target_physical_bytes} physical bytes, but only {available} temporary-copy bytes remain"
11495                )));
11496            }
11497        } else {
11498            ensure_decommission_target_owner_admission(
11499                &snapshot,
11500                owner,
11501                target_pool_index,
11502                expected_target_physical_bytes,
11503                OffsetDateTime::now_utc(),
11504            )?;
11505        }
11506        let pending_added = if temporary_release {
11507            0
11508        } else {
11509            reserve_decommission_target_pending(
11510                &mut snapshot,
11511                source_pool_index,
11512                target_pool_index,
11513                expected_target_physical_bytes,
11514                mutation_id,
11515                OffsetDateTime::now_utc(),
11516            )?
11517        };
11518        if pending_added > 0 {
11519            let outcome = snapshot
11520                .save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &[source_pool_index])
11521                .await?;
11522            ensure_pool_meta_write_fence(&write_guard, "decommission target capacity intent save failed")?;
11523            snapshot = outcome.committed.clone();
11524            {
11525                let mut pool_meta = self.pool_meta.write().await;
11526                publish_decommission_capacity_update(&mut pool_meta, &snapshot, source_pool_index)?;
11527            }
11528            ensure_pool_meta_write_fence(&write_guard, "decommission target capacity intent save failed")?;
11529            outcome.disarm();
11530        }
11531        let mut write_guard = Some(write_guard);
11532        let mut save_guard = Some(save_guard);
11533        let capacity_lease = if let Some(target_guard) = target_guard.as_ref() {
11534            ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity intent prepare")?;
11535            let capacity_lease = target_guard.lock_lost_signal();
11536            drop(write_guard.take());
11537            drop(save_guard.take());
11538            capacity_lease
11539        } else {
11540            write_guard
11541                .as_ref()
11542                .ok_or_else(|| Error::other("legacy decommission capacity fence disappeared before target mutation"))?
11543                .lock_lost_signal()
11544        };
11545
11546        let capacity_infos = if pending_added > 0 {
11547            self.get_decommission_all_pool_capacity_infos().await?
11548        } else {
11549            capacity_infos
11550        };
11551        let before_free = capacity_infos
11552            .iter()
11553            .find(|capacity| capacity.pool_index == target_pool_index)
11554            .map(|capacity| capacity.physical_free)
11555            .ok_or_else(|| decommission_capacity_blocked_error("target capacity snapshot is missing before mutation"))?;
11556
11557        let result = operation.take().expect("capacity-admitted operation should run once")(capacity_lease).await;
11558        let clear_pending_on_temporary_release = result.as_ref().ok().is_some_and(&clear_pending_on_temporary_release);
11559        if let Some(target_guard) = target_guard.as_ref() {
11560            ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "target mutation")?;
11561            let mut finalize_save_guard = self.pool_meta_save_gate.lock().await;
11562            let (finalize_write_guard, finalize_snapshot) = self
11563                .acquire_pool_meta_write_guard(&mut finalize_save_guard, "decommission target capacity finalize failed")
11564                .await?;
11565            ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity finalize")?;
11566            save_guard = Some(finalize_save_guard);
11567            write_guard = Some(finalize_write_guard);
11568            snapshot = finalize_snapshot;
11569        }
11570        let write_guard = write_guard
11571            .as_ref()
11572            .ok_or_else(|| Error::other("decommission capacity write fence disappeared before finalize"))?;
11573        let save_guard = save_guard
11574            .as_mut()
11575            .ok_or_else(|| Error::other("decommission capacity save fence disappeared before finalize"))?;
11576        ensure_pool_meta_write_fence(write_guard, "decommission target capacity finalize failed")?;
11577        ensure_decommission_capacity_mutation_intent_current(
11578            &snapshot,
11579            owner,
11580            target_pool_index,
11581            expected_target_physical_bytes,
11582            mutation_id,
11583            temporary_release,
11584            model_version,
11585        )?;
11586        let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
11587        let after_free = capacity_infos
11588            .iter()
11589            .find(|capacity| capacity.pool_index == target_pool_index)
11590            .map(|capacity| capacity.physical_free)
11591            .ok_or_else(|| decommission_capacity_blocked_error("target capacity snapshot is missing after mutation"))?;
11592        let observed_physical_bytes = before_free.saturating_sub(after_free);
11593        let released_physical_bytes = after_free.saturating_sub(before_free);
11594        let now = OffsetDateTime::now_utc();
11595        let progress_changed = if temporary_release {
11596            if result.is_ok() {
11597                release_decommission_target_inflight(
11598                    &mut snapshot,
11599                    source_pool_index,
11600                    target_pool_index,
11601                    released_physical_bytes,
11602                    mutation_id,
11603                    DecommissionCapacityReleaseProof::confirmed_absence(clear_pending_on_temporary_release),
11604                    now,
11605                )?
11606            } else {
11607                false
11608            }
11609        } else if result.is_ok() {
11610            resolve_decommission_target_pending(
11611                &mut snapshot,
11612                source_pool_index,
11613                target_pool_index,
11614                expected_target_physical_bytes,
11615                mutation_id,
11616            )?;
11617            if temporary {
11618                if non_growing_replacement {
11619                    settle_decommission_target_non_growing_replacement(
11620                        &mut snapshot,
11621                        source_pool_index,
11622                        target_pool_index,
11623                        mutation_id,
11624                        now,
11625                    )?;
11626                } else {
11627                    record_decommission_target_inflight(
11628                        &mut snapshot,
11629                        source_pool_index,
11630                        target_pool_index,
11631                        observed_physical_bytes,
11632                        mutation_id,
11633                        now,
11634                    )?;
11635                }
11636            } else {
11637                let consumed_physical_bytes = expected_data_bytes
11638                    .map(|_| expected_target_physical_bytes)
11639                    .unwrap_or(observed_physical_bytes.max(expected_target_physical_bytes));
11640                let committed_data_bytes =
11641                    expected_data_bytes.unwrap_or(capacity_source_data_equivalent(observed_physical_bytes, target_layout)?);
11642                record_decommission_target_consumption(
11643                    &mut snapshot,
11644                    source_pool_index,
11645                    target_pool_index,
11646                    DecommissionTargetConsumption {
11647                        committed_data_bytes,
11648                        target_physical_bytes: consumed_physical_bytes,
11649                        observed_physical_bytes,
11650                    },
11651                    mutation_id,
11652                    now,
11653                )?;
11654            }
11655            true
11656        } else if temporary {
11657            record_decommission_target_inflight(
11658                &mut snapshot,
11659                source_pool_index,
11660                target_pool_index,
11661                observed_physical_bytes,
11662                mutation_id,
11663                now,
11664            )?
11665        } else {
11666            record_decommission_target_observation(
11667                &mut snapshot,
11668                source_pool_index,
11669                target_pool_index,
11670                observed_physical_bytes,
11671                now,
11672            )?;
11673            observed_physical_bytes > 0
11674        };
11675        let capacity_result = if temporary_release {
11676            Ok(())
11677        } else {
11678            ensure_decommission_capacity_reservations_available(&snapshot, &capacity_infos, "mutation")
11679        };
11680        if let Err(err) = &capacity_result {
11681            snapshot.mark_decommission_capacity_blocked(source_pool_index, err.to_string(), now)?;
11682        }
11683        if temporary_release && !progress_changed {
11684            ensure_pool_meta_write_fence(write_guard, "decommission target capacity cleanup fence failed")?;
11685            if let Some(target_guard) = target_guard.as_ref() {
11686                ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity cleanup")?;
11687            }
11688            capacity_result?;
11689            return result;
11690        }
11691        let outcome = snapshot
11692            .save_no_lock_armed(self.pools.clone(), save_guard, write_guard.lock_lost_signal(), &[source_pool_index])
11693            .await?;
11694        ensure_pool_meta_write_fence(write_guard, "decommission target capacity progress save failed")?;
11695        if let Some(target_guard) = target_guard.as_ref() {
11696            ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity progress save")?;
11697        }
11698        {
11699            let mut pool_meta = self.pool_meta.write().await;
11700            publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, source_pool_index)?;
11701        }
11702        ensure_pool_meta_write_fence(write_guard, "decommission target capacity progress save failed")?;
11703        if let Some(target_guard) = target_guard.as_ref() {
11704            ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity progress publication")?;
11705        }
11706        outcome.disarm();
11707        capacity_result?;
11708        result
11709    }
11710
11711    pub(crate) async fn ensure_pool_meta_side_effects_safe(&self, operation: &str) -> Result<()> {
11712        self.pool_meta_save_gate.lock().await.ensure_write_safe(operation)
11713    }
11714
11715    /// Reports whether pool metadata side effects are currently writable.
11716    /// Read-only admission probes do not change this state; startup and real
11717    /// metadata transactions still latch it on unrecoverable conditions.
11718    pub async fn pool_meta_writes_ready(&self) -> bool {
11719        self.pool_meta_write_status().await.is_ok()
11720    }
11721
11722    /// A health probe must not wait behind a metadata transaction's disk I/O.
11723    pub async fn pool_meta_write_gate_status(&self) -> PoolMetaWriteGateStatus {
11724        let Ok(write_state) = tokio::time::timeout(std::time::Duration::from_millis(100), self.pool_meta_save_gate.lock()).await
11725        else {
11726            return PoolMetaWriteGateStatus {
11727                writes_ready: false,
11728                check_timed_out: true,
11729                ..PoolMetaWriteGateStatus::default()
11730            };
11731        };
11732        let transaction_aborted = write_state.aborted_transaction.load(Ordering::SeqCst);
11733        let writes_ready = !write_state.write_blocked && !transaction_aborted;
11734        let failure = if writes_ready {
11735            None
11736        } else {
11737            write_state.ensure_write_safe("pool metadata snapshot").err()
11738        };
11739        let context = failure.as_ref().and_then(Error::pool_metadata_failure);
11740        PoolMetaWriteGateStatus {
11741            writes_ready,
11742            check_timed_out: false,
11743            write_blocked: write_state.write_blocked,
11744            transaction_aborted,
11745            pool_meta_absent: write_state.pool_meta_absent,
11746            identity_initialized: write_state.identity_initialized,
11747            identity_needs_repair: write_state.identity_needs_repair,
11748            cluster_epoch: write_state.cluster_epoch,
11749            reason: context.map(|context| context.kind.as_str()),
11750            phase: context.map(|context| context.phase),
11751            since_unix_secs: context.map(|context| context.since.unix_timestamp()),
11752        }
11753    }
11754
11755    /// A health probe must not wait behind a metadata transaction's disk I/O.
11756    pub async fn pool_meta_write_status(&self) -> Result<()> {
11757        let write_state = tokio::time::timeout(std::time::Duration::from_millis(100), self.pool_meta_save_gate.lock())
11758            .await
11759            .map_err(|_| Error::Timeout)?;
11760        write_state.ensure_write_safe("pool metadata readiness")
11761    }
11762
11763    pub(crate) async fn recover_pool_meta_transaction(&self) -> Result<bool> {
11764        {
11765            let Ok(state) = self.pool_meta_save_gate.try_lock() else {
11766                return Ok(false);
11767            };
11768            if state.ensure_write_safe("pool metadata recovery").is_ok() {
11769                return Ok(false);
11770            }
11771            if state.write_blocked
11772                || state.active_transactions.load(Ordering::SeqCst) != 0
11773                || state
11774                    .transaction_failure
11775                    .lock()
11776                    .unwrap_or_else(std::sync::PoisonError::into_inner)
11777                    .is_none()
11778            {
11779                return Ok(false);
11780            }
11781        }
11782        let _start_guard = self.start_gate.lock().await;
11783        // Cancellation must precede the movement writer. Let the existing
11784        // supervisor join every worker and relinquish its operation ownership;
11785        // a new worker must never overlap an old terminal/progress callback.
11786        {
11787            let cancelers = self.decommission_cancelers.read().await;
11788            let mut draining = false;
11789            for canceler in cancelers.iter().flatten().filter(|canceler| canceler.is_active()) {
11790                canceler.cancel();
11791                draining = true;
11792            }
11793            if draining {
11794                return Err(Error::OperationCanceled);
11795            }
11796        }
11797        let movement_gate = self.ctx.data_movement_operation_gate();
11798        let _movement_guard = movement_gate.write().await;
11799        let mut state = self.pool_meta_save_gate.lock().await;
11800        // Outcomes can outlive the save-gate guard. Do not let an old Drop
11801        // relatch, or an older recovery clear a newly installed block.
11802        if state.write_blocked || state.active_transactions.load(Ordering::SeqCst) != 0 {
11803            return Ok(false);
11804        }
11805        let Some(blocked) = state
11806            .transaction_failure
11807            .lock()
11808            .unwrap_or_else(std::sync::PoisonError::into_inner)
11809            .clone()
11810        else {
11811            return Ok(false);
11812        };
11813        if self
11814            .rebalance_meta
11815            .read()
11816            .await
11817            .as_ref()
11818            .is_some_and(|meta| meta.cancel.is_some())
11819        {
11820            return Err(Error::other("pool metadata recovery requires rebalance workers to be quiescent"));
11821        }
11822        let pool = self
11823            .pools
11824            .first()
11825            .ok_or_else(|| Error::other("pool metadata recovery has no storage pools"))?;
11826        let lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
11827        let guard = lock.get_write_lock(get_lock_acquire_timeout()).await?;
11828        let fence = PoolMetaPersistenceFence::Distributed(guard.lock_lost_signal());
11829        let mut candidate = PoolMetaWriteState {
11830            aborted_transaction: Arc::new(AtomicBool::new(false)),
11831            transaction_failure: Arc::default(),
11832            active_transactions: Arc::default(),
11833            block_context: None,
11834            recovery_failure: None,
11835            write_blocked: false,
11836            ..state.clone()
11837        };
11838        let selection = load_pool_meta_for_transaction_recovery(self.pools.clone(), &mut candidate).await?;
11839        if selection.meta.pools.len() != self.pools.len()
11840            || selection
11841                .meta
11842                .pools
11843                .iter()
11844                .zip(&self.pools)
11845                .enumerate()
11846                .any(|(index, (meta, pool))| meta.id != index || meta.cmd_line != pool.endpoints.cmd_line)
11847        {
11848            return Err(Error::other(
11849                "pool metadata recovery requires operator reconciliation of the pool topology",
11850            ));
11851        }
11852        let outcome = repair_pool_meta_transaction(self.pools.clone(), &mut candidate, selection, &fence).await?;
11853        let mut pool_meta = self.pool_meta.write().await;
11854        fence.ensure_held()?;
11855        self.ctx.advance_data_movement_operation_epoch();
11856        *pool_meta = outcome.committed.clone();
11857        fence.ensure_held()?;
11858        outcome.disarm();
11859        // No await between final fence validation, publication, and clearing.
11860        candidate.aborted_transaction = Arc::clone(&state.aborted_transaction);
11861        candidate.transaction_failure = Arc::clone(&state.transaction_failure);
11862        candidate.active_transactions = Arc::clone(&state.active_transactions);
11863        candidate.block_started_at = Arc::clone(&state.block_started_at);
11864        *state
11865            .block_started_at
11866            .lock()
11867            .unwrap_or_else(std::sync::PoisonError::into_inner) = None;
11868        *state
11869            .transaction_failure
11870            .lock()
11871            .unwrap_or_else(std::sync::PoisonError::into_inner) = None;
11872        state.aborted_transaction.store(false, Ordering::SeqCst);
11873        *state = candidate;
11874        info!(
11875            event = EVENT_DECOMMISSION_STATE,
11876            component = LOG_COMPONENT_ECSTORE,
11877            subsystem = LOG_SUBSYSTEM_POOLS,
11878            state = "pool_metadata_recovered",
11879            phase = blocked.phase,
11880            blocked_since = %blocked.since,
11881            "Pool metadata writes recovered"
11882        );
11883        metrics::counter!("rustfs_pool_metadata_recoveries_total").increment(1);
11884        Ok(true)
11885    }
11886
11887    pub(crate) fn record_pool_meta_recovery_failure(&self, error: Error) {
11888        let Ok(mut state) = self.pool_meta_save_gate.try_lock() else {
11889            return;
11890        };
11891        if state.ensure_write_safe("pool metadata recovery").is_ok() {
11892            return;
11893        }
11894        let classify = |error: &Error| match error.pool_metadata_failure() {
11895            Some(context) => (context.kind.as_str(), context.phase),
11896            None => match error {
11897                Error::OperationCanceled => ("workers_draining", "recovery_quiesce"),
11898                Error::Timeout => ("timeout", "recovery"),
11899                Error::Lock(_) => ("fence_unavailable", "recovery"),
11900                _ => ("unavailable", "recovery"),
11901            },
11902        };
11903        let (reason, phase) = classify(&error);
11904        if state
11905            .recovery_failure
11906            .as_ref()
11907            .is_none_or(|previous| classify(previous) != (reason, phase))
11908        {
11909            warn!(
11910                event = EVENT_DECOMMISSION_STATE,
11911                component = LOG_COMPONENT_ECSTORE,
11912                subsystem = LOG_SUBSYSTEM_POOLS,
11913                state = "pool_metadata_recovery_pending",
11914                reason,
11915                phase,
11916                "Pool metadata recovery remains blocked"
11917            );
11918        }
11919        state.recovery_failure = Some(Arc::new(error));
11920    }
11921
11922    async fn load_runtime_pool_meta_observing(&self, write_state: &mut PoolMetaWriteState, operation: &str) -> Result<PoolMeta> {
11923        write_state.ensure_write_safe(operation)?;
11924        load_pool_meta_identity_observing(self.pools.clone(), write_state).await?;
11925        let mut pool_meta = PoolMeta::default();
11926        let replica_state = pool_meta
11927            .load_no_lock_from_replicas_observing(self.pools.clone(), write_state)
11928            .await?;
11929        write_state.observe_replicas(replica_state);
11930        write_state.ensure_missing_metadata_can_initialize()?;
11931        write_state.ensure_write_safe(operation)?;
11932        Ok(pool_meta)
11933    }
11934
11935    pub(crate) async fn load_runtime_pool_meta_under_activation_fence(
11936        &self,
11937        write_state: &mut PoolMetaWriteState,
11938        activation_fence: &PoolRebalanceActivationFence,
11939        operation: &str,
11940    ) -> Result<PoolMeta> {
11941        activation_fence.ensure_held()?;
11942        let pool_meta = self.load_runtime_pool_meta_observing(write_state, operation).await?;
11943        activation_fence.ensure_held()?;
11944        Ok(pool_meta)
11945    }
11946
11947    pub(crate) async fn load_runtime_pool_meta(&self, operation: &str) -> Result<PoolMeta> {
11948        let mut write_state = self.pool_meta_save_gate.lock().await;
11949        write_state.ensure_write_safe(operation)?;
11950        let pool = self
11951            .pools
11952            .first()
11953            .cloned()
11954            .ok_or_else(|| Error::other(format!("{operation}: no storage pools available")))?;
11955        let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
11956        let _pool_meta_guard = pool_meta_lock.get_read_lock(get_lock_acquire_timeout()).await?;
11957        self.load_runtime_pool_meta_observing(&mut write_state, operation).await
11958    }
11959
11960    async fn run_guarded_decommission_side_effect<T, E, F, Fut>(
11961        &self,
11962        rx: &CancellationToken,
11963        operation_gate: &Arc<tokio::sync::RwLock<()>>,
11964        operation: F,
11965    ) -> std::result::Result<T, E>
11966    where
11967        F: FnOnce() -> Fut,
11968        Fut: std::future::Future<Output = std::result::Result<T, E>>,
11969        E: From<Error>,
11970    {
11971        let _operation_guard = tokio::select! {
11972            biased;
11973            _ = rx.cancelled() => return Err(Error::OperationCanceled.into()),
11974            guard = operation_gate.read() => guard,
11975        };
11976
11977        if rx.is_cancelled() {
11978            return Err(Error::OperationCanceled.into());
11979        }
11980        self.ensure_pool_meta_side_effects_safe("decommission side effect blocked because pool metadata requires recovery")
11981            .await
11982            .map_err(E::from)?;
11983        if rx.is_cancelled() {
11984            return Err(Error::OperationCanceled.into());
11985        }
11986
11987        let result = operation().await;
11988        if rx.is_cancelled() {
11989            return Err(Error::OperationCanceled.into());
11990        }
11991        result
11992    }
11993
11994    async fn save_current_pool_meta(&self, indices: &[usize]) -> Result<()> {
11995        let mut save_guard = self.pool_meta_save_gate.lock().await;
11996        let (pool_meta_guard, mut snapshot) = self
11997            .acquire_pool_meta_write_guard(&mut save_guard, "pool metadata save failed")
11998            .await?;
11999        {
12000            let pool_meta = self.pool_meta.read().await;
12001            merge_pool_meta_updates_for_save(&mut snapshot, &pool_meta, indices, "pool metadata save failed")?;
12002        }
12003        let outcome = snapshot
12004            .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), indices)
12005            .await?;
12006        let mut pool_meta = self.pool_meta.write().await;
12007        ensure_pool_meta_write_fence(&pool_meta_guard, "pool metadata save failed")?;
12008        publish_pool_meta_updates(&mut pool_meta, &outcome.committed, indices);
12009        ensure_pool_meta_write_fence(&pool_meta_guard, "pool metadata save failed")?;
12010        drop(pool_meta);
12011        outcome.disarm();
12012        Ok(())
12013    }
12014
12015    #[cfg(test)]
12016    pub(crate) async fn save_current_pool_meta_for_test(&self, indices: &[usize]) -> Result<()> {
12017        self.save_current_pool_meta(indices).await
12018    }
12019
12020    async fn persist_decommission_unresolved_entry(
12021        &self,
12022        idx: usize,
12023        generation: OffsetDateTime,
12024        entry: DecommissionUnresolvedEntry,
12025    ) -> Result<()> {
12026        {
12027            let rebalance_meta = self.rebalance_meta.read().await.clone();
12028            let mut pool_meta = self.pool_meta.write().await;
12029            ensure_decommission_ledger_persistence_supported(&pool_meta)?;
12030            record_decommission_unresolved_entry(
12031                &mut pool_meta,
12032                idx,
12033                generation,
12034                entry,
12035                OffsetDateTime::now_utc(),
12036                rebalance_meta.as_ref(),
12037            )?;
12038        }
12039        self.save_current_pool_meta(&[idx])
12040            .await
12041            .map_err(|err| Error::other(format!("decommission unresolved entry ledger save failed: {err}")))
12042    }
12043
12044    async fn save_decommission_progress_checkpoint(&self, idx: usize, generation: OffsetDateTime) -> Result<bool> {
12045        self.save_decommission_progress_checkpoint_at(idx, generation, OffsetDateTime::now_utc())
12046            .await
12047    }
12048
12049    async fn save_decommission_progress_checkpoint_at(
12050        &self,
12051        idx: usize,
12052        generation: OffsetDateTime,
12053        now: OffsetDateTime,
12054    ) -> Result<bool> {
12055        // Lock order: save gate, rebalance metadata, then the short pool
12056        // metadata read/write sections. Peer reloads are intentionally
12057        // performed by the caller after both locks are released.
12058        let mut save_guard = self.pool_meta_save_gate.lock().await;
12059        let (pool_meta_guard, mut snapshot) = self
12060            .acquire_pool_meta_write_guard(&mut save_guard, "decommission progress save failed")
12061            .await?;
12062        let (snapshot, checkpoint) = {
12063            let rebalance_meta = self.rebalance_meta.read().await.clone();
12064            let pool_meta = self.pool_meta.read().await;
12065            ensure_decommission_generation(&pool_meta, idx, generation)?;
12066            let Some(checkpoint) = pool_meta.decommission_progress_checkpoint(
12067                idx,
12068                DECOMMISSION_PROGRESS_SAVE_INTERVAL,
12069                now,
12070                rebalance_meta.as_ref(),
12071            )?
12072            else {
12073                return Ok(false);
12074            };
12075
12076            let mut current = pool_meta.clone();
12077            let current_count = current.pools.len();
12078            let Some(pool) = current.pools.get_mut(idx) else {
12079                return Err(invalid_decommission_pool_index_error(current_count, idx));
12080            };
12081            pool.last_update = checkpoint.checkpoint_at;
12082            if let (Some(operation_id), Some(owner_nonce), Some(expires_at), Some(reservation)) = (
12083                checkpoint.capacity_operation_id,
12084                checkpoint.capacity_owner_nonce,
12085                checkpoint.capacity_lease_expires_at,
12086                pool.decommission.as_mut().and_then(|info| info.capacity_reservation.as_mut()),
12087            ) && reservation.operation_id == operation_id
12088                && reservation.owner_nonce == owner_nonce
12089                && reservation.active()
12090            {
12091                reservation.renewed_at = checkpoint.checkpoint_at;
12092                reservation.expires_at = expires_at;
12093            }
12094            merge_pool_meta_updates_for_save(&mut snapshot, &current, &[idx], "decommission progress save failed")?;
12095            (snapshot, checkpoint)
12096        };
12097
12098        let outcome = match snapshot
12099            .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx])
12100            .await
12101        {
12102            Ok(outcome) => outcome,
12103            Err(err) => {
12104                let retry_after = OffsetDateTime::now_utc() + DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
12105                let mut pool_meta = self.pool_meta.write().await;
12106                pool_meta.defer_decommission_progress_checkpoint(idx, checkpoint, retry_after);
12107                return Err(err);
12108            }
12109        };
12110
12111        let mut pool_meta = self.pool_meta.write().await;
12112        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission progress save failed")?;
12113        pool_meta.version = pool_meta.version.max(outcome.committed.version);
12114        let committed = pool_meta.commit_decommission_progress_checkpoint(idx, checkpoint);
12115        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission progress save failed")?;
12116        drop(pool_meta);
12117        outcome.disarm();
12118        Ok(committed)
12119    }
12120
12121    async fn mark_decommission_bucket_done_and_save(
12122        &self,
12123        idx: usize,
12124        generation: OffsetDateTime,
12125        bucket: &DecomBucketInfo,
12126    ) -> Result<bool> {
12127        // Lock order: save gate -> pool metadata namespace -> rebalance metadata
12128        // -> short local pool metadata sections. Publish only after the save.
12129        let mut save_guard = self.pool_meta_save_gate.lock().await;
12130        let (pool_meta_guard, mut snapshot) = self
12131            .acquire_pool_meta_write_guard(&mut save_guard, "decommission bucket completion save failed")
12132            .await?;
12133        ensure_decommission_generation(&snapshot, idx, generation)?;
12134        let rebalance_meta = self.rebalance_meta.read().await.clone();
12135        let checkpoint = {
12136            let pool_meta = self.pool_meta.read().await;
12137            ensure_decommission_generation(&pool_meta, idx, generation)?;
12138            let mut current = pool_meta.clone();
12139            publish_decommission_capacity_update(&mut current, &snapshot, idx)?;
12140            if !mark_decommission_bucket_done(&mut current, idx, bucket)? {
12141                return Ok(false);
12142            }
12143            let checkpoint_at = current.next_scanner_data_movement_update(OffsetDateTime::now_utc(), rebalance_meta.as_ref());
12144            current.pools[idx].last_update = checkpoint_at;
12145            let info = current.pools[idx]
12146                .decommission
12147                .as_ref()
12148                .ok_or_else(|| decommission_metadata_not_initialized_error("prepare decommission bucket completion"))?;
12149            let checkpoint = DecommissionProgressCheckpoint {
12150                start_time: Some(generation),
12151                queued: false,
12152                counted_items: info.counted_items(),
12153                checkpoint_at,
12154                capacity_operation_id: None,
12155                capacity_owner_nonce: None,
12156                capacity_lease_expires_at: None,
12157            };
12158            merge_pool_meta_updates_for_save(&mut snapshot, &current, &[idx], "decommission bucket completion save failed")?;
12159            checkpoint
12160        };
12161
12162        #[cfg(test)]
12163        decommission_test_wrap_result("bucket_completion_before_save", &bucket.name, &bucket.prefix, 0, Ok(()))?;
12164        let outcome = snapshot
12165            .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx])
12166            .await?;
12167        #[cfg(test)]
12168        decommission_test_wrap_result("bucket_completion_after_save", &bucket.name, &bucket.prefix, 0, Ok(()))?;
12169        let mut pool_meta = self.pool_meta.write().await;
12170        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission bucket completion save failed")?;
12171        ensure_decommission_generation(&pool_meta, idx, generation)?;
12172        publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, idx)?;
12173        mark_decommission_bucket_done(&mut pool_meta, idx, bucket)?;
12174        pool_meta.commit_decommission_progress_checkpoint(idx, checkpoint);
12175        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission bucket completion save failed")?;
12176        drop(pool_meta);
12177        outcome.disarm();
12178        Ok(true)
12179    }
12180
12181    pub(crate) async fn save_current_pool_meta_for_decommission_start(
12182        &self,
12183        indices: &[usize],
12184        decom_buckets: Vec<DecomBucketInfo>,
12185    ) -> Result<PoolMeta> {
12186        let mut save_guard = self.pool_meta_save_gate.lock().await;
12187        save_guard.ensure_write_safe("decommission start failed")?;
12188        let rebalance_pool = self
12189            .pools
12190            .first()
12191            .cloned()
12192            .ok_or_else(|| Error::other("decommission start rebalance metadata load failed: no storage pools available"))?;
12193        #[cfg(test)]
12194        observe_pool_activation_start_attempt(PoolActivationStartKind::Decommission);
12195        let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
12196        let mut activation_fence = acquire_pool_rebalance_activation_locks(rebalance_pool.clone(), fleet_proof).await?;
12197
12198        let mut rebalance_meta = RebalanceMeta::new();
12199        match rebalance_meta
12200            .load_with_opts(
12201                rebalance_pool.clone(),
12202                ObjectOptions {
12203                    no_lock: true,
12204                    ..Default::default()
12205                },
12206            )
12207            .await
12208        {
12209            Ok(()) => ensure_decommission_start_rebalance_meta_allowed(Some(&rebalance_meta))?,
12210            Err(Error::ConfigNotFound) => {}
12211            Err(err) => {
12212                return Err(Error::other(format!(
12213                    "rebalance metadata load before decommission start save failed: {err}"
12214                )));
12215            }
12216        }
12217
12218        let current_pool_meta = {
12219            let pool_meta = self.pool_meta.read().await;
12220            pool_meta.clone()
12221        };
12222        let mut latest_pool_meta = PoolMeta::default();
12223        let replica_state = latest_pool_meta
12224            .load_no_lock_from_replicas_observing(self.pools.clone(), &mut save_guard)
12225            .await?;
12226        save_guard.observe_replicas(replica_state);
12227        save_guard.ensure_write_safe("decommission start failed")?;
12228        if latest_pool_meta.pools.is_empty() {
12229            latest_pool_meta = current_pool_meta;
12230        }
12231
12232        ensure_decommission_start_pool_states(&latest_pool_meta, indices)?;
12233        ensure_decommission_ledger_persistence_supported(&latest_pool_meta)?;
12234        ensure_decommission_capacity_writer_supported(&latest_pool_meta)?;
12235        let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
12236        activation_fence.ensure_held()?;
12237        let target_fence_proof = crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof();
12238
12239        let previous_pool_meta = latest_pool_meta.clone();
12240        let capacity_generation = next_decommission_capacity_generation(&latest_pool_meta)?;
12241        let first_idx = indices.first().copied();
12242        let now = OffsetDateTime::now_utc();
12243        for idx in indices.iter().copied() {
12244            let capacity = capacity_infos
12245                .iter()
12246                .find(|capacity| capacity.pool_index == idx)
12247                .ok_or_else(|| Error::DecommissionCapacity(format!("decommission capacity snapshot is missing pool {idx}")))?;
12248            latest_pool_meta.set_decommission_state_at(
12249                idx,
12250                capacity.space,
12251                Some(idx) != first_idx,
12252                now,
12253                Some(&rebalance_meta),
12254            )?;
12255            latest_pool_meta.queue_buckets(idx, decom_buckets.clone());
12256        }
12257        let model_version = select_decommission_capacity_model(&latest_pool_meta, target_fence_proof.is_some())?;
12258        if model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION {
12259            activation_fence.set_decommission_target_fence_proof(target_fence_proof);
12260        }
12261        reserve_decommission_start_target_capacity(
12262            &mut latest_pool_meta,
12263            indices,
12264            &capacity_infos,
12265            uuid::Uuid::new_v4(),
12266            capacity_generation,
12267            now,
12268            model_version,
12269        )?;
12270
12271        activation_fence.ensure_held()?;
12272        let outcome = latest_pool_meta
12273            .save_no_lock_with_activation_fence(self.pools.clone(), &mut save_guard, &activation_fence, indices)
12274            .await?;
12275        activation_fence.ensure_held()?;
12276        {
12277            let mut pool_meta = self.pool_meta.write().await;
12278            publish_pool_meta_updates(&mut pool_meta, &outcome.committed, indices);
12279        }
12280        activation_fence.ensure_held()?;
12281        outcome.disarm();
12282
12283        Ok(previous_pool_meta)
12284    }
12285
12286    async fn rollback_decommission_start_after_reload_failure(
12287        &self,
12288        movement_gate: &Arc<tokio::sync::RwLock<()>>,
12289        previous_pool_meta: &PoolMeta,
12290        indices: &[usize],
12291    ) -> Result<()> {
12292        let _movement_guard = movement_gate.write().await;
12293        let mut save_guard = self.pool_meta_save_gate.lock().await;
12294        let (pool_meta_guard, mut snapshot) = self
12295            .acquire_pool_meta_write_guard(&mut save_guard, "decommission start rollback failed")
12296            .await?;
12297        rollback_start_decommission_pool_meta(&mut snapshot, previous_pool_meta, indices);
12298        let outcome = snapshot
12299            .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), indices)
12300            .await?;
12301        let mut pool_meta = self.pool_meta.write().await;
12302        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission start rollback failed")?;
12303        publish_pool_meta_updates(&mut pool_meta, &outcome.committed, indices);
12304        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission start rollback failed")?;
12305        drop(pool_meta);
12306        outcome.disarm();
12307        self.ctx.advance_data_movement_operation_epoch();
12308        Ok(())
12309    }
12310
12311    async fn ensure_decommission_rebalance_idle_after_refresh(&self) -> Result<()> {
12312        self.load_rebalance_meta().await?;
12313        ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)
12314    }
12315
12316    async fn ensure_decommission_rebalance_idle_after_refresh_under_start_gate(&self) -> Result<()> {
12317        self.load_rebalance_meta_under_start_gate().await?;
12318        ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)
12319    }
12320
12321    pub async fn status(&self, idx: usize) -> Result<PoolStatus> {
12322        let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
12323        let space_info = capacity_infos
12324            .iter()
12325            .find(|capacity| capacity.pool_index == idx)
12326            .map(|capacity| capacity.space)
12327            .ok_or_else(|| invalid_decommission_pool_index_error(self.pools.len(), idx))?;
12328
12329        let pool_meta = self.pool_meta.read().await;
12330        let active_target_reservations = active_decommission_target_reservations(&pool_meta);
12331
12332        let mut pool_info = get_by_index(pool_meta.pools.as_slice(), idx, "fetch decommission status")?.clone();
12333        if let Some(info) = pool_info.decommission.as_mut() {
12334            observe_decommission_capacity_reservation(info, &capacity_infos, &active_target_reservations);
12335        }
12336        Ok(apply_decommission_status_space_info(pool_info, space_info))
12337    }
12338
12339    #[tracing::instrument(skip_all)]
12340    pub async fn refresh_pool_status_meta(&self) -> Result<()> {
12341        let movement_gate = self.ctx.data_movement_operation_gate();
12342        let _movement_guard = movement_gate.write().await;
12343        let persisted = self.load_runtime_pool_meta("refresh pool status metadata failed").await?;
12344
12345        let active_workers = {
12346            let cancelers = self.decommission_cancelers.read().await;
12347            cancelers
12348                .iter()
12349                .map(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
12350                .collect::<Vec<_>>()
12351        };
12352
12353        let mut pool_meta = self.pool_meta.write().await;
12354        if merge_pool_status_refresh(&mut pool_meta, persisted, &active_workers) {
12355            self.ctx.advance_data_movement_operation_epoch();
12356        }
12357        Ok(())
12358    }
12359
12360    async fn get_decommission_pool_capacity_info(&self, idx: usize) -> Result<DecommissionPoolCapacityInfo> {
12361        if let Some(sets) = self.pools.get(idx) {
12362            let mut info = sets.storage_info_snapshot().await;
12363            info.backend = StorageAdminApi::backend_info(self).await;
12364
12365            let total = get_total_usable_capacity(&info.disks, &info);
12366            let free = get_total_usable_capacity_free(&info.disks, &info);
12367            let space = PoolSpaceInfo {
12368                free,
12369                total,
12370                used: total.saturating_sub(free),
12371            };
12372            let layout = DecommissionErasureLayout {
12373                data: info
12374                    .backend
12375                    .standard_sc_data
12376                    .get(idx)
12377                    .copied()
12378                    .unwrap_or_else(|| sets.set_drive_count.saturating_sub(sets.parity_count)),
12379                parity: info
12380                    .backend
12381                    .standard_sc_parities
12382                    .get(idx)
12383                    .copied()
12384                    .unwrap_or(sets.parity_count),
12385            };
12386            if !layout.is_valid() {
12387                return Err(Error::DecommissionCapacity(format!(
12388                    "failed to read decommission capacity for pool {idx}: invalid erasure layout data={} parity={}",
12389                    layout.data, layout.parity
12390                )));
12391            }
12392            let (physical_total, physical_free, physical_used) =
12393                decommission_physical_pool_capacity(&info.disks, idx, layout, space);
12394
12395            Ok(DecommissionPoolCapacityInfo {
12396                pool_index: idx,
12397                space,
12398                layout,
12399                physical_free,
12400                physical_total,
12401                physical_used,
12402            })
12403        } else {
12404            Err(invalid_decommission_pool_index_error(self.pools.len(), idx))
12405        }
12406    }
12407
12408    async fn get_decommission_all_pool_capacity_infos(&self) -> Result<Vec<DecommissionPoolCapacityInfo>> {
12409        #[cfg(any(test, feature = "test-util"))]
12410        if let Some(capacity_infos) = take_decommission_capacity_info_override_for_test(self.id) {
12411            return Ok(capacity_infos);
12412        }
12413
12414        let mut capacity_infos = Vec::with_capacity(self.pools.len());
12415        for idx in 0..self.pools.len() {
12416            capacity_infos.push(self.get_decommission_pool_capacity_info(idx).await?);
12417        }
12418        Ok(capacity_infos)
12419    }
12420
12421    async fn ensure_decommission_runtime_capacity_available(&self, idx: usize, generation: OffsetDateTime) -> Result<()> {
12422        let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
12423        let pool_meta = self.pool_meta.read().await;
12424        ensure_decommission_generation(&pool_meta, idx, generation)?;
12425        #[cfg(test)]
12426        if pool_meta
12427            .pools
12428            .get(idx)
12429            .and_then(|pool| pool.decommission.as_ref())
12430            .is_some_and(|info| info.capacity_reservation.is_none())
12431        {
12432            return Ok(());
12433        }
12434        ensure_decommission_capacity_reservations_available(&pool_meta, &capacity_infos, "migration")
12435    }
12436
12437    async fn decommission_capacity_owner_for_worker(
12438        &self,
12439        idx: usize,
12440        generation: OffsetDateTime,
12441    ) -> Result<Option<DecommissionCapacityOwner>> {
12442        let pool_meta = self.pool_meta.read().await;
12443        ensure_decommission_generation(&pool_meta, idx, generation)?;
12444        let Some(reservation) = pool_meta
12445            .pools
12446            .get(idx)
12447            .and_then(|pool| pool.decommission.as_ref())
12448            .and_then(|info| info.capacity_reservation.as_ref())
12449            .filter(|reservation| reservation.lease_active_at(OffsetDateTime::now_utc()))
12450        else {
12451            return Ok(None);
12452        };
12453        Ok(Some(DecommissionCapacityOwner {
12454            source_pool_index: idx,
12455            operation_id: reservation.operation_id,
12456            generation: reservation.generation,
12457            owner_nonce: reservation.owner_nonce,
12458            mutation_id: None,
12459        }))
12460    }
12461
12462    pub(crate) async fn is_decommission_capacity_target_reserved(
12463        &self,
12464        owner: DecommissionCapacityOwner,
12465        target_pool_index: usize,
12466    ) -> Result<bool> {
12467        let pool_meta = self.pool_meta.read().await;
12468        let reservation = pool_meta
12469            .pools
12470            .get(owner.source_pool_index)
12471            .and_then(|pool| pool.decommission.as_ref())
12472            .and_then(|info| info.capacity_reservation.as_ref())
12473            .filter(|reservation| reservation.admits_owner(owner, OffsetDateTime::now_utc()))
12474            .ok_or_else(|| decommission_capacity_blocked_error("decommission target selection reservation is stale"))?;
12475        Ok(reservation
12476            .targets
12477            .iter()
12478            .any(|target| target.pool_index == target_pool_index))
12479    }
12480
12481    pub(crate) async fn select_decommission_capacity_target_pool(
12482        &self,
12483        owner: DecommissionCapacityOwner,
12484        expected_data_bytes: usize,
12485    ) -> Result<usize> {
12486        let pool_meta = self.pool_meta.read().await;
12487        let reservation = pool_meta
12488            .pools
12489            .get(owner.source_pool_index)
12490            .and_then(|pool| pool.decommission.as_ref())
12491            .and_then(|info| info.capacity_reservation.as_ref())
12492            .filter(|reservation| reservation.admits_owner(owner, OffsetDateTime::now_utc()))
12493            .ok_or_else(|| decommission_capacity_blocked_error("decommission target selection reservation is stale"))?;
12494        let candidate = |target: &DecommissionCapacityTarget| {
12495            let expected_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target.layout).ok()?;
12496            let required_peak = expected_physical_bytes.saturating_mul(1usize.saturating_add(reservation.temporary_copies));
12497            let remaining = target.remaining_reserved_physical_bytes(reservation.temporary_copies);
12498            (required_peak <= remaining).then_some((target.pool_index, remaining))
12499        };
12500        if let Some(permitted_target_pool_index) = decommission_capacity_target_permit_index(self.id, owner) {
12501            if let Some((pool_index, _)) = reservation
12502                .targets
12503                .iter()
12504                .find(|target| target.pool_index == permitted_target_pool_index)
12505                .and_then(&candidate)
12506            {
12507                return Ok(pool_index);
12508            }
12509            // The holder that preceded this waiter may have consumed the
12510            // remaining allocation. Release that guard before selecting a
12511            // different target so one mutation never holds two target gates.
12512            discard_decommission_capacity_target_permit_except(self.id, owner, None);
12513        }
12514        reservation
12515            .targets
12516            .iter()
12517            .filter_map(candidate)
12518            .max_by_key(|(_, remaining)| *remaining)
12519            .map(|(pool_index, _)| pool_index)
12520            .ok_or_else(|| {
12521                decommission_capacity_blocked_error(format!(
12522                    "source pool {} has no target allocation for {expected_data_bytes} data bytes",
12523                    owner.source_pool_index
12524                ))
12525            })
12526    }
12527
12528    pub(crate) async fn next_scanner_data_movement_update(&self, now: OffsetDateTime) -> OffsetDateTime {
12529        let pool_meta = self.pool_meta.read().await;
12530        let rebalance_meta = self.rebalance_meta.read().await;
12531        pool_meta.next_scanner_data_movement_update(now, rebalance_meta.as_ref())
12532    }
12533
12534    #[tracing::instrument(skip(self))]
12535    pub async fn decommission_cancel(self: &Arc<Self>, idx: usize) -> Result<()> {
12536        self.decommission_cancel_with_owner(idx, None).await
12537    }
12538
12539    async fn decommission_cancel_for_operation(self: &Arc<Self>, idx: usize, owner: &DecommissionCanceler) -> Result<()> {
12540        self.decommission_cancel_with_owner(idx, Some(owner)).await
12541    }
12542
12543    #[cfg(test)]
12544    async fn decommission_cancel_with_owner_and_save<Save, SaveFuture>(
12545        self: &Arc<Self>,
12546        idx: usize,
12547        owner: Option<&DecommissionCanceler>,
12548        save_pool_meta: Save,
12549    ) -> Result<()>
12550    where
12551        Save: FnOnce(PoolMeta, Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> SaveFuture + Send + 'static,
12552        SaveFuture: Future<Output = Result<()>> + Send + 'static,
12553    {
12554        let store = self.clone();
12555        let owner = owner.cloned();
12556        // Dropping the RPC waiter detaches this task; the transaction retains
12557        // the store and exact owner until persistence is resolved.
12558        tokio::spawn(async move { store.decommission_cancel_transaction(idx, owner, false, save_pool_meta).await })
12559            .await
12560            .map_err(|err| Error::other(format!("decommission cancel transaction task join error: {err}")))?
12561    }
12562
12563    async fn decommission_cancel_transaction<Save, SaveFuture>(
12564        &self,
12565        idx: usize,
12566        owner: Option<DecommissionCanceler>,
12567        acquire_runtime_fence: bool,
12568        save_pool_meta: Save,
12569    ) -> Result<()>
12570    where
12571        Save: FnOnce(PoolMeta, Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> SaveFuture,
12572        SaveFuture: Future<Output = Result<()>>,
12573    {
12574        let owner = owner.as_ref();
12575        ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?;
12576        #[cfg(test)]
12577        if acquire_runtime_fence {
12578            pause_decommission_cancel_before_start_gate(self.id).await;
12579        }
12580        let _start_guard = self.start_gate.lock().await;
12581        // Read the fence model from durable metadata without retaining the
12582        // global lock, then fence the exact target cohort before taking the
12583        // write lock used to publish the terminal transition.
12584        let terminal_fence_plan = if acquire_runtime_fence {
12585            let read_save_guard = self.pool_meta_save_gate.lock().await;
12586            let (read_guard, snapshot) = self
12587                .acquire_pool_meta_read_guard(&read_save_guard, "decommission cancel fence planning failed")
12588                .await?;
12589            let plan = decommission_capacity_terminal_fence_plan(&snapshot, idx)?;
12590            drop(read_guard);
12591            drop(read_save_guard);
12592            plan
12593        } else {
12594            None
12595        };
12596        let _capacity_target_guards = if acquire_runtime_fence {
12597            self.acquire_decommission_capacity_terminal_guards(idx, terminal_fence_plan.as_ref())
12598                .await?
12599        } else {
12600            Vec::new()
12601        };
12602        let mut save_guard = self.pool_meta_save_gate.lock().await;
12603        let (_pool_meta_guard, mut persisted_pool_meta) = if acquire_runtime_fence {
12604            let (guard, pool_meta) = self
12605                .acquire_pool_meta_write_guard(&mut save_guard, "decommission cancel failed")
12606                .await?;
12607            (Some(guard), Some(pool_meta))
12608        } else {
12609            save_guard.ensure_write_safe("decommission cancel failed")?;
12610            (None, None)
12611        };
12612        if let Some(persisted_pool_meta) = persisted_pool_meta.as_ref() {
12613            let committed_plan = decommission_capacity_terminal_fence_plan(persisted_pool_meta, idx)?;
12614            if committed_plan != terminal_fence_plan {
12615                return Err(decommission_capacity_blocked_error(
12616                    "decommission capacity owner or target cohort changed while acquiring terminal fences",
12617                ));
12618            }
12619        }
12620        let pool_meta_fence = _pool_meta_guard
12621            .as_ref()
12622            .and_then(rustfs_lock::NamespaceLockGuard::lock_lost_signal);
12623
12624        // Lock order: start gate, target gates, save gate, distributed pool
12625        // metadata fence, rebalance_meta, decommission_cancelers, then
12626        // pool_meta. The state guards stay held across persistence so the
12627        // active generation cannot change before the cancel is published.
12628        let rebalance_meta = self.rebalance_meta.read().await.clone();
12629        let terminal_at = OffsetDateTime::now_utc();
12630        let mut cancelers = self.decommission_cancelers.write().await;
12631        let mut pool_meta = self.pool_meta.write().await;
12632        if acquire_runtime_fence {
12633            let local_plan = decommission_capacity_terminal_fence_plan(&pool_meta, idx)?;
12634            if local_plan != terminal_fence_plan {
12635                return Err(decommission_capacity_blocked_error(
12636                    "local decommission capacity owner differs from the durable terminal fence plan",
12637                ));
12638            }
12639        }
12640        let (pending, should_reload_pool_meta, already_canceled, terminal_canceler, durable_movement_generation) = {
12641            let mut already_canceled = false;
12642            let (pool_present, decommission_present, terminal) = if let Some(pool) = pool_meta.pools.get(idx) {
12643                if let Some(info) = pool.decommission.as_ref() {
12644                    already_canceled = info.canceled;
12645                    (
12646                        true,
12647                        info.has_decommission_state(),
12648                        should_reject_decommission_cancel_as_terminal(info.complete, info.failed),
12649                    )
12650                } else {
12651                    (true, false, false)
12652                }
12653            } else {
12654                (false, false, false)
12655            };
12656
12657            ensure_decommission_cancel_allowed(pool_present, decommission_present, terminal)?;
12658            let previous_pool = pool_meta
12659                .pools
12660                .get(idx)
12661                .cloned()
12662                .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), idx))?;
12663            let previous_decommission = previous_pool
12664                .decommission
12665                .as_ref()
12666                .ok_or_else(|| decommission_metadata_not_initialized_error("cancel decommission"))?;
12667            let mut snapshot = pool_meta.clone();
12668            let Some(changed) = update_decommission_for_operation(cancelers.as_slice(), &mut snapshot, idx, owner, |pool_meta| {
12669                pool_meta.decommission_cancel_at(idx, terminal_at, rebalance_meta.as_ref())
12670            }) else {
12671                return Ok(());
12672            };
12673            let pending = if changed {
12674                let canceled_pool = snapshot
12675                    .pools
12676                    .get(idx)
12677                    .cloned()
12678                    .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), idx))?;
12679                if let Some(persisted) = persisted_pool_meta.as_mut() {
12680                    merge_pool_meta_updates_for_save(persisted, &snapshot, &[idx], "decommission cancel failed")?;
12681                    snapshot = persisted.clone();
12682                }
12683                Some((
12684                    snapshot,
12685                    DecommissionCancelCommit {
12686                        previous_start_time: previous_decommission.start_time,
12687                        previous_queued: previous_decommission.queued,
12688                        previous_last_update: previous_pool.last_update,
12689                        canceled_pool,
12690                    },
12691                ))
12692            } else {
12693                None
12694            };
12695            let terminal_canceler = if let Some(owner) = owner {
12696                Some(owner.clone())
12697            } else {
12698                cancelers.get(idx).and_then(Option::as_ref).cloned()
12699            };
12700            let durable_movement_generation = pending
12701                .as_ref()
12702                .map(|(_, commit)| crate::store::scanner_data_movement_timestamp_generation(commit.canceled_pool.last_update))
12703                .unwrap_or_default();
12704            (
12705                pending,
12706                should_retry_decommission_cancel_reload(changed, already_canceled),
12707                already_canceled,
12708                terminal_canceler,
12709                durable_movement_generation,
12710            )
12711        };
12712        let active_worker = terminal_canceler.as_ref().is_some_and(DecommissionCanceler::is_active);
12713        if !active_worker && !already_canceled {
12714            warn!(
12715                event = EVENT_DECOMMISSION_STATE,
12716                component = LOG_COMPONENT_ECSTORE,
12717                subsystem = LOG_SUBSYSTEM_POOLS,
12718                pool_index = idx,
12719                state = "cancel_skipped",
12720                reason = "no_active_canceler",
12721                "Decommission cancel skipped"
12722            );
12723        }
12724
12725        let changed = pending.is_some();
12726        let commit_result = if let Some((snapshot, commit)) = pending {
12727            if let Err(err) = save_pool_meta(snapshot, pool_meta_fence).await {
12728                save_guard.block_writes();
12729                return Err(err);
12730            }
12731            if let Some(pool_meta_guard) = _pool_meta_guard.as_ref()
12732                && let Err(err) = ensure_pool_meta_write_fence(pool_meta_guard, "decommission cancel failed")
12733            {
12734                save_guard.block_writes();
12735                return Err(err);
12736            }
12737            commit_decommission_cancel(&mut pool_meta, idx, commit)
12738        } else {
12739            Ok(())
12740        };
12741
12742        commit_result?;
12743        if let Some(pool_meta_guard) = _pool_meta_guard.as_ref()
12744            && let Err(err) = ensure_pool_meta_write_fence(pool_meta_guard, "decommission cancel failed")
12745        {
12746            save_guard.block_writes();
12747            return Err(err);
12748        }
12749        if let Some(canceler) = terminal_canceler.as_ref() {
12750            take_and_cancel_decommission_canceler_for_operation(cancelers.as_mut_slice(), idx, canceler);
12751        }
12752        drop(pool_meta);
12753        drop(cancelers);
12754        drop(_pool_meta_guard);
12755        drop(save_guard);
12756
12757        if changed {
12758            // Persistence must commit before the cancellation signal. Wait for
12759            // in-flight movement only after signaling so readers can release
12760            // the shared gate without deadlocking the terminal transition.
12761            let movement_gate = self.ctx.data_movement_operation_gate();
12762            let _movement_guard = movement_gate.write().await;
12763            self.ctx
12764                .advance_data_movement_operation_epoch_to_durable_generation(durable_movement_generation);
12765        }
12766
12767        if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
12768            let stage = format!("decommission_cancel for pool {idx}");
12769            if let Err(err) =
12770                resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
12771            {
12772                warn!(
12773                    event = EVENT_DECOMMISSION_STATE,
12774                    component = LOG_COMPONENT_ECSTORE,
12775                    subsystem = LOG_SUBSYSTEM_POOLS,
12776                    pool_index = idx,
12777                    state = "terminal_reload_failed",
12778                    error = %err,
12779                    "Decommission cancel saved locally but pool meta reload failed"
12780                );
12781            }
12782        }
12783
12784        Ok(())
12785    }
12786
12787    async fn release_decommission_canceler_slot(&self, idx: usize, owner: &DecommissionCanceler) {
12788        let mut cancelers = self.decommission_cancelers.write().await;
12789        take_and_cancel_decommission_canceler_for_operation(cancelers.as_mut_slice(), idx, owner);
12790    }
12791
12792    async fn decommission_terminal_retryable_for_operation(&self, idx: usize, owner: &DecommissionCanceler) -> bool {
12793        let _start_guard = self.start_gate.lock().await;
12794        let mut cancelers = self.decommission_cancelers.write().await;
12795        if !decommission_canceler_is_owned_by(cancelers.as_slice(), idx, owner) {
12796            owner.release();
12797            return false;
12798        }
12799
12800        let retryable = {
12801            let pool_meta = self.pool_meta.read().await;
12802            pool_meta
12803                .pools
12804                .get(idx)
12805                .and_then(|pool| pool.decommission.as_ref())
12806                .is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled)
12807        };
12808        if !retryable {
12809            take_and_cancel_decommission_canceler_for_operation(cancelers.as_mut_slice(), idx, owner);
12810        }
12811        retryable
12812    }
12813
12814    async fn retry_decommission_cancel_for_operation(self: &Arc<Self>, idx: usize, owner: &DecommissionCanceler) {
12815        if !self.decommission_terminal_retryable_for_operation(idx, owner).await {
12816            return;
12817        }
12818
12819        let mut attempt = 0usize;
12820        loop {
12821            if self
12822                .ensure_pool_meta_side_effects_safe("decommission cancel retry paused because pool metadata requires recovery")
12823                .await
12824                .is_err()
12825            {
12826                self.release_decommission_canceler_slot(idx, owner).await;
12827                return;
12828            }
12829            let Err(err) = self.decommission_cancel_for_operation(idx, owner).await else {
12830                return;
12831            };
12832            if !self.decommission_terminal_retryable_for_operation(idx, owner).await {
12833                return;
12834            }
12835            attempt += 1;
12836            warn!(
12837                event = EVENT_DECOMMISSION_STATE,
12838                component = LOG_COMPONENT_ECSTORE,
12839                subsystem = LOG_SUBSYSTEM_POOLS,
12840                pool_index = idx,
12841                state = "terminal_save_retry",
12842                terminal = "canceled",
12843                attempt,
12844                error = %err,
12845                "Decommission terminal save will be retried"
12846            );
12847            tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await;
12848        }
12849    }
12850
12851    async fn retry_decommission_failed_for_operation(&self, idx: usize, owner: &DecommissionCanceler) {
12852        let mut attempt = 0usize;
12853        loop {
12854            if self
12855                .ensure_pool_meta_side_effects_safe("decommission failure retry paused because pool metadata requires recovery")
12856                .await
12857                .is_err()
12858            {
12859                self.release_decommission_canceler_slot(idx, owner).await;
12860                return;
12861            }
12862            let Err(err) = self.decommission_failed_for_operation(idx, owner).await else {
12863                return;
12864            };
12865            if !self.decommission_terminal_retryable_for_operation(idx, owner).await {
12866                return;
12867            }
12868            attempt += 1;
12869            warn!(
12870                event = EVENT_DECOMMISSION_STATE,
12871                component = LOG_COMPONENT_ECSTORE,
12872                subsystem = LOG_SUBSYSTEM_POOLS,
12873                pool_index = idx,
12874                state = "terminal_save_retry",
12875                terminal = "failed",
12876                attempt,
12877                error = %err,
12878                "Decommission terminal save will be retried"
12879            );
12880            tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await;
12881        }
12882    }
12883
12884    async fn decommission_cancel_with_owner(self: &Arc<Self>, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> {
12885        let pools = self.pools.clone();
12886        let store = self.clone();
12887        let owner = owner.cloned();
12888        tokio::spawn(async move {
12889            store
12890                .decommission_cancel_transaction(idx, owner, true, move |snapshot, pool_meta_fence| async move {
12891                    snapshot.save_no_lock_with_fence(pools, pool_meta_fence, &[idx]).await
12892                })
12893                .await
12894        })
12895        .await
12896        .map_err(|err| Error::other(format!("decommission cancel transaction task join error: {err}")))?
12897    }
12898
12899    #[cfg(test)]
12900    async fn clear_decommission_with_save<Save, SaveFuture>(self: &Arc<Self>, idx: usize, save_pool_meta: Save) -> Result<()>
12901    where
12902        Save: FnOnce() -> SaveFuture + Send + 'static,
12903        SaveFuture: Future<Output = Result<()>> + Send + 'static,
12904    {
12905        let store = self.clone();
12906        tokio::spawn(async move { store.clear_decommission_transaction(idx, save_pool_meta).await })
12907            .await
12908            .map_err(|err| Error::other(format!("clear decommission transaction task join error: {err}")))?
12909    }
12910
12911    #[tracing::instrument(skip(self))]
12912    pub async fn clear_decommission(self: &Arc<Self>, idx: usize) -> Result<()> {
12913        let store = self.clone();
12914        let save_store = store.clone();
12915        // Dropping the RPC waiter detaches this task; once in-memory state can
12916        // change, the transaction must persist or roll it back before ending.
12917        tokio::spawn(async move {
12918            store
12919                .clear_decommission_transaction(idx, move || async move { save_store.save_current_pool_meta(&[idx]).await })
12920                .await
12921        })
12922        .await
12923        .map_err(|err| Error::other(format!("clear decommission transaction task join error: {err}")))?
12924    }
12925
12926    async fn clear_decommission_transaction<Save, SaveFuture>(&self, idx: usize, save_pool_meta: Save) -> Result<()>
12927    where
12928        Save: FnOnce() -> SaveFuture,
12929        SaveFuture: Future<Output = Result<()>>,
12930    {
12931        ensure_decommission_terminal_operation_supported(self.single_pool(), "clear decommission")?;
12932        let _start_guard = self.start_gate.lock().await;
12933
12934        {
12935            let pool_meta = self.pool_meta.read().await;
12936            let pool_count = pool_meta.pools.len();
12937            ensure_valid_decommission_pool_index(pool_count, idx)?;
12938            let Some(pool) = pool_meta.pools.get(idx) else {
12939                return Err(invalid_decommission_pool_index_error(pool_count, idx));
12940            };
12941            let (decommission_present, complete, failed, canceled, unresolved_entries) = pool
12942                .decommission
12943                .as_ref()
12944                .map(|info| {
12945                    (
12946                        info.has_decommission_state(),
12947                        info.complete,
12948                        info.failed,
12949                        info.canceled,
12950                        info.unresolved_entries.len(),
12951                    )
12952                })
12953                .unwrap_or((false, false, false, false, 0));
12954            ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled, unresolved_entries)?;
12955        }
12956        // Cancel workers before waiting for the movement writer so active
12957        // object operations can observe the signal and release read guards.
12958        self.cancel_decommission_routines(&[idx]).await;
12959        let movement_gate = self.ctx.data_movement_operation_gate();
12960        let _movement_guard = movement_gate.write().await;
12961
12962        let (should_reload_pool_meta, previous_pool_meta) = {
12963            let rebalance_meta = self.rebalance_meta.read().await.clone();
12964            let terminal_at = OffsetDateTime::now_utc();
12965            let mut pool_meta = self.pool_meta.write().await;
12966            let previous_pool_meta = pool_meta.clone();
12967            let changed = pool_meta.clear_decommission_at(idx, terminal_at, rebalance_meta.as_ref())?;
12968            (changed, changed.then_some(previous_pool_meta))
12969        };
12970
12971        if should_reload_pool_meta && let Err(err) = save_pool_meta().await {
12972            if let Some(previous_pool_meta) = previous_pool_meta {
12973                let mut pool_meta = self.pool_meta.write().await;
12974                rollback_decommission_pool_meta(&mut pool_meta, &previous_pool_meta, &[idx]);
12975            }
12976            return Err(err);
12977        }
12978
12979        if should_reload_pool_meta {
12980            self.ctx.advance_data_movement_operation_epoch();
12981        }
12982        drop(_movement_guard);
12983
12984        if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
12985            let stage = format!("clear_decommission for pool {idx}");
12986            if let Err(err) =
12987                resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
12988            {
12989                warn!(
12990                    event = EVENT_DECOMMISSION_STATE,
12991                    component = LOG_COMPONENT_ECSTORE,
12992                    subsystem = LOG_SUBSYSTEM_POOLS,
12993                    pool_index = idx,
12994                    state = "terminal_reload_failed",
12995                    error = %err,
12996                    "Decommission clear saved locally but pool meta reload failed"
12997                );
12998            }
12999        }
13000
13001        Ok(())
13002    }
13003
13004    async fn promote_queued_decommission(&self, idx: usize, owner: &DecommissionCanceler) -> Result<OffsetDateTime> {
13005        // Serialize promotion and generation capture with clear/restart transitions.
13006        let (changed, generation, save_error) = {
13007            let _start_guard = self.start_gate.lock().await;
13008            let movement_gate = self.ctx.data_movement_operation_gate();
13009            let _movement_guard = movement_gate.write().await;
13010            let mut save_guard = self.pool_meta_save_gate.lock().await;
13011            let (pool_meta_guard, mut snapshot) = self
13012                .acquire_pool_meta_write_guard(&mut save_guard, "decommission promotion failed")
13013                .await?;
13014            let rebalance_meta = self.rebalance_meta.read().await.clone();
13015            let capacity_infos = if self.pools.is_empty() {
13016                Vec::new()
13017            } else {
13018                self.get_decommission_all_pool_capacity_infos().await?
13019            };
13020            let target_fence_proof = crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof();
13021            let mut pool_meta = self.pool_meta.write().await;
13022            if pool_meta.pools.get(idx).is_none() {
13023                return Err(Error::other("failed to start decommission: target pool was not found"));
13024            }
13025            let capacity_recovery_needed = active_decommission_source_indices(&pool_meta).into_iter().any(|source_idx| {
13026                pool_meta
13027                    .pools
13028                    .get(source_idx)
13029                    .and_then(|pool| pool.decommission.as_ref())
13030                    .and_then(|info| info.capacity_reservation.as_ref())
13031                    .is_none_or(|reservation| !reservation.active())
13032            });
13033            let capacity_indices = if capacity_infos.is_empty() {
13034                Vec::new()
13035            } else {
13036                recover_decommission_capacity_reservations(
13037                    &mut pool_meta,
13038                    &capacity_infos,
13039                    OffsetDateTime::now_utc(),
13040                    target_fence_proof.is_some(),
13041                )?
13042            };
13043            let target_fence_proof_required = capacity_recovery_needed
13044                && active_decommission_capacity_model(&pool_meta)? == Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION);
13045            ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?;
13046            let reconciled = reconcile_decommission_meta_buckets(&mut pool_meta, idx);
13047            let promoted = pool_meta.promote_queued_decommission_at(idx, OffsetDateTime::now_utc(), rebalance_meta.as_ref());
13048            let mut changed_indices = capacity_indices;
13049            let changed = !changed_indices.is_empty() || reconciled || promoted;
13050            if changed {
13051                if !changed_indices.contains(&idx) {
13052                    changed_indices.push(idx);
13053                }
13054                merge_pool_meta_updates_for_save(&mut snapshot, &pool_meta, &changed_indices, "decommission promotion failed")?;
13055            }
13056            drop(pool_meta);
13057
13058            let (save_outcome, save_error) = if changed {
13059                ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?;
13060                match snapshot
13061                    .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &changed_indices)
13062                    .await
13063                {
13064                    Ok(outcome) => (Some(outcome), None),
13065                    Err(err) => (None, Some(err)),
13066                }
13067            } else {
13068                (None, None)
13069            };
13070            let generation = self.active_decommission_generation(idx).await?;
13071            ensure_pool_meta_write_fence(&pool_meta_guard, "decommission promotion failed")?;
13072            ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?;
13073            if let Some(outcome) = save_outcome {
13074                let mut pool_meta = self.pool_meta.write().await;
13075                pool_meta.version = pool_meta.version.max(outcome.committed.version);
13076                drop(pool_meta);
13077                outcome.disarm();
13078            }
13079            (changed, generation, save_error)
13080        };
13081
13082        if let Some(err) = save_error {
13083            resolve_decommission_terminal_mark_after_error_result(
13084                self.decommission_failed_for_operation(idx, owner).await,
13085                idx,
13086                &err,
13087            )?;
13088            return Err(err);
13089        }
13090
13091        if changed {
13092            self.ctx.advance_data_movement_operation_epoch();
13093        }
13094        if changed && let Some(notification_sys) = runtime_sources::notification_sys() {
13095            let stage = format!("promote_queued_decommission for pool {idx}");
13096            if let Err(err) =
13097                resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
13098            {
13099                resolve_decommission_terminal_mark_after_error_result(
13100                    self.decommission_failed_for_operation(idx, owner).await,
13101                    idx,
13102                    &err,
13103                )?;
13104                return Err(err);
13105            }
13106        }
13107
13108        Ok(generation)
13109    }
13110
13111    #[cfg(test)]
13112    pub(crate) async fn promote_queued_decommission_for_test(&self, idx: usize) -> Result<()> {
13113        let owner = DecommissionCanceler::new(CancellationToken::new());
13114        self.promote_queued_decommission(idx, &owner).await?;
13115        let mut cancelers = self.decommission_cancelers.write().await;
13116        if let Some(slot) = cancelers.get_mut(idx)
13117            && let Some(previous) = slot.replace(owner)
13118        {
13119            previous.release();
13120        }
13121        Ok(())
13122    }
13123
13124    async fn record_decommission_terminal_reload_failure(&self, idx: usize, stage: &str, err: Error) -> Result<()> {
13125        let movement_gate = self.ctx.data_movement_operation_gate();
13126        let _movement_guard = movement_gate.write().await;
13127        let changed = {
13128            let rebalance_meta = self.rebalance_meta.read().await.clone();
13129            let mut pool_meta = self.pool_meta.write().await;
13130            pool_meta.record_decommission_terminal_reload_failure_at(
13131                idx,
13132                stage,
13133                err.to_string(),
13134                OffsetDateTime::now_utc(),
13135                rebalance_meta.as_ref(),
13136            )?
13137        };
13138
13139        if changed {
13140            self.save_current_pool_meta(&[idx]).await?;
13141        }
13142
13143        Ok(())
13144    }
13145
13146    async fn pause_decommission_for_capacity(&self, idx: usize, err: &Error) -> Result<()> {
13147        let mut save_guard = self.pool_meta_save_gate.lock().await;
13148        let (pool_meta_guard, mut snapshot) = self
13149            .acquire_pool_meta_write_guard(&mut save_guard, "decommission capacity pause failed")
13150            .await?;
13151        snapshot.mark_decommission_capacity_blocked(idx, err.to_string(), OffsetDateTime::now_utc())?;
13152        let outcome = snapshot
13153            .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx])
13154            .await?;
13155        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission capacity pause failed")?;
13156        {
13157            let mut pool_meta = self.pool_meta.write().await;
13158            publish_pool_meta_updates(&mut pool_meta, &outcome.committed, &[idx]);
13159        }
13160        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission capacity pause failed")?;
13161        outcome.disarm();
13162        Ok(())
13163    }
13164
13165    pub async fn is_decommission_running(&self) -> bool {
13166        {
13167            let cancelers = self.decommission_cancelers.read().await;
13168            if has_active_decommission_canceler(cancelers.as_slice()) {
13169                return true;
13170            }
13171        }
13172
13173        let pool_meta = self.pool_meta.read().await;
13174        for pool in pool_meta.pools.iter() {
13175            if let Some(ref info) = pool.decommission
13176                && info.has_decommission_state()
13177                && !info.complete
13178                && !info.failed
13179                && !info.canceled
13180            {
13181                return true;
13182            }
13183        }
13184
13185        false
13186    }
13187
13188    async fn decommission_cancel_requested(&self, idx: usize, rx: &CancellationToken) -> bool {
13189        let pool_meta = self.pool_meta.read().await;
13190        is_decommission_cancel_requested(rx.is_cancelled(), pool_meta.pools.get(idx))
13191    }
13192
13193    #[cfg(test)]
13194    async fn cancel_decommission_routines_and_wait(&self, indices: &[usize]) {
13195        self.cancel_decommission_routines(indices).await;
13196        let movement_gate = self.ctx.data_movement_operation_gate();
13197        let _movement_guard = movement_gate.write().await;
13198    }
13199
13200    async fn cancel_decommission_routines(&self, indices: &[usize]) {
13201        {
13202            let mut cancelers = self.decommission_cancelers.write().await;
13203            for idx in indices {
13204                take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), *idx);
13205            }
13206        }
13207    }
13208
13209    async fn quiesce_decommission_worker_after_join_error(&self, canceler: &DecommissionCanceler) {
13210        canceler.cancel();
13211        let movement_gate = self.ctx.data_movement_operation_gate();
13212        let _movement_guard = movement_gate.write().await;
13213    }
13214
13215    async fn reserve_decommission_routines(
13216        &self,
13217        rx: &CancellationToken,
13218        indices: &[usize],
13219    ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
13220        let indices = dedup_indices(indices);
13221        if indices.is_empty() {
13222            return Ok(Vec::new());
13223        }
13224
13225        let _start_guard = self.start_gate.lock().await;
13226        let save_guard = self.pool_meta_save_gate.lock().await;
13227        save_guard.ensure_write_safe("decommission cannot be scheduled while pool metadata requires recovery")?;
13228        let indices = {
13229            let pool_meta = self.pool_meta.read().await;
13230            let indices = resumable_decommission_queue_indices(&pool_meta)
13231                .into_iter()
13232                .filter(|idx| indices.contains(idx))
13233                .collect::<Vec<_>>();
13234            if !indices.is_empty() {
13235                ensure_decommission_ledger_persistence_supported(&pool_meta)?;
13236            }
13237            indices
13238        };
13239        if indices.is_empty() {
13240            return Ok(Vec::new());
13241        }
13242
13243        let index_cancelers = {
13244            let mut cancelers = self.decommission_cancelers.write().await;
13245            let missing = missing_decommission_worker_prefix(indices.as_slice(), cancelers.as_slice());
13246            if missing.is_empty() {
13247                return Ok(Vec::new());
13248            }
13249            let bound = bind_missing_decommission_cancelers(missing.as_slice(), rx, cancelers.as_mut_slice());
13250            let guards = guard_decommission_cancelers(bound);
13251            ensure_decommission_routines_scheduled(guards.len(), missing.len())?;
13252            guards
13253        };
13254        Ok(index_cancelers)
13255    }
13256
13257    async fn reserve_missing_local_decommission_routines(
13258        &self,
13259        rx: &CancellationToken,
13260        endpoints: &EndpointServerPools,
13261    ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
13262        let indices = {
13263            let pool_meta = self.pool_meta.read().await;
13264            resumable_decommission_queue_indices(&pool_meta)
13265        };
13266        let indices = local_decommission_queue_prefix(endpoints, &indices)?;
13267        self.reserve_decommission_routines(rx, indices.as_slice()).await
13268    }
13269
13270    pub async fn spawn_missing_local_decommission_routines(self: &Arc<Self>) -> Result<()> {
13271        self.spawn_missing_local_decommission_routines_with_token(CancellationToken::new())
13272            .await
13273    }
13274
13275    pub(crate) async fn has_active_local_decommission_worker(&self) -> bool {
13276        let cancelers = self.decommission_cancelers.read().await;
13277        has_active_decommission_canceler(cancelers.as_slice())
13278    }
13279
13280    pub(crate) async fn spawn_missing_local_decommission_routines_with_token(
13281        self: &Arc<Self>,
13282        rx: CancellationToken,
13283    ) -> Result<()> {
13284        #[cfg(test)]
13285        let endpoints = self.instance_endpoints().unwrap_or_else(|| self.endpoints());
13286        #[cfg(not(test))]
13287        let endpoints = self.endpoints();
13288        let index_cancelers = self.reserve_missing_local_decommission_routines(&rx, &endpoints).await?;
13289        if index_cancelers.is_empty() {
13290            return Ok(());
13291        }
13292
13293        drop(spawn_decommission_index_cancelers(
13294            self.clone(),
13295            rx,
13296            index_cancelers,
13297            Arc::new(Semaphore::new(decommission_entry_concurrency_limit())),
13298        ));
13299        Ok(())
13300    }
13301
13302    #[tracing::instrument(skip(self, rx))]
13303    pub async fn decommission(&self, rx: CancellationToken, indices: Vec<usize>) -> Result<()> {
13304        let indices = dedup_indices(&indices);
13305
13306        info!(
13307            event = EVENT_DECOMMISSION_STATE,
13308            component = LOG_COMPONENT_ECSTORE,
13309            subsystem = LOG_SUBSYSTEM_POOLS,
13310            pool_indices = ?indices,
13311            state = "requested",
13312            "Decommission requested"
13313        );
13314        validate_start_decommission_request(&indices, self.single_pool())?;
13315
13316        self.ensure_decommission_rebalance_idle_after_refresh().await?;
13317
13318        let store = require_decommission_store(runtime_sources::object_store_handle(), "start decommission")?;
13319        let local_indices = local_decommission_queue_prefix(&self.endpoints(), &indices)?;
13320        let index_cancelers = self
13321            .start_decommission_with_routines(indices, &rx, local_indices.as_slice())
13322            .await?;
13323        drop(spawn_decommission_index_cancelers(
13324            store,
13325            rx,
13326            index_cancelers,
13327            Arc::new(Semaphore::new(decommission_entry_concurrency_limit())),
13328        ));
13329
13330        Ok(())
13331    }
13332
13333    async fn active_decommission_generation(&self, idx: usize) -> Result<OffsetDateTime> {
13334        let pool_meta = self.pool_meta.read().await;
13335        let Some(pool) = pool_meta.pools.get(idx) else {
13336            return Err(invalid_decommission_pool_index_error(pool_meta.pools.len(), idx));
13337        };
13338        let Some(info) = pool.decommission.as_ref() else {
13339            return Err(decommission_metadata_not_initialized_error("load decommission generation"));
13340        };
13341        let Some(generation) = info.start_time else {
13342            return Err(Error::OperationCanceled);
13343        };
13344        ensure_decommission_generation(&pool_meta, idx, generation)?;
13345        Ok(generation)
13346    }
13347
13348    async fn ensure_decommission_generation_current(&self, idx: usize, generation: OffsetDateTime) -> Result<()> {
13349        let pool_meta = self.pool_meta.read().await;
13350        ensure_decommission_generation(&pool_meta, idx, generation)
13351    }
13352
13353    #[allow(clippy::too_many_arguments)]
13354    async fn decommission_entry_worker(
13355        self: Arc<Self>,
13356        rx: CancellationToken,
13357        idx: usize,
13358        set_idx: usize,
13359        generation: OffsetDateTime,
13360        bucket: String,
13361        set: Arc<SetDisks>,
13362        lifecycle_config: Option<BucketLifecycleConfiguration>,
13363        object_lock_config: Option<ObjectLockConfiguration>,
13364        replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
13365        expected_bucket_incarnation_id: Option<uuid::Uuid>,
13366        source_changed_exhaustions: Arc<AtomicUsize>,
13367        entry_budget: Arc<Semaphore>,
13368        queue: Arc<tokio::sync::Mutex<mpsc::Receiver<QueuedDecommissionEntry>>>,
13369        entry_error: Arc<tokio::sync::Mutex<Option<Error>>>,
13370    ) {
13371        loop {
13372            let queued = tokio::select! {
13373                biased;
13374                _ = rx.cancelled() => return,
13375                item = async {
13376                    let mut queue = queue.lock().await;
13377                    queue.recv().await
13378                } => item,
13379            };
13380            let Some(QueuedDecommissionEntry { entry, queue_permit }) = queued else {
13381                return;
13382            };
13383            let object_name = entry.name.clone();
13384
13385            if entry_error.lock().await.is_some() {
13386                drop(queue_permit);
13387                continue;
13388            }
13389
13390            if let Err(err) = self.ensure_decommission_generation_current(idx, generation).await {
13391                if matches!(err, Error::OperationCanceled) {
13392                    rx.cancel();
13393                } else {
13394                    record_decommission_entry_error(&entry_error, &rx, err).await;
13395                }
13396                return;
13397            }
13398
13399            if let Err(err) = backpressure::wait_for_data_movement_admission(DataMovementOperation::Decommission, idx, &rx).await
13400            {
13401                if matches!(err, Error::OperationCanceled) {
13402                    return;
13403                }
13404                error!(
13405                    event = EVENT_DECOMMISSION_ENTRY,
13406                    component = LOG_COMPONENT_ECSTORE,
13407                    subsystem = LOG_SUBSYSTEM_POOLS,
13408                    pool_index = idx,
13409                    set_index = set_idx,
13410                    bucket = %bucket,
13411                    object = %object_name,
13412                    state = "entry_admission_failed",
13413                    error = %err,
13414                    "Decommission entry admission failed"
13415                );
13416                record_decommission_entry_error(&entry_error, &rx, err).await;
13417                return;
13418            }
13419
13420            let entry_budget_permit = match tokio::select! {
13421                biased;
13422                _ = rx.cancelled() => return,
13423                permit = entry_budget.clone().acquire_owned() => permit,
13424            } {
13425                Ok(permit) => permit,
13426                Err(err) => {
13427                    let err = Error::other(format!("decommission entry budget permit acquire failed: {err}"));
13428                    error!(
13429                        event = EVENT_DECOMMISSION_ENTRY,
13430                        component = LOG_COMPONENT_ECSTORE,
13431                        subsystem = LOG_SUBSYSTEM_POOLS,
13432                        pool_index = idx,
13433                        set_index = set_idx,
13434                        bucket = %bucket,
13435                        object = %object_name,
13436                        state = "entry_budget_acquire_failed",
13437                        error = %err,
13438                        "Decommission entry budget permit acquire failed"
13439                    );
13440                    record_decommission_entry_error(&entry_error, &rx, err).await;
13441                    return;
13442                }
13443            };
13444
13445            let result = self
13446                .decommission_entry(
13447                    rx.clone(),
13448                    idx,
13449                    generation,
13450                    entry,
13451                    bucket.clone(),
13452                    set.clone(),
13453                    lifecycle_config.clone(),
13454                    object_lock_config.clone(),
13455                    replication_config.clone(),
13456                    expected_bucket_incarnation_id,
13457                    Arc::clone(&source_changed_exhaustions),
13458                )
13459                .await;
13460            drop(entry_budget_permit);
13461            drop(queue_permit);
13462
13463            if let Err(err) = result {
13464                error!(
13465                    event = EVENT_DECOMMISSION_ENTRY,
13466                    component = LOG_COMPONENT_ECSTORE,
13467                    subsystem = LOG_SUBSYSTEM_POOLS,
13468                    pool_index = idx,
13469                    set_index = set_idx,
13470                    bucket = %bucket,
13471                    object = %object_name,
13472                    state = "entry_failed",
13473                    error = %err,
13474                    "Decommission entry failed"
13475                );
13476                record_decommission_entry_error(&entry_error, &rx, err).await;
13477                return;
13478            }
13479        }
13480    }
13481
13482    #[allow(clippy::too_many_arguments)]
13483    async fn decommission_set(
13484        self: Arc<Self>,
13485        rx: CancellationToken,
13486        idx: usize,
13487        set_idx: usize,
13488        generation: OffsetDateTime,
13489        set: Arc<SetDisks>,
13490        bi: DecomBucketInfo,
13491        lifecycle_config: Option<BucketLifecycleConfiguration>,
13492        object_lock_config: Option<ObjectLockConfiguration>,
13493        replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
13494        expected_bucket_incarnation_id: Option<uuid::Uuid>,
13495        source_changed_exhaustions: Arc<AtomicUsize>,
13496        entry_budget: Arc<Semaphore>,
13497        entry_error: Arc<tokio::sync::Mutex<Option<Error>>>,
13498    ) -> Result<()> {
13499        let worker_count = DECOMMISSION_ENTRY_WORKERS_PER_SET;
13500        let queue_capacity = decommission_entry_queue_capacity(worker_count);
13501        let outstanding_capacity = queue_capacity.saturating_add(worker_count);
13502        let outstanding = Arc::new(Semaphore::new(outstanding_capacity));
13503        let (tx, rx_queue) = mpsc::channel(queue_capacity);
13504        let queue = Arc::new(tokio::sync::Mutex::new(rx_queue));
13505
13506        let mut entry_workers = tokio::task::JoinSet::new();
13507        for _ in 0..worker_count {
13508            let this = self.clone();
13509            let rx = rx.clone();
13510            let bucket = bi.name.clone();
13511            let set = set.clone();
13512            let lifecycle_config = lifecycle_config.clone();
13513            let object_lock_config = object_lock_config.clone();
13514            let replication_config = replication_config.clone();
13515            let source_changed_exhaustions = Arc::clone(&source_changed_exhaustions);
13516            let queue = queue.clone();
13517            let entry_budget = entry_budget.clone();
13518            let entry_error = entry_error.clone();
13519            entry_workers.spawn(async move {
13520                this.decommission_entry_worker(
13521                    rx,
13522                    idx,
13523                    set_idx,
13524                    generation,
13525                    bucket,
13526                    set,
13527                    lifecycle_config,
13528                    object_lock_config,
13529                    replication_config,
13530                    expected_bucket_incarnation_id,
13531                    source_changed_exhaustions,
13532                    entry_budget,
13533                    queue,
13534                    entry_error,
13535                )
13536                .await;
13537            });
13538        }
13539
13540        let callback: ListCallback = Arc::new({
13541            let tx = tx.clone();
13542            let outstanding = outstanding.clone();
13543            let callback_rx = rx.clone();
13544            let entry_error = entry_error.clone();
13545            let bucket = bi.name.clone();
13546            move |entry: MetaCacheEntry| {
13547                let tx = tx.clone();
13548                let outstanding = outstanding.clone();
13549                let callback_rx = callback_rx.clone();
13550                let entry_error = entry_error.clone();
13551                let bucket = bucket.clone();
13552                Box::pin(async move {
13553                    if callback_rx.is_cancelled() || entry_error.lock().await.is_some() {
13554                        return;
13555                    }
13556
13557                    if matches!(
13558                        enqueue_decommission_entry(&callback_rx, &outstanding, &tx, entry).await,
13559                        DecommissionEntryEnqueueResult::Closed
13560                    ) {
13561                        let err = Error::other("decommission entry queue closed");
13562                        error!(
13563                            event = EVENT_DECOMMISSION_ENTRY,
13564                            component = LOG_COMPONENT_ECSTORE,
13565                            subsystem = LOG_SUBSYSTEM_POOLS,
13566                            pool_index = idx,
13567                            set_index = set_idx,
13568                            bucket = %bucket,
13569                            state = "entry_queue_closed",
13570                            error = %err,
13571                            "Decommission entry queue closed"
13572                        );
13573                        record_decommission_entry_error(&entry_error, &callback_rx, err).await;
13574                    }
13575                })
13576            }
13577        });
13578
13579        let list_set = set.clone();
13580        let list_rx = rx.clone();
13581        let list_rx_for_list = list_rx.clone();
13582        let list_rx_for_drain = list_rx.clone();
13583        let list_bi = bi.clone();
13584        let list_outstanding = outstanding.clone();
13585        let list_entry_error = entry_error.clone();
13586        let list_store = self.clone();
13587        let mut listing = tokio::spawn(async move {
13588            run_decommission_listing_with_retry_and_drain(
13589                list_rx.clone(),
13590                list_bi.name.clone(),
13591                callback,
13592                idx,
13593                set_idx,
13594                DECOMMISSION_LISTING_MAX_ATTEMPTS,
13595                move |callback| {
13596                    let set = list_set.clone();
13597                    let rx = list_rx_for_list.clone();
13598                    let bucket = list_bi.clone();
13599                    let entry_error = list_entry_error.clone();
13600                    let store = list_store.clone();
13601                    async move {
13602                        set.list_objects_to_decommission(
13603                            store,
13604                            rx,
13605                            bucket,
13606                            callback,
13607                            entry_error,
13608                            idx,
13609                            set_idx,
13610                            generation,
13611                            false,
13612                        )
13613                        .await
13614                    }
13615                },
13616                move || {
13617                    let rx = list_rx_for_drain.clone();
13618                    let outstanding = list_outstanding.clone();
13619                    async move { drain_decommission_entry_queue(&rx, &outstanding, outstanding_capacity).await }
13620                },
13621            )
13622            .await
13623        });
13624
13625        let mut listing_result = None;
13626        let mut workers_left = worker_count;
13627        let mut sender = Some(tx);
13628        while listing_result.is_none() || workers_left > 0 {
13629            tokio::select! {
13630                biased;
13631                result = &mut listing, if listing_result.is_none() => {
13632                    let result = resolve_decommission_listing_worker_result(set_idx, result);
13633                    if result.is_err() {
13634                        rx.cancel();
13635                    }
13636                    listing_result = Some(result);
13637                    drop(sender.take());
13638                }
13639                worker_result = entry_workers.join_next(), if workers_left > 0 => {
13640                    workers_left -= 1;
13641                    if let Some(Err(err)) = worker_result {
13642                        let err = Error::other(format!("decommission entry worker {set_idx} task join error: {err}"));
13643                        error!(
13644                            event = EVENT_DECOMMISSION_ENTRY,
13645                            component = LOG_COMPONENT_ECSTORE,
13646                            subsystem = LOG_SUBSYSTEM_POOLS,
13647                            pool_index = idx,
13648                            set_index = set_idx,
13649                            bucket = %bi.name,
13650                            state = "entry_worker_join_failed",
13651                            error = %err,
13652                            "Decommission entry worker task failed"
13653                        );
13654                        record_decommission_entry_error(&entry_error, &rx, err).await;
13655                    }
13656                }
13657            }
13658        }
13659
13660        let listing_result = listing_result.unwrap_or_else(|| Err(Error::other("decommission listing task did not complete")));
13661        if let Some(err) = entry_error.lock().await.clone() {
13662            return Err(err);
13663        }
13664        listing_result
13665    }
13666
13667    async fn track_decommission_entry_progress_stage(
13668        &self,
13669        idx: usize,
13670        generation: OffsetDateTime,
13671        bucket: &str,
13672        object: &str,
13673        stage: &'static str,
13674    ) -> Result<()> {
13675        {
13676            let mut pool_meta = self.pool_meta.write().await;
13677            ensure_decommission_generation(&pool_meta, idx, generation)?;
13678            track_decommission_current_object_stage(&mut pool_meta, idx, bucket, object, stage)
13679                .map_err(|err| with_decommission_entry_context(stage, bucket, object, err))?;
13680        }
13681
13682        Ok(())
13683    }
13684
13685    #[allow(clippy::too_many_arguments)]
13686    async fn wait_decommission_target_gate_retry(
13687        &self,
13688        rx: &CancellationToken,
13689        idx: usize,
13690        generation: OffsetDateTime,
13691        target_pool_index: usize,
13692        capacity_owner: Option<DecommissionCapacityOwner>,
13693        set: &SetDisks,
13694        entry: &MetaCacheEntry,
13695        bucket: &str,
13696        expected_version: &FileInfo,
13697        target_busy_attempt: usize,
13698    ) -> Result<Option<DecommissionCapacityTargetPermit>> {
13699        #[cfg(test)]
13700        notify_decommission_target_gate_retry(self.id);
13701        let mut wait_attempt = target_busy_attempt;
13702        let target_guard = loop {
13703            let retry_delay = decommission_retry_backoff_delay(
13704                DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY,
13705                wait_attempt.min(DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS),
13706            );
13707            if wait_decommission_retry_backoff(rx, retry_delay).await {
13708                decommission_cancel_signal_result(rx.is_cancelled())?;
13709            }
13710            if self.decommission_cancel_requested(idx, rx).await {
13711                rx.cancel();
13712            }
13713            decommission_cancel_signal_result(rx.is_cancelled())?;
13714            self.ensure_decommission_generation_current(idx, generation).await?;
13715
13716            match self.acquire_decommission_capacity_target_guard(target_pool_index).await {
13717                Ok(guard) => break guard,
13718                Err(err) if is_decommission_capacity_target_gate_busy(&err) => {
13719                    wait_attempt = wait_attempt.saturating_add(1);
13720                }
13721                Err(err) => return Err(err),
13722            }
13723        };
13724
13725        #[cfg(test)]
13726        notify_decommission_target_gate_exact_reload(self.id);
13727        let current = load_decommission_entry_exact_versions(set, entry, bucket, "target_gate_retry").await?;
13728        ensure_decommission_capacity_target_fence(&target_guard, target_pool_index, "source identity revalidation")?;
13729        let expected_identity = data_movement::source_cleanup_version_identity(expected_version);
13730        let identity_current = current
13731            .versions
13732            .iter()
13733            .any(|version| data_movement::source_cleanup_version_identity(version) == expected_identity);
13734        if !identity_current {
13735            return Ok(None);
13736        }
13737        let owner = capacity_owner.ok_or_else(|| Error::other("target-gate retry is missing its decommission capacity owner"))?;
13738        let owner = owner.with_mutation_id(decommission_capacity_version_mutation_id(owner, bucket, expected_version));
13739        install_decommission_capacity_target_permit(self.id, target_pool_index, owner, target_guard).map(Some)
13740    }
13741
13742    #[cfg(feature = "test-util")]
13743    pub async fn prepare_scanner_pause_backlog_retirement_for_test(
13744        &self,
13745        source_pool_index: usize,
13746        source_bytes: usize,
13747    ) -> Result<()> {
13748        let source_bytes = source_bytes.max(1);
13749        let total = source_bytes.saturating_mul(8).saturating_add(64 * 1024);
13750        let capacities = (0..self.pools.len())
13751            .map(|pool_index| {
13752                DecommissionPoolCapacityInfo::for_test(
13753                    pool_index,
13754                    DecommissionErasureLayout { data: 1, parity: 0 },
13755                    if pool_index == source_pool_index { 0 } else { total },
13756                    total,
13757                    if pool_index == source_pool_index { source_bytes } else { 0 },
13758                )
13759            })
13760            .collect();
13761        set_decommission_capacity_info_overrides_for_test(self.id, vec![capacities]);
13762        self.save_current_pool_meta_for_decommission_start(&[source_pool_index], Vec::new())
13763            .await
13764            .map(|_| ())
13765    }
13766
13767    #[cfg(feature = "test-util")]
13768    pub async fn retire_scanner_pause_backlog_for_test(
13769        self: &Arc<Self>,
13770        source_pool_index: usize,
13771        source_set_index: usize,
13772    ) -> Result<()> {
13773        let set = self
13774            .pools
13775            .get(source_pool_index)
13776            .and_then(|pool| pool.disk_set.get(source_set_index))
13777            .cloned()
13778            .ok_or_else(|| Error::other("scanner retirement test requested an unknown set"))?;
13779        let generation = self.active_decommission_generation(source_pool_index).await?;
13780        let owner = self
13781            .decommission_capacity_owner_for_worker(source_pool_index, generation)
13782            .await?;
13783        let expected = set
13784            .load_file_info_versions_exact(RUSTFS_META_BUCKET, data_movement::scanner_backlog::SCANNER_PAUSE_BACKLOG_PATH)
13785            .await?
13786            .unwrap_or_default();
13787        match self
13788            .retire_scanner_pause_backlog_entry(CancellationToken::new(), source_pool_index, generation, set, expected, owner)
13789            .await?
13790        {
13791            DecommissionEntryAttemptOutcome::Complete => Ok(()),
13792            DecommissionEntryAttemptOutcome::SourceChanged => Err(Error::other("scanner retirement source changed")),
13793        }
13794    }
13795
13796    #[cfg(feature = "test-util")]
13797    pub async fn stage_scanner_pause_backlog_retirement_intent_for_test(
13798        &self,
13799        source_pool_index: usize,
13800        source_set_index: usize,
13801    ) -> Result<()> {
13802        let generation = self.active_decommission_generation(source_pool_index).await?;
13803        let owner = self
13804            .decommission_capacity_owner_for_worker(source_pool_index, generation)
13805            .await?
13806            .ok_or_else(|| Error::other("scanner retirement test has no capacity owner"))?;
13807        let versions = self.pools[source_pool_index].disk_set[source_set_index]
13808            .load_file_info_versions_exact(RUSTFS_META_BUCKET, data_movement::scanner_backlog::SCANNER_PAUSE_BACKLOG_PATH)
13809            .await?
13810            .ok_or_else(|| Error::other("scanner retirement test source is missing"))?;
13811        let version = versions
13812            .versions
13813            .first()
13814            .ok_or_else(|| Error::other("scanner retirement test source is empty"))?;
13815        let owner = owner.with_mutation_id(decommission_capacity_version_mutation_id(owner, RUSTFS_META_BUCKET, version));
13816        let size = usize::try_from(version.size).map_err(|_| Error::other("scanner retirement test source size is invalid"))?;
13817        let target = self.select_decommission_capacity_target_pool(owner, size).await?;
13818        let failed: Result<()> = self
13819            .run_decommission_capacity_admitted_mutation(target, Some(owner), Some(size), || async {
13820                Err(Error::other("injected scanner retirement target failure"))
13821            })
13822            .await;
13823        match failed {
13824            Err(err) if err.to_string().contains("injected scanner retirement target failure") => Ok(()),
13825            Err(err) => Err(err),
13826            Ok(()) => Err(Error::other("scanner retirement test did not retain its target intent")),
13827        }
13828    }
13829
13830    #[allow(clippy::too_many_arguments)]
13831    async fn retire_scanner_pause_backlog_entry(
13832        self: &Arc<Self>,
13833        rx: CancellationToken,
13834        idx: usize,
13835        generation: OffsetDateTime,
13836        set: Arc<SetDisks>,
13837        expected: FileInfoVersions,
13838        capacity_owner: Option<DecommissionCapacityOwner>,
13839    ) -> Result<DecommissionEntryAttemptOutcome> {
13840        let store = Arc::clone(self);
13841        // The disk layer may own a rename after its waiter is canceled. Keep
13842        // the topology and object fences in that operation's owning task.
13843        tokio::spawn(async move {
13844            let operation_gate = store.ctx.data_movement_operation_gate();
13845            store
13846                .run_guarded_decommission_side_effect(&rx, &operation_gate, || {
13847                    store.retire_scanner_pause_backlog_entry_inner(idx, generation, set, &expected, capacity_owner)
13848                })
13849                .await
13850        })
13851        .await
13852        .map_err(Error::from)?
13853    }
13854
13855    async fn retire_scanner_pause_backlog_entry_inner(
13856        &self,
13857        idx: usize,
13858        generation: OffsetDateTime,
13859        set: Arc<SetDisks>,
13860        expected: &FileInfoVersions,
13861        capacity_owner: Option<DecommissionCapacityOwner>,
13862    ) -> Result<DecommissionEntryAttemptOutcome> {
13863        use data_movement::scanner_backlog::{
13864            SCANNER_PAUSE_BACKLOG_PATH, persist_native_scanner_pause_backlog_replica, plan_scanner_pause_backlog_retirement,
13865            read_scanner_pause_backlog_retirement_replicas,
13866        };
13867
13868        if expected.versions.is_empty() && expected.free_versions.is_empty() {
13869            // A canceled entry waiter may resume after the owned cleanup
13870            // finished deleting its source. There is no remaining record to move.
13871            return Ok(DecommissionEntryAttemptOutcome::Complete);
13872        }
13873        let [version] = expected.versions.as_slice() else {
13874            return Err(Error::other("scanner pause backlog retirement requires exactly one source version"));
13875        };
13876        if version.version_id.is_some_and(|version| !version.is_nil())
13877            || version.deleted
13878            || version.tier_free_version()
13879            || version.is_remote()
13880            || !expected.free_versions.is_empty()
13881        {
13882            return Err(Error::other(
13883                "scanner pause backlog retirement requires an unversioned local source record",
13884            ));
13885        }
13886        let object_fence = self
13887            .acquire_decommission_source_cleanup_fence(RUSTFS_META_BUCKET, SCANNER_PAUSE_BACKLOG_PATH, set.as_ref())
13888            .await?;
13889        // Native replica writers acquire this same fixed object domain before
13890        // durable pool metadata. Keep both fences through physical source deletion.
13891        let save_guard = self.pool_meta_save_gate.lock().await;
13892        let (pool_meta_guard, snapshot) = self
13893            .acquire_pool_meta_read_guard(&save_guard, "scanner pause backlog retirement admission failed")
13894            .await?;
13895        ensure_decommission_generation(&snapshot, idx, generation)?;
13896        let owner = capacity_owner.ok_or_else(|| {
13897            decommission_capacity_blocked_error("scanner pause backlog retirement has no active capacity owner")
13898        })?;
13899        let reservation = snapshot.pools[idx]
13900            .decommission
13901            .as_ref()
13902            .and_then(|info| info.capacity_reservation.as_ref())
13903            .filter(|reservation| reservation.admits_owner(owner, OffsetDateTime::now_utc()))
13904            .ok_or_else(|| decommission_capacity_blocked_error("scanner pause backlog retirement reservation is stale"))?;
13905        let mutation_id = decommission_capacity_version_mutation_id(owner, RUSTFS_META_BUCKET, version);
13906        if reservation.targets.iter().any(|target| {
13907            target.pending_mutation_id == Some(mutation_id)
13908                || target
13909                    .temporary_mutations
13910                    .iter()
13911                    .any(|mutation| mutation.mutation_id == mutation_id)
13912        }) {
13913            return Err(decommission_capacity_blocked_error(
13914                "scanner pause backlog retirement has an unresolved target capacity intent",
13915            ));
13916        }
13917        if snapshot.scanner_pause_backlog_pool_writable(idx) {
13918            return Err(Error::other("scanner pause backlog source still belongs to writable membership"));
13919        }
13920        let sets: Vec<_> = self
13921            .pools
13922            .iter()
13923            .enumerate()
13924            .filter(|(pool_index, _)| *pool_index == idx || snapshot.scanner_pause_backlog_pool_writable(*pool_index))
13925            .flat_map(|(_, pool)| pool.disk_set.iter().cloned())
13926            .collect();
13927        let mut replicas = read_scanner_pause_backlog_retirement_replicas(idx, set.set_index, sets.clone()).await?;
13928        if let Some(plan) = plan_scanner_pause_backlog_retirement(idx, &replicas)? {
13929            let expected_stable_record = plan.stable_record.clone();
13930            let mut phases = Vec::with_capacity(3);
13931            if let Some(record) = plan.seed_record {
13932                phases.push(("seed", record));
13933            }
13934            phases.push(("commit", plan.commit_record));
13935            phases.push(("stabilize", plan.stable_record));
13936            for (phase, record) in phases {
13937                if object_fence.is_lock_lost()
13938                    || pool_meta_guard.is_lock_lost()
13939                    || !reservation.admits_owner(owner, OffsetDateTime::now_utc())
13940                {
13941                    return Err(decommission_capacity_blocked_error(
13942                        "scanner pause backlog retirement fence expired during native repair",
13943                    ));
13944                }
13945                for read in replicas.iter().filter(|read| read.replica.pool_index != idx) {
13946                    ensure_external_decommission_target_admission(
13947                        &snapshot,
13948                        read.replica.pool_index,
13949                        DecommissionCapacityAdmission::ScannerBacklog,
13950                    )?;
13951                }
13952                let results = join_all(replicas.iter().filter(|read| read.replica.pool_index != idx).map(|read| {
13953                    let target = Arc::clone(&self.pools[read.replica.pool_index].disk_set[read.replica.set_index]);
13954                    let record = record.clone();
13955                    let object_fence = &object_fence;
13956                    let pool_meta_guard = &pool_meta_guard;
13957                    async move {
13958                        let mut opts = ObjectOptions {
13959                            no_lock: self.pools[0].disk_set[0].shares_namespace_lock_domain(&target).await,
13960                            ..Default::default()
13961                        };
13962                        object_fence.add_namespace_lock_fence(&mut opts);
13963                        opts.add_namespace_lock_guard(pool_meta_guard);
13964                        persist_native_scanner_pause_backlog_replica(target, record, read.preconditions(), opts, phase).await
13965                    }
13966                }))
13967                .await;
13968                // Every dispatched native CAS has finished before an error can
13969                // release the owned task's object and durable membership fences.
13970                for result in results {
13971                    result?;
13972                }
13973                replicas = read_scanner_pause_backlog_retirement_replicas(idx, set.set_index, sets.clone()).await?;
13974                if replicas
13975                    .iter()
13976                    .filter(|read| read.replica.pool_index != idx)
13977                    .any(|read| read.replica.data.as_deref() != Some(record.as_slice()))
13978                {
13979                    return Err(Error::other(
13980                        "scanner pause backlog native repair did not persist every surviving replica",
13981                    ));
13982                }
13983                if let Some(next) = plan_scanner_pause_backlog_retirement(idx, &replicas)?
13984                    && next.stable_record != expected_stable_record
13985                {
13986                    return Err(Error::other("scanner pause backlog native authority changed during repair"));
13987                }
13988            }
13989            if plan_scanner_pause_backlog_retirement(idx, &replicas)?.is_some() {
13990                return Err(Error::other("scanner pause backlog native repair is not fully committed and stable"));
13991            }
13992        }
13993        if object_fence.is_lock_lost()
13994            || pool_meta_guard.is_lock_lost()
13995            || !reservation.admits_owner(owner, OffsetDateTime::now_utc())
13996        {
13997            return Err(decommission_capacity_blocked_error(
13998                "scanner pause backlog retirement fence expired before cleanup",
13999            ));
14000        }
14001        let result = data_movement::cleanup_source_entry_if_unchanged(
14002            set,
14003            RUSTFS_META_BUCKET,
14004            SCANNER_PAUSE_BACKLOG_PATH,
14005            expected,
14006            &[],
14007            data_movement::SourceCleanupBucketFence {
14008                expected_incarnation_id: None,
14009                lifecycle_guard: None,
14010                namespace_lock_lost_signal: pool_meta_guard.lock_lost_signal(),
14011                object_mutation_fence: Some(&object_fence),
14012            },
14013            "scanner pause backlog retirement",
14014        )
14015        .await;
14016        match result {
14017            Ok(_) => Ok(DecommissionEntryAttemptOutcome::Complete),
14018            Err(data_movement::SourceCleanupError::SourceChanged) => Ok(DecommissionEntryAttemptOutcome::SourceChanged),
14019            Err(data_movement::SourceCleanupError::Storage(err)) => Err(err),
14020        }
14021    }
14022
14023    #[allow(clippy::too_many_arguments)]
14024    #[tracing::instrument(
14025        level = "trace",
14026        skip_all,
14027        fields(
14028            event = EVENT_DECOMMISSION_ENTRY,
14029            component = LOG_COMPONENT_ECSTORE,
14030            subsystem = LOG_SUBSYSTEM_POOLS,
14031            state = "processing",
14032            pool_index = idx,
14033            bucket = %bucket,
14034            object = %entry.name,
14035            generation = %generation,
14036        )
14037    )]
14038    async fn decommission_entry(
14039        self: &Arc<Self>,
14040        rx: CancellationToken,
14041        idx: usize,
14042        generation: OffsetDateTime,
14043        entry: MetaCacheEntry,
14044        bucket: String,
14045        set: Arc<SetDisks>,
14046        lifecycle_config: Option<BucketLifecycleConfiguration>,
14047        object_lock_config: Option<ObjectLockConfiguration>,
14048        replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
14049        expected_bucket_incarnation_id: Option<uuid::Uuid>,
14050        source_changed_exhaustions: Arc<AtomicUsize>,
14051    ) -> Result<()> {
14052        let mut counted_versions = HashSet::new();
14053
14054        for entry_attempt in 1..=DECOMMISSION_ENTRY_MAX_ATTEMPTS {
14055            let attempt_result = {
14056                let mut conflict_attempt = 0;
14057                loop {
14058                    let result = self
14059                        .decommission_entry_attempt(
14060                            rx.clone(),
14061                            idx,
14062                            generation,
14063                            entry.clone(),
14064                            bucket.clone(),
14065                            Arc::clone(&set),
14066                            lifecycle_config.clone(),
14067                            object_lock_config.clone(),
14068                            replication_config.clone(),
14069                            expected_bucket_incarnation_id,
14070                            entry_attempt,
14071                            source_changed_exhaustions.as_ref(),
14072                            &mut counted_versions,
14073                        )
14074                        .await;
14075                    let retry = result
14076                        .as_ref()
14077                        .err()
14078                        .and_then(|err| decommission_capacity_retry_kind(err, conflict_attempt));
14079                    let retry_attempt = match retry {
14080                        Some(DecommissionCapacityRetryKind::IntentConflict) => {
14081                            conflict_attempt += 1;
14082                            conflict_attempt
14083                        }
14084                        None => break result,
14085                    };
14086                    let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, retry_attempt);
14087                    if wait_decommission_retry_backoff(&rx, retry_delay).await {
14088                        decommission_cancel_signal_result(rx.is_cancelled())?;
14089                    }
14090                }
14091            };
14092            match attempt_result {
14093                Ok(DecommissionEntryAttemptOutcome::Complete) => return Ok(()),
14094                Ok(DecommissionEntryAttemptOutcome::SourceChanged) => {
14095                    let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, entry_attempt);
14096                    warn!(
14097                        event = EVENT_DECOMMISSION_ENTRY,
14098                        component = LOG_COMPONENT_ECSTORE,
14099                        subsystem = LOG_SUBSYSTEM_POOLS,
14100                        state = "source_changed_retry",
14101                        pool_index = idx,
14102                        bucket = %bucket,
14103                        object = %entry.name,
14104                        attempt = entry_attempt,
14105                        max_attempts = DECOMMISSION_ENTRY_MAX_ATTEMPTS,
14106                        retry_delay_ms = retry_delay.as_millis(),
14107                        "Decommission source changed during cleanup preflight; retrying entry"
14108                    );
14109                    if wait_decommission_retry_backoff(&rx, retry_delay).await {
14110                        decommission_cancel_signal_result(rx.is_cancelled())?;
14111                    }
14112                }
14113                Err(err) => return Err(err),
14114            }
14115        }
14116
14117        Err(Error::other(format!(
14118            "decommission entry retry loop ended without a terminal result for {bucket}/{}",
14119            entry.name
14120        )))
14121    }
14122
14123    #[allow(unused_assignments, clippy::too_many_arguments)]
14124    async fn decommission_entry_attempt(
14125        self: &Arc<Self>,
14126        rx: CancellationToken,
14127        idx: usize,
14128        generation: OffsetDateTime,
14129        entry: MetaCacheEntry,
14130        bucket: String,
14131        set: Arc<SetDisks>,
14132        lifecycle_config: Option<BucketLifecycleConfiguration>,
14133        object_lock_config: Option<ObjectLockConfiguration>,
14134        replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
14135        expected_bucket_incarnation_id: Option<uuid::Uuid>,
14136        entry_attempt: usize,
14137        source_changed_exhaustions: &AtomicUsize,
14138        counted_versions: &mut HashSet<(Option<uuid::Uuid>, bool)>,
14139    ) -> Result<DecommissionEntryAttemptOutcome> {
14140        debug!(
14141            event = EVENT_DECOMMISSION_ENTRY,
14142            component = LOG_COMPONENT_ECSTORE,
14143            subsystem = LOG_SUBSYSTEM_POOLS,
14144            state = "started",
14145            pool_index = idx,
14146            bucket = %bucket,
14147            object = %entry.name,
14148            attempt = entry_attempt,
14149            max_attempts = DECOMMISSION_ENTRY_MAX_ATTEMPTS,
14150            "Decommission entry started"
14151        );
14152        if entry.is_dir() {
14153            debug!(
14154                event = EVENT_DECOMMISSION_ENTRY,
14155                component = LOG_COMPONENT_ECSTORE,
14156                subsystem = LOG_SUBSYSTEM_POOLS,
14157                pool_index = idx,
14158                bucket = %bucket,
14159                object = %entry.name,
14160                state = "skipped_directory",
14161                "Decommission entry skipped directory"
14162            );
14163            return Ok(DecommissionEntryAttemptOutcome::Complete);
14164        }
14165        // Scanner caches describe their own erasure set and are rebuilt there.
14166        // Copying one onto another set can overwrite unrelated cache contents or
14167        // leave an unresolvable target-capacity intent after a conditional PUT.
14168        if is_decommission_set_local_usage_cache(&bucket, &entry.name) {
14169            return Ok(DecommissionEntryAttemptOutcome::Complete);
14170        }
14171        let durable_ilm_record = if bucket == RUSTFS_META_BUCKET {
14172            classify_durable_ilm_record(&entry.name)
14173                .map_err(|err| with_decommission_entry_context("durable_ilm_namespace", &bucket, &entry.name, err))?
14174        } else {
14175            None
14176        };
14177        if self.decommission_cancel_requested(idx, &rx).await {
14178            rx.cancel();
14179        }
14180        decommission_cancel_signal_result(rx.is_cancelled())?;
14181        self.ensure_decommission_generation_current(idx, generation).await?;
14182        self.ensure_decommission_runtime_capacity_available(idx, generation).await?;
14183        let capacity_owner = self.decommission_capacity_owner_for_worker(idx, generation).await?;
14184        let operation_gate = self.ctx.data_movement_operation_gate();
14185
14186        let bucket_incarnation_fence = match expected_bucket_incarnation_id {
14187            Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
14188            None => None,
14189        };
14190
14191        let mut fivs = load_decommission_entry_exact_versions(&set, &entry, &bucket, "file_info_versions").await?;
14192
14193        if data_movement::scanner_backlog::is_scanner_pause_backlog(&bucket, &entry.name) {
14194            let outcome = self
14195                .retire_scanner_pause_backlog_entry(rx, idx, generation, Arc::clone(&set), fivs.clone(), capacity_owner)
14196                .await?;
14197            if matches!(outcome, DecommissionEntryAttemptOutcome::Complete) {
14198                let mut pool_meta = self.pool_meta.write().await;
14199                ensure_decommission_generation(&pool_meta, idx, generation)?;
14200                if let Some(version) = fivs.versions.first()
14201                    && counted_versions.insert((version.version_id, false))
14202                {
14203                    count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), false)?;
14204                }
14205                track_decommission_current_object(&mut pool_meta, idx, &bucket, &entry.name)?;
14206                drop(pool_meta);
14207                self.track_decommission_entry_progress_stage(
14208                    idx,
14209                    generation,
14210                    &bucket,
14211                    &entry.name,
14212                    DECOMMISSION_STAGE_ENTRY_FINISHED,
14213                )
14214                .await?;
14215            }
14216            return Ok(outcome);
14217        }
14218
14219        let pending_mutations = if let Some(owner) = capacity_owner {
14220            self.pool_meta
14221                .read()
14222                .await
14223                .pools
14224                .get(owner.source_pool_index)
14225                .and_then(|pool| pool.decommission.as_ref())
14226                .and_then(|info| info.capacity_reservation.as_ref())
14227                .filter(|reservation| reservation.admits_cleanup_owner(owner))
14228                .map(|reservation| {
14229                    reservation
14230                        .targets
14231                        .iter()
14232                        .filter_map(|target| target.pending_mutation_id)
14233                        .collect::<HashSet<_>>()
14234                })
14235                .unwrap_or_default()
14236        } else {
14237            HashSet::new()
14238        };
14239        fivs.versions.sort_by_key(|version| {
14240            let mutation_id = capacity_owner.map(|owner| decommission_capacity_version_mutation_id(owner, &bucket, version));
14241            (
14242                mutation_id.is_none_or(|mutation_id| !pending_mutations.contains(&mutation_id)),
14243                version.mod_time.is_none(),
14244                std::cmp::Reverse(version.mod_time),
14245            )
14246        });
14247
14248        let mut decommissioned: usize = 0;
14249        let mut expired: usize = 0;
14250        let mut free_version_disposition = DecommissionFreeVersionDisposition::default();
14251        let mut cleanup_preflight_allowed_missing = Vec::new();
14252        let mut entry_blocked = false;
14253
14254        for version in fivs.versions.iter() {
14255            if self.decommission_cancel_requested(idx, &rx).await {
14256                rx.cancel();
14257            }
14258            decommission_cancel_signal_result(rx.is_cancelled())?;
14259
14260            if version.tier_free_version() {
14261                let version_id = version.version_id.map(|v| v.to_string());
14262                let mut migration_error = None;
14263                let mut migrated = false;
14264                let mut consumed = false;
14265                let mut capacity_failure = false;
14266                let mut version_attempt = 1;
14267                let mut target_busy_attempt: usize = 0;
14268                let mut target_permit = None;
14269                while version_attempt <= DECOMMISSION_VERSION_COPY_ATTEMPTS {
14270                    let result = self
14271                        .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
14272                            self.decommission_tiered_object(
14273                                bucket.as_str(),
14274                                &version.name,
14275                                version,
14276                                &decommission_capacity_owned_opts(
14277                                    decommission_remote_tiered_opts(
14278                                        version,
14279                                        version_id.clone(),
14280                                        idx,
14281                                        expected_bucket_incarnation_id,
14282                                    ),
14283                                    capacity_owner,
14284                                ),
14285                            )
14286                            .await
14287                        })
14288                        .await;
14289                    drop(target_permit.take());
14290                    if let Some(target_pool_index) = result.as_ref().err().and_then(decommission_capacity_target_gate_busy_index)
14291                    {
14292                        target_busy_attempt = target_busy_attempt.saturating_add(1);
14293                        let Some(permit) = self
14294                            .wait_decommission_target_gate_retry(
14295                                &rx,
14296                                idx,
14297                                generation,
14298                                target_pool_index,
14299                                capacity_owner,
14300                                set.as_ref(),
14301                                &entry,
14302                                &bucket,
14303                                version,
14304                                target_busy_attempt,
14305                            )
14306                            .await?
14307                        else {
14308                            return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
14309                        };
14310                        target_permit = Some(permit);
14311                        continue;
14312                    }
14313
14314                    match classify_decommission_free_version_attempt(result) {
14315                        DecommissionFreeVersionAttempt::Migrated => {
14316                            migrated = true;
14317                            migration_error = None;
14318                            break;
14319                        }
14320                        DecommissionFreeVersionAttempt::Consumed => {
14321                            consumed = true;
14322                            migration_error = None;
14323                            break;
14324                        }
14325                        DecommissionFreeVersionAttempt::CapacityFailure(err) => {
14326                            capacity_failure = true;
14327                            migration_error = Some(err);
14328                            break;
14329                        }
14330                        DecommissionFreeVersionAttempt::Retry(err) => {
14331                            migration_error = Some(err);
14332                            version_attempt += 1;
14333                        }
14334                    }
14335                }
14336
14337                if counted_versions.insert((version.version_id, version.deleted)) {
14338                    let mut pool_meta = self.pool_meta.write().await;
14339                    ensure_decommission_generation(&pool_meta, idx, generation)?;
14340                    if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, !migrated && !consumed) {
14341                        return Err(with_decommission_entry_context(
14342                            "count_decommission_item",
14343                            bucket.as_str(),
14344                            entry.name.as_str(),
14345                            err,
14346                        ));
14347                    }
14348                }
14349
14350                if migrated || consumed {
14351                    decommissioned += 1;
14352                    cleanup_preflight_allowed_missing.push(data_movement::source_cleanup_version_identity(version));
14353                }
14354                if migrated {
14355                    free_version_disposition.record_migrated();
14356                } else if consumed {
14357                    free_version_disposition.record_consumed();
14358                } else {
14359                    free_version_disposition.record_retained();
14360                }
14361
14362                debug!(
14363                    event = EVENT_DECOMMISSION_ENTRY,
14364                    component = LOG_COMPONENT_ECSTORE,
14365                    subsystem = LOG_SUBSYSTEM_POOLS,
14366                    pool_index = idx,
14367                    bucket = %bucket,
14368                    object = %version.name,
14369                    version_id = ?version_id,
14370                    result = ?migration_error,
14371                    reason = if migrated {
14372                        DECOMMISSION_FREE_VERSION_MIGRATED_REASON
14373                    } else if consumed {
14374                        DECOMMISSION_FREE_VERSION_CONSUMED_REASON
14375                    } else {
14376                        DECOMMISSION_FREE_VERSION_RETAINED_REASON
14377                    },
14378                    state = if migrated {
14379                        "free_version_migrated"
14380                    } else if consumed {
14381                        "free_version_consumed"
14382                    } else {
14383                        "free_version_retained"
14384                    },
14385                    "Decommission free-version disposition recorded"
14386                );
14387
14388                if capacity_failure {
14389                    return Err(with_decommission_entry_context(
14390                        "decommission_tier_free_version",
14391                        bucket.as_str(),
14392                        version.name.as_str(),
14393                        migration_error.expect("capacity failure must retain its error"),
14394                    ));
14395                }
14396
14397                if !migrated && !consumed {
14398                    break;
14399                }
14400                continue;
14401            }
14402
14403            if self
14404                .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
14405                    should_skip_lifecycle_for_data_movement(
14406                        self.clone(),
14407                        &bucket,
14408                        version,
14409                        lifecycle_config.as_ref(),
14410                        object_lock_config.as_ref(),
14411                        true,
14412                        &LcEventSrc::Decom,
14413                        None,
14414                    )
14415                    .await
14416                })
14417                .await
14418                .map_err(|err| with_decommission_entry_context("lifecycle_expiry", bucket.as_str(), version.name.as_str(), err))?
14419            {
14420                expired += 1;
14421                cleanup_preflight_allowed_missing.push(data_movement::source_cleanup_version_identity(version));
14422                continue;
14423            }
14424
14425            let remaining_versions = decommission_remaining_version_count(&fivs.versions, expired);
14426            if should_skip_decommission_delete_marker(version, remaining_versions, replication_config.is_some()) {
14427                //
14428                decommissioned += 1;
14429                debug!(
14430                    event = EVENT_DECOMMISSION_ENTRY,
14431                    component = LOG_COMPONENT_ECSTORE,
14432                    subsystem = LOG_SUBSYSTEM_POOLS,
14433                    pool_index = idx,
14434                    bucket = %bucket,
14435                    object = %version.name,
14436                    state = "skipped_delete_marker",
14437                    "Decommission delete marker skipped"
14438                );
14439                continue;
14440            }
14441
14442            let version_id = version.version_id.map(|v| v.to_string());
14443
14444            let mut ignore = false;
14445            let mut cleanup_ignored = false;
14446            let mut failure = false;
14447            let mut error = None;
14448            if version.deleted {
14449                let mut version_attempt = 1;
14450                let mut target_busy_attempt: usize = 0;
14451                let mut target_permit = None;
14452                while version_attempt <= DECOMMISSION_VERSION_COPY_ATTEMPTS {
14453                    let result = self
14454                        .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
14455                            self.delete_object(
14456                                bucket.as_str(),
14457                                &version.name,
14458                                decommission_capacity_owned_opts(
14459                                    decommission_delete_marker_opts(
14460                                        version,
14461                                        version_id.clone(),
14462                                        idx,
14463                                        expected_bucket_incarnation_id,
14464                                    ),
14465                                    capacity_owner,
14466                                ),
14467                            )
14468                            .await
14469                        })
14470                        .await;
14471                    drop(target_permit.take());
14472                    if let Some(target_pool_index) = result.as_ref().err().and_then(decommission_capacity_target_gate_busy_index)
14473                    {
14474                        target_busy_attempt = target_busy_attempt.saturating_add(1);
14475                        let Some(permit) = self
14476                            .wait_decommission_target_gate_retry(
14477                                &rx,
14478                                idx,
14479                                generation,
14480                                target_pool_index,
14481                                capacity_owner,
14482                                set.as_ref(),
14483                                &entry,
14484                                &bucket,
14485                                version,
14486                                target_busy_attempt,
14487                            )
14488                            .await?
14489                        else {
14490                            return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
14491                        };
14492                        target_permit = Some(permit);
14493                        continue;
14494                    }
14495                    #[cfg(test)]
14496                    let result = decommission_test_wrap_result(
14497                        "delete_marker_copy",
14498                        bucket.as_str(),
14499                        version.name.as_str(),
14500                        version_attempt,
14501                        result,
14502                    );
14503
14504                    match result {
14505                        Ok(_) => {
14506                            failure = false;
14507                            error = None;
14508                            break;
14509                        }
14510                        Err(err) if is_decommission_copy_cleanup_safe_error(&err) => {
14511                            warn!(
14512                                event = EVENT_DECOMMISSION_ENTRY,
14513                                component = LOG_COMPONENT_ECSTORE,
14514                                subsystem = LOG_SUBSYSTEM_POOLS,
14515                                state = "ignored_delete_marker_copy",
14516                                pool_index = idx,
14517                                bucket = %bucket,
14518                                object = %version.name,
14519                                version_id = ?version_id,
14520                                error = ?err,
14521                                "Decommission delete marker copy ignored"
14522                            );
14523                            ignore = true;
14524                            cleanup_ignored = true;
14525                            break;
14526                        }
14527                        Err(err) if is_decommission_target_capacity_error(&err) => {
14528                            return Err(with_decommission_entry_context(
14529                                "delete_marker_copy",
14530                                bucket.as_str(),
14531                                version.name.as_str(),
14532                                err,
14533                            ));
14534                        }
14535                        Err(err) => {
14536                            failure = true;
14537                            if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS {
14538                                error!(
14539                                    event = EVENT_DECOMMISSION_ENTRY,
14540                                    component = LOG_COMPONENT_ECSTORE,
14541                                    subsystem = LOG_SUBSYSTEM_POOLS,
14542                                    state = "delete_marker_copy_failed",
14543                                    pool_index = idx,
14544                                    bucket = %bucket,
14545                                    object = %version.name,
14546                                    version_id = ?version_id,
14547                                    attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
14548                                    error = ?err,
14549                                    "Decommission delete marker copy failed"
14550                                );
14551                                error = Some(err);
14552                                break;
14553                            }
14554
14555                            let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt);
14556                            warn!(
14557                                event = EVENT_DECOMMISSION_ENTRY,
14558                                component = LOG_COMPONENT_ECSTORE,
14559                                subsystem = LOG_SUBSYSTEM_POOLS,
14560                                state = "delete_marker_copy_retry",
14561                                pool_index = idx,
14562                                bucket = %bucket,
14563                                object = %version.name,
14564                                version_id = ?version_id,
14565                                attempt = version_attempt,
14566                                max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
14567                                retry_delay_ms = retry_delay.as_millis(),
14568                                error = ?err,
14569                                "Decommission delete marker copy failed; retrying"
14570                            );
14571                            error = Some(err);
14572                            if wait_decommission_retry_backoff(&rx, retry_delay).await {
14573                                decommission_cancel_signal_result(rx.is_cancelled())?;
14574                            }
14575                            version_attempt += 1;
14576                        }
14577                    }
14578                }
14579
14580                if ignore {
14581                    if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
14582                        decommissioned += 1;
14583                    }
14584                    debug!(
14585                        event = EVENT_DECOMMISSION_ENTRY,
14586                        component = LOG_COMPONENT_ECSTORE,
14587                        subsystem = LOG_SUBSYSTEM_POOLS,
14588                        pool_index = idx,
14589                        bucket = %bucket,
14590                        object = %version.name,
14591                        state = "ignored",
14592                        "Decommission entry ignored"
14593                    );
14594                    continue;
14595                }
14596
14597                if counted_versions.insert((version.version_id, version.deleted)) {
14598                    let mut pool_meta = self.pool_meta.write().await;
14599                    ensure_decommission_generation(&pool_meta, idx, generation)?;
14600                    if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, failure) {
14601                        return Err(with_decommission_entry_context(
14602                            "count_decommission_item",
14603                            bucket.as_str(),
14604                            entry.name.as_str(),
14605                            err,
14606                        ));
14607                    }
14608                }
14609
14610                if !failure {
14611                    decommissioned += 1;
14612                }
14613
14614                if !failure {
14615                    debug!(
14616                        event = EVENT_DECOMMISSION_ENTRY,
14617                        component = LOG_COMPONENT_ECSTORE,
14618                        subsystem = LOG_SUBSYSTEM_POOLS,
14619                        state = "delete_marker_copied",
14620                        pool_index = idx,
14621                        bucket = %bucket,
14622                        object = %version.name,
14623                        version_id = ?version_id,
14624                        result = ?error,
14625                        "Decommission delete marker copied"
14626                    );
14627                }
14628                continue;
14629            }
14630
14631            let mut version_attempt = 1;
14632            let mut target_busy_attempt: usize = 0;
14633            let mut target_permit = None;
14634            while version_attempt <= DECOMMISSION_VERSION_COPY_ATTEMPTS {
14635                if version.is_remote() {
14636                    let result = self
14637                        .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
14638                            self.decommission_tiered_object(
14639                                bucket.as_str(),
14640                                &version.name,
14641                                version,
14642                                &decommission_capacity_owned_opts(
14643                                    decommission_remote_tiered_opts(
14644                                        version,
14645                                        version_id.clone(),
14646                                        idx,
14647                                        expected_bucket_incarnation_id,
14648                                    ),
14649                                    capacity_owner,
14650                                ),
14651                            )
14652                            .await
14653                        })
14654                        .await;
14655                    drop(target_permit.take());
14656                    if let Some(target_pool_index) = result.as_ref().err().and_then(decommission_capacity_target_gate_busy_index)
14657                    {
14658                        target_busy_attempt = target_busy_attempt.saturating_add(1);
14659                        let Some(permit) = self
14660                            .wait_decommission_target_gate_retry(
14661                                &rx,
14662                                idx,
14663                                generation,
14664                                target_pool_index,
14665                                capacity_owner,
14666                                set.as_ref(),
14667                                &entry,
14668                                &bucket,
14669                                version,
14670                                target_busy_attempt,
14671                            )
14672                            .await?
14673                        else {
14674                            return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
14675                        };
14676                        target_permit = Some(permit);
14677                        continue;
14678                    }
14679                    #[cfg(test)]
14680                    let result = decommission_test_wrap_result(
14681                        "decommission_tiered_object",
14682                        bucket.as_str(),
14683                        version.name.as_str(),
14684                        version_attempt,
14685                        result,
14686                    );
14687
14688                    match result {
14689                        Ok(_) => {
14690                            failure = false;
14691                            error = None;
14692                        }
14693                        Err(err) if is_decommission_copy_cleanup_safe_error(&err) => {
14694                            ignore = true;
14695                            cleanup_ignored = true;
14696                        }
14697                        Err(err) if is_decommission_target_capacity_error(&err) => {
14698                            return Err(with_decommission_entry_context(
14699                                "decommission_tiered_object",
14700                                bucket.as_str(),
14701                                version.name.as_str(),
14702                                err,
14703                            ));
14704                        }
14705                        Err(err) => {
14706                            failure = true;
14707                            if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS {
14708                                error!(
14709                                    event = EVENT_DECOMMISSION_ENTRY,
14710                                    component = LOG_COMPONENT_ECSTORE,
14711                                    subsystem = LOG_SUBSYSTEM_POOLS,
14712                                    state = "tiered_copy_failed",
14713                                    pool_index = idx,
14714                                    bucket = %bucket,
14715                                    object = %version.name,
14716                                    version_id = ?version_id,
14717                                    attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
14718                                    error = ?err,
14719                                    "Decommission tiered version copy failed"
14720                                );
14721                                error = Some(err);
14722                            } else {
14723                                let retry_delay =
14724                                    decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt);
14725                                warn!(
14726                                    event = EVENT_DECOMMISSION_ENTRY,
14727                                    component = LOG_COMPONENT_ECSTORE,
14728                                    subsystem = LOG_SUBSYSTEM_POOLS,
14729                                    state = "tiered_copy_retry",
14730                                    pool_index = idx,
14731                                    bucket = %bucket,
14732                                    object = %version.name,
14733                                    version_id = ?version_id,
14734                                    attempt = version_attempt,
14735                                    max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
14736                                    retry_delay_ms = retry_delay.as_millis(),
14737                                    error = ?err,
14738                                    "Decommission tiered version copy failed; retrying"
14739                                );
14740                                error = Some(err);
14741                                if wait_decommission_retry_backoff(&rx, retry_delay).await {
14742                                    decommission_cancel_signal_result(rx.is_cancelled())?;
14743                                }
14744                                version_attempt += 1;
14745                                continue;
14746                            }
14747                        }
14748                    }
14749                    break;
14750                }
14751
14752                let bucket = bucket.clone();
14753
14754                let read_result = set
14755                    .get_object_reader(
14756                        bucket.as_str(),
14757                        &encode_dir_object(&version.name),
14758                        None,
14759                        HeaderMap::new(),
14760                        &decommission_object_migration_read_opts(version_id.clone()),
14761                    )
14762                    .await;
14763                #[cfg(test)]
14764                let read_result =
14765                    decommission_test_wrap_result("object_read", &bucket, &version.name, version_attempt, read_result);
14766                let rd = match read_result {
14767                    Ok(rd) => rd,
14768                    Err(err) => {
14769                        if is_err_object_not_found(&err) || is_err_version_not_found(&err) {
14770                            ignore = true;
14771                            cleanup_ignored = true;
14772                            break;
14773                        }
14774
14775                        failure = true;
14776                        if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS {
14777                            error!(
14778                                event = EVENT_DECOMMISSION_ENTRY,
14779                                component = LOG_COMPONENT_ECSTORE,
14780                                subsystem = LOG_SUBSYSTEM_POOLS,
14781                                state = "object_read_failed",
14782                                pool_index = idx,
14783                                bucket = %bucket,
14784                                object = %version.name,
14785                                version_id = ?version_id,
14786                                attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
14787                                error = ?err,
14788                                "Decommission source object read failed"
14789                            );
14790                            error = Some(err);
14791                            break;
14792                        }
14793
14794                        let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt);
14795                        warn!(
14796                            event = EVENT_DECOMMISSION_ENTRY,
14797                            component = LOG_COMPONENT_ECSTORE,
14798                            subsystem = LOG_SUBSYSTEM_POOLS,
14799                            state = "object_read_retry",
14800                            pool_index = idx,
14801                            bucket = %bucket,
14802                            object = %version.name,
14803                            version_id = ?version_id,
14804                            attempt = version_attempt,
14805                            max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
14806                            retry_delay_ms = retry_delay.as_millis(),
14807                            error = ?err,
14808                            "Decommission source object read failed; retrying"
14809                        );
14810                        error = Some(err);
14811                        if wait_decommission_retry_backoff(&rx, retry_delay).await {
14812                            decommission_cancel_signal_result(rx.is_cancelled())?;
14813                        }
14814                        version_attempt += 1;
14815                        continue;
14816                    }
14817                };
14818
14819                let bucket_name = bucket.clone();
14820                let object_name = rd.object_info.name.clone();
14821
14822                self.track_decommission_entry_progress_stage(
14823                    idx,
14824                    generation,
14825                    bucket_name.as_str(),
14826                    object_name.as_str(),
14827                    DECOMMISSION_STAGE_MIGRATE_OBJECT,
14828                )
14829                .await?;
14830
14831                let migrate_result = self
14832                    .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
14833                        self.clone()
14834                            .decommission_object(idx, bucket, rd, expected_bucket_incarnation_id, capacity_owner)
14835                            .await
14836                    })
14837                    .await;
14838                drop(target_permit.take());
14839                if let Some(target_pool_index) = migrate_result
14840                    .as_ref()
14841                    .err()
14842                    .and_then(decommission_capacity_target_gate_busy_index)
14843                {
14844                    target_busy_attempt = target_busy_attempt.saturating_add(1);
14845                    let Some(permit) = self
14846                        .wait_decommission_target_gate_retry(
14847                            &rx,
14848                            idx,
14849                            generation,
14850                            target_pool_index,
14851                            capacity_owner,
14852                            set.as_ref(),
14853                            &entry,
14854                            &bucket_name,
14855                            version,
14856                            target_busy_attempt,
14857                        )
14858                        .await?
14859                    else {
14860                        return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
14861                    };
14862                    target_permit = Some(permit);
14863                    continue;
14864                }
14865                #[cfg(test)]
14866                let migrate_result = decommission_test_wrap_result(
14867                    DECOMMISSION_STAGE_MIGRATE_OBJECT,
14868                    bucket_name.as_str(),
14869                    object_name.as_str(),
14870                    version_attempt,
14871                    migrate_result,
14872                );
14873
14874                if let Err(err) = migrate_result {
14875                    if is_decommission_copy_cleanup_safe_error(&err) {
14876                        ignore = true;
14877                        cleanup_ignored = true;
14878                        break;
14879                    }
14880
14881                    if is_decommission_target_capacity_error(&err) {
14882                        return Err(with_decommission_entry_context(
14883                            DECOMMISSION_STAGE_MIGRATE_OBJECT,
14884                            bucket_name.as_str(),
14885                            object_name.as_str(),
14886                            err,
14887                        ));
14888                    }
14889
14890                    failure = true;
14891                    if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS {
14892                        error!(
14893                            event = EVENT_DECOMMISSION_ENTRY,
14894                            component = LOG_COMPONENT_ECSTORE,
14895                            subsystem = LOG_SUBSYSTEM_POOLS,
14896                            state = "object_migration_failed",
14897                            pool_index = idx,
14898                            bucket = %bucket_name,
14899                            object = %object_name,
14900                            version = %version.name,
14901                            attempt = version_attempt,
14902                            max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
14903                            error = ?err,
14904                            "Decommission object migration failed"
14905                        );
14906                        error = Some(err);
14907                        break;
14908                    }
14909
14910                    let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt);
14911                    warn!(
14912                        event = EVENT_DECOMMISSION_ENTRY,
14913                        component = LOG_COMPONENT_ECSTORE,
14914                        subsystem = LOG_SUBSYSTEM_POOLS,
14915                        state = "object_migration_retry",
14916                        pool_index = idx,
14917                        bucket = %bucket_name,
14918                        object = %object_name,
14919                        version = %version.name,
14920                        attempt = version_attempt,
14921                        max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
14922                        retry_delay_ms = retry_delay.as_millis(),
14923                        error = ?err,
14924                        "Decommission object migration failed; retrying"
14925                    );
14926                    error = Some(err);
14927                    if wait_decommission_retry_backoff(&rx, retry_delay).await {
14928                        decommission_cancel_signal_result(rx.is_cancelled())?;
14929                    }
14930                    version_attempt += 1;
14931                    continue;
14932                }
14933
14934                warn!(
14935                    event = EVENT_DECOMMISSION_ENTRY,
14936                    component = LOG_COMPONENT_ECSTORE,
14937                    subsystem = LOG_SUBSYSTEM_POOLS,
14938                    pool_index = idx,
14939                    bucket = %bucket_name,
14940                    object = %object_name,
14941                    version = %version.name,
14942                    state = "object_migrated",
14943                    "Decommission object migrated"
14944                );
14945
14946                failure = false;
14947                break;
14948            }
14949
14950            if ignore {
14951                if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
14952                    decommissioned += 1;
14953                }
14954                debug!(
14955                    event = EVENT_DECOMMISSION_ENTRY,
14956                    component = LOG_COMPONENT_ECSTORE,
14957                    subsystem = LOG_SUBSYSTEM_POOLS,
14958                    pool_index = idx,
14959                    bucket = %bucket,
14960                    object = %version.name,
14961                    state = "ignored",
14962                    "Decommission entry ignored"
14963                );
14964                continue;
14965            }
14966
14967            if counted_versions.insert((version.version_id, version.deleted)) {
14968                let mut pool_meta = self.pool_meta.write().await;
14969                ensure_decommission_generation(&pool_meta, idx, generation)?;
14970                if let Err(err) = count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), failure) {
14971                    return Err(with_decommission_entry_context(
14972                        "count_decommission_item",
14973                        bucket.as_str(),
14974                        entry.name.as_str(),
14975                        err,
14976                    ));
14977                }
14978            }
14979
14980            if failure {
14981                break;
14982            }
14983
14984            if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
14985                decommissioned += 1;
14986            }
14987        }
14988
14989        if free_version_disposition.total() > 0 {
14990            debug!(
14991                event = EVENT_DECOMMISSION_ENTRY,
14992                component = LOG_COMPONENT_ECSTORE,
14993                subsystem = LOG_SUBSYSTEM_POOLS,
14994                pool_index = idx,
14995                bucket = %bucket,
14996                object = %entry.name,
14997                free_versions_migrated = free_version_disposition.migrated,
14998                free_versions_consumed = free_version_disposition.consumed,
14999                free_versions_retained = free_version_disposition.retained,
15000                free_versions_total = free_version_disposition.total(),
15001                reason = DECOMMISSION_FREE_VERSION_DISPOSITION_REASON,
15002                state = "free_version_disposition",
15003                "Decommission free-version disposition summary"
15004            );
15005        }
15006
15007        if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) && durable_ilm_record.is_none()
15008        {
15009            if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
15010                return Err(Error::other("decommission bucket incarnation fence was lost before source cleanup"));
15011            }
15012            decommission_cancel_signal_result(rx.is_cancelled())?;
15013            self.ensure_decommission_generation_current(idx, generation).await?;
15014
15015            self.track_decommission_entry_progress_stage(
15016                idx,
15017                generation,
15018                bucket.as_str(),
15019                entry.name.as_str(),
15020                DECOMMISSION_STAGE_CLEANUP_PREFLIGHT,
15021            )
15022            .await?;
15023
15024            self.track_decommission_entry_progress_stage(
15025                idx,
15026                generation,
15027                bucket.as_str(),
15028                entry.name.as_str(),
15029                DECOMMISSION_STAGE_SOURCE_CLEANUP,
15030            )
15031            .await?;
15032
15033            #[cfg(test)]
15034            run_decommission_cleanup_mutation_hook(bucket.as_str(), entry.name.as_str(), entry_attempt).await;
15035
15036            let source_cleanup_mutation_fence = self
15037                .acquire_decommission_source_cleanup_fence(bucket.as_str(), entry.name.as_str(), set.as_ref())
15038                .await?;
15039            let cleanup_result = self
15040                .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
15041                    data_movement::cleanup_source_entry_if_unchanged(
15042                        set.clone(),
15043                        bucket.as_str(),
15044                        entry.name.as_str(),
15045                        &fivs,
15046                        &cleanup_preflight_allowed_missing,
15047                        data_movement::SourceCleanupBucketFence {
15048                            expected_incarnation_id: expected_bucket_incarnation_id,
15049                            lifecycle_guard: bucket_incarnation_fence
15050                                .as_ref()
15051                                .and_then(|guard| guard.namespace_lock_guard()),
15052                            namespace_lock_lost_signal: None,
15053                            object_mutation_fence: Some(&source_cleanup_mutation_fence),
15054                        },
15055                        "decommission",
15056                    )
15057                    .await
15058                })
15059                .await;
15060            match cleanup_result {
15061                Ok(_) => {}
15062                Err(data_movement::SourceCleanupError::Storage(err)) => {
15063                    resolve_decommission_entry_cleanup_delete_result(
15064                        Err::<(), Error>(err),
15065                        bucket.as_str(),
15066                        entry.name.as_str(),
15067                    )?;
15068                }
15069                Err(data_movement::SourceCleanupError::SourceChanged) if entry_attempt < DECOMMISSION_ENTRY_MAX_ATTEMPTS => {
15070                    return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
15071                }
15072                Err(data_movement::SourceCleanupError::SourceChanged) => {
15073                    let exhausted_entries = source_changed_exhaustions.fetch_add(1, Ordering::Relaxed) + 1;
15074                    if should_fail_decommission_pool_after_exhausted_source_changed(exhausted_entries) {
15075                        return Err(Error::other(format!(
15076                            "decommission source cleanup retries exhausted for {}/{} on all {} attempts; exhausted entries exceed pool limit {}",
15077                            bucket, entry.name, DECOMMISSION_ENTRY_MAX_ATTEMPTS, DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT
15078                        )));
15079                    }
15080
15081                    {
15082                        let mut pool_meta = self.pool_meta.write().await;
15083                        ensure_decommission_generation(&pool_meta, idx, generation)?;
15084                        count_decommission_item(&mut pool_meta, idx, 0, true).map_err(|err| {
15085                            with_decommission_entry_context(
15086                                "count_source_changed_exhaustion",
15087                                bucket.as_str(),
15088                                entry.name.as_str(),
15089                                err,
15090                            )
15091                        })?;
15092                    }
15093
15094                    error!(
15095                        event = EVENT_DECOMMISSION_ENTRY,
15096                        component = LOG_COMPONENT_ECSTORE,
15097                        subsystem = LOG_SUBSYSTEM_POOLS,
15098                        state = "source_cleanup_exhausted",
15099                        pool_index = idx,
15100                        bucket = %bucket,
15101                        object = %entry.name,
15102                        attempts = DECOMMISSION_ENTRY_MAX_ATTEMPTS,
15103                        exhausted_entries,
15104                        exhaustion_limit = DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT,
15105                        "Decommission source cleanup retries exhausted; source retained and entry marked failed"
15106                    );
15107                    entry_blocked = true;
15108                }
15109            }
15110        } else if durable_ilm_record.is_some() {
15111            debug!(
15112                event = EVENT_DECOMMISSION_ENTRY,
15113                component = LOG_COMPONENT_ECSTORE,
15114                subsystem = LOG_SUBSYSTEM_POOLS,
15115                pool_index = idx,
15116                bucket = %bucket,
15117                object = %entry.name,
15118                state = "retained_for_final_verification",
15119                "Decommission durable ILM source retained for final verification"
15120            );
15121        } else if decommissioned != fivs.versions.len() || expired > 0 {
15122            warn!(
15123                event = EVENT_DECOMMISSION_ENTRY,
15124                component = LOG_COMPONENT_ECSTORE,
15125                subsystem = LOG_SUBSYSTEM_POOLS,
15126                pool_index = idx,
15127                bucket = %bucket,
15128                object = %entry.name,
15129                decommissioned,
15130                total_versions = fivs.versions.len(),
15131                expired,
15132                state = "source_retained",
15133                "Decommission source object retained"
15134            );
15135        }
15136
15137        let should_save_progress = {
15138            let mut pool_meta = self.pool_meta.write().await;
15139            ensure_decommission_generation(&pool_meta, idx, generation)?;
15140
15141            if let Err(err) = track_decommission_current_object(&mut pool_meta, idx, bucket.as_str(), entry.name.as_str()) {
15142                return Err(with_decommission_entry_context(
15143                    "track_decommission_current_object",
15144                    bucket.as_str(),
15145                    entry.name.as_str(),
15146                    err,
15147                ));
15148            }
15149
15150            match resolve_decommission_update_after_result(pool_meta.update_after(idx, DECOMMISSION_PROGRESS_SAVE_INTERVAL)) {
15151                Ok(ok) => ok,
15152                Err(err) => {
15153                    return Err(with_decommission_entry_context("update_after", bucket.as_str(), entry.name.as_str(), err));
15154                }
15155            }
15156        };
15157
15158        self.track_decommission_entry_progress_stage(
15159            idx,
15160            generation,
15161            bucket.as_str(),
15162            entry.name.as_str(),
15163            DECOMMISSION_STAGE_ENTRY_FINISHED,
15164        )
15165        .await?;
15166
15167        if should_save_progress {
15168            match self.save_decommission_progress_checkpoint(idx, generation).await {
15169                Ok(true) => {
15170                    if let Some(notification_sys) = runtime_sources::notification_sys()
15171                        && let Err(err) = resolve_decommission_entry_reload_result(
15172                            notification_sys.reload_pool_meta().await,
15173                            bucket.as_str(),
15174                            entry.name.as_str(),
15175                        )
15176                    {
15177                        warn!("{err}");
15178                    }
15179                }
15180                Ok(false) => {}
15181                Err(err) => {
15182                    if let Some(err) = resolve_decommission_progress_save_result(Err(err)) {
15183                        warn!(
15184                            event = EVENT_DECOMMISSION_ENTRY,
15185                            component = LOG_COMPONENT_ECSTORE,
15186                            subsystem = LOG_SUBSYSTEM_POOLS,
15187                            pool_index = idx,
15188                            bucket = %bucket,
15189                            object = %entry.name,
15190                            state = "progress_save_failed",
15191                            error = %err,
15192                            "Decommission progress save failed; continuing and will retry at the next checkpoint"
15193                        );
15194                    }
15195                }
15196            }
15197        }
15198
15199        debug!(
15200            event = EVENT_DECOMMISSION_ENTRY,
15201            component = LOG_COMPONENT_ECSTORE,
15202            subsystem = LOG_SUBSYSTEM_POOLS,
15203            state = if entry_blocked { "blocked" } else { "completed" },
15204            pool_index = idx,
15205            bucket = %bucket,
15206            object = %entry.name,
15207            "Decommission entry finished"
15208        );
15209        Ok(DecommissionEntryAttemptOutcome::Complete)
15210    }
15211
15212    #[cfg(test)]
15213    pub(crate) async fn decommission_entry_for_test(
15214        self: &Arc<Self>,
15215        idx: usize,
15216        entry: MetaCacheEntry,
15217        bucket: String,
15218        set: Arc<SetDisks>,
15219    ) -> Result<()> {
15220        self.decommission_entry_with_retry_state_for_test(
15221            CancellationToken::new(),
15222            idx,
15223            entry,
15224            bucket,
15225            set,
15226            None,
15227            Arc::new(AtomicUsize::new(0)),
15228        )
15229        .await
15230    }
15231
15232    #[cfg(test)]
15233    #[allow(clippy::too_many_arguments)]
15234    pub(crate) async fn decommission_entry_with_retry_state_for_test(
15235        self: &Arc<Self>,
15236        rx: CancellationToken,
15237        idx: usize,
15238        entry: MetaCacheEntry,
15239        bucket: String,
15240        set: Arc<SetDisks>,
15241        expected_bucket_incarnation_id: Option<uuid::Uuid>,
15242        source_changed_exhaustions: Arc<AtomicUsize>,
15243    ) -> Result<()> {
15244        {
15245            let mut cancelers = self.decommission_cancelers.write().await;
15246            if cancelers.get(idx).and_then(Option::as_ref).is_none()
15247                && let Some(slot) = cancelers.get_mut(idx)
15248            {
15249                *slot = Some(DecommissionCanceler::new(CancellationToken::new()));
15250            }
15251        }
15252        let needs_capacity_reservation = {
15253            let pool_meta = self.pool_meta.read().await;
15254            pool_meta
15255                .pools
15256                .get(idx)
15257                .and_then(|pool| pool.decommission.as_ref())
15258                .and_then(|info| info.capacity_reservation.as_ref())
15259                .is_some_and(|reservation| !reservation.active())
15260        };
15261        if needs_capacity_reservation {
15262            let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
15263            let mut pool_meta = self.pool_meta.write().await;
15264            let version = pool_meta.version;
15265            pool_meta.version = POOL_META_VERSION;
15266            recover_decommission_capacity_reservations(
15267                &mut pool_meta,
15268                &capacity_infos,
15269                OffsetDateTime::now_utc(),
15270                crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof().is_some(),
15271            )?;
15272            pool_meta.version = version;
15273        }
15274        let generation = self.active_decommission_generation(idx).await?;
15275        self.decommission_entry(
15276            rx,
15277            idx,
15278            generation,
15279            entry,
15280            bucket,
15281            set,
15282            None,
15283            None,
15284            None,
15285            expected_bucket_incarnation_id,
15286            source_changed_exhaustions,
15287        )
15288        .await
15289    }
15290
15291    #[cfg(all(test, feature = "test-util"))]
15292    pub(crate) async fn decommission_entry_for_test_with_bucket_incarnation(
15293        self: &Arc<Self>,
15294        idx: usize,
15295        entry: MetaCacheEntry,
15296        bucket: String,
15297        set: Arc<SetDisks>,
15298    ) -> Result<()> {
15299        let expected_bucket_incarnation_id = if is_meta_bucketname(&bucket) {
15300            None
15301        } else {
15302            Some(self.bucket_incarnation_id_from_disk(&bucket).await?)
15303        };
15304        let generation = self.active_decommission_generation(idx).await?;
15305        self.decommission_entry(
15306            CancellationToken::new(),
15307            idx,
15308            generation,
15309            entry,
15310            bucket,
15311            set,
15312            None,
15313            None,
15314            None,
15315            expected_bucket_incarnation_id,
15316            Arc::new(AtomicUsize::new(0)),
15317        )
15318        .await
15319    }
15320
15321    #[tracing::instrument(skip(self, rx))]
15322    async fn decommission_pool(
15323        self: &Arc<Self>,
15324        rx: CancellationToken,
15325        idx: usize,
15326        pool: Arc<Sets>,
15327        bi: DecomBucketInfo,
15328        entry_budget: Arc<Semaphore>,
15329        source_changed_exhaustions: Arc<AtomicUsize>,
15330    ) -> Result<()> {
15331        let entry_error = Arc::new(tokio::sync::Mutex::new(None::<Error>));
15332        let generation = self.active_decommission_generation(idx).await?;
15333        let mut listing_workers = Vec::with_capacity(pool.disk_set.len());
15334
15335        let mut lifecycle_config = None;
15336        let mut object_lock_config = None;
15337        let mut replication_config = None;
15338        let expected_bucket_incarnation_id = if bi.name == RUSTFS_META_BUCKET {
15339            None
15340        } else {
15341            Some(self.bucket_incarnation_id_from_disk(&bi.name).await?)
15342        };
15343
15344        if bi.name != RUSTFS_META_BUCKET {
15345            let _ = resolve_decommission_optional_bucket_config_result(
15346                &bi.name,
15347                "versioning",
15348                BucketVersioningSys::get_in(&self.ctx, &bi.name).await,
15349            )?;
15350            let expiry_configs = get_expiry_configs(self, &bi.name).await?;
15351            lifecycle_config = expiry_configs.lifecycle.map(|config| (*config).clone());
15352            object_lock_config = expiry_configs.object_lock.map(|config| (*config).clone());
15353            replication_config = resolve_decommission_optional_bucket_config_result(
15354                &bi.name,
15355                "replication",
15356                metadata_sys::get_replication_config_in(&self.ctx, &bi.name).await,
15357            )?;
15358        }
15359
15360        for (set_idx, set) in pool.disk_set.iter().enumerate() {
15361            debug!(
15362                event = EVENT_DECOMMISSION_BUCKET,
15363                component = LOG_COMPONENT_ECSTORE,
15364                subsystem = LOG_SUBSYSTEM_POOLS,
15365                pool_index = idx,
15366                set_index = set_idx,
15367                bucket = %bi.name,
15368                state = "listing_worker_started",
15369                "Decommission listing worker started"
15370            );
15371
15372            let set = set.clone();
15373            let store = Arc::clone(self);
15374            let rx_clone = rx.clone();
15375            let bi_clone = bi.clone();
15376            let lifecycle_config = lifecycle_config.clone();
15377            let object_lock_config = object_lock_config.clone();
15378            let replication_config = replication_config.clone();
15379            let entry_budget = entry_budget.clone();
15380            let source_changed_exhaustions = Arc::clone(&source_changed_exhaustions);
15381            let entry_error = entry_error.clone();
15382            let worker = tokio::spawn(async move {
15383                store
15384                    .decommission_set(
15385                        rx_clone,
15386                        idx,
15387                        set_idx,
15388                        generation,
15389                        set,
15390                        bi_clone,
15391                        lifecycle_config,
15392                        object_lock_config,
15393                        replication_config,
15394                        expected_bucket_incarnation_id,
15395                        source_changed_exhaustions,
15396                        entry_budget,
15397                        entry_error,
15398                    )
15399                    .await
15400            });
15401            listing_workers.push((set_idx, worker));
15402        }
15403
15404        debug!(
15405            event = EVENT_DECOMMISSION_BUCKET,
15406            component = LOG_COMPONENT_ECSTORE,
15407            subsystem = LOG_SUBSYSTEM_POOLS,
15408            pool_index = idx,
15409            bucket = %bi.name,
15410            state = "waiting_for_workers",
15411            "Decommission waiting for workers"
15412        );
15413
15414        let mut listing_worker_error = None;
15415        for (set_id, worker) in listing_workers {
15416            if let Err(err) = resolve_decommission_listing_worker_result(set_id, worker.await) {
15417                rx.cancel();
15418                if listing_worker_error.is_none() {
15419                    listing_worker_error = Some(err);
15420                }
15421            }
15422        }
15423
15424        if let Some(err) = listing_worker_error {
15425            return Err(err);
15426        }
15427
15428        if let Some(err) = entry_error.lock().await.clone() {
15429            return Err(err);
15430        }
15431
15432        if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) {
15433            warn!(
15434                event = EVENT_DECOMMISSION_BUCKET,
15435                component = LOG_COMPONENT_ECSTORE,
15436                subsystem = LOG_SUBSYSTEM_POOLS,
15437                pool_index = idx,
15438                bucket = %bi.name,
15439                state = "cancelled_after_wait",
15440                error = %err,
15441                "Decommission bucket cancelled after wait"
15442            );
15443            return Err(err);
15444        }
15445
15446        debug!(
15447            event = EVENT_DECOMMISSION_BUCKET,
15448            component = LOG_COMPONENT_ECSTORE,
15449            subsystem = LOG_SUBSYSTEM_POOLS,
15450            pool_index = idx,
15451            bucket = %bi.name,
15452            state = "completed",
15453            "Decommission bucket completed"
15454        );
15455
15456        Ok(())
15457    }
15458
15459    #[cfg(all(test, feature = "test-util"))]
15460    pub(crate) async fn decommission_pool_for_test(
15461        self: &Arc<Self>,
15462        rx: CancellationToken,
15463        idx: usize,
15464        pool: Arc<Sets>,
15465        bucket: DecomBucketInfo,
15466    ) -> Result<()> {
15467        self.decommission_pool(
15468            rx,
15469            idx,
15470            pool,
15471            bucket,
15472            Arc::new(Semaphore::new(decommission_entry_concurrency_limit())),
15473            Arc::new(AtomicUsize::new(0)),
15474        )
15475        .await
15476    }
15477
15478    #[tracing::instrument(skip(self, canceler))]
15479    pub async fn do_decommission_in_routine(
15480        self: &Arc<Self>,
15481        canceler: DecommissionCanceler,
15482        idx: usize,
15483        entry_budget: Arc<Semaphore>,
15484    ) -> Result<()> {
15485        let rx = canceler.token().clone();
15486        self.run_decommission_in_routine(rx, idx, &canceler, entry_budget).await
15487    }
15488
15489    async fn run_decommission_in_routine(
15490        self: &Arc<Self>,
15491        rx: CancellationToken,
15492        idx: usize,
15493        canceler: &DecommissionCanceler,
15494        entry_budget: Arc<Semaphore>,
15495    ) -> Result<()> {
15496        self.ensure_pool_meta_side_effects_safe("decommission cannot run while pool metadata requires recovery")
15497            .await?;
15498        let generation = match self.promote_queued_decommission(idx, canceler).await {
15499            Ok(generation) => generation,
15500            Err(Error::OperationCanceled) => return Ok(()),
15501            Err(err) if is_decommission_capacity_blocked_error(&err) || is_decommission_target_capacity_error(&err) => {
15502                if let Err(pause_err) = self.pause_decommission_for_capacity(idx, &err).await {
15503                    return Err(decommission_capacity_blocked_error(format!(
15504                        "failed to persist paused state for pool {idx}: {pause_err}"
15505                    )));
15506                }
15507                return Ok(());
15508            }
15509            Err(err) => {
15510                resolve_decommission_terminal_mark_after_error_result(
15511                    self.decommission_failed_for_operation(idx, canceler).await,
15512                    idx,
15513                    &err,
15514                )?;
15515                return Err(err);
15516            }
15517        };
15518        if rx.is_cancelled() {
15519            let already_canceled = {
15520                let pool_meta = self.pool_meta.read().await;
15521                should_skip_canceled_decommission_routine(true, pool_meta.pools.get(idx))
15522            };
15523            if already_canceled {
15524                warn!(
15525                    event = EVENT_DECOMMISSION_STATE,
15526                    component = LOG_COMPONENT_ECSTORE,
15527                    subsystem = LOG_SUBSYSTEM_POOLS,
15528                    pool_index = idx,
15529                    state = "canceled_preserved",
15530                    "Decommission routine skipped because pool is already canceled"
15531                );
15532                return Ok(());
15533            }
15534            if let Err(err) = self.decommission_cancel_for_operation(idx, canceler).await {
15535                resolve_decommission_terminal_mark_after_error_result(
15536                    self.decommission_failed_for_operation(idx, canceler).await,
15537                    idx,
15538                    &err,
15539                )?;
15540                return Err(err);
15541            }
15542            return Ok(());
15543        }
15544        let result = self
15545            .decommission_in_background(rx.clone(), idx, generation, entry_budget)
15546            .await;
15547
15548        if let Err(err) = &result
15549            && (is_decommission_capacity_blocked_error(err) || is_decommission_target_capacity_error(err))
15550        {
15551            if let Err(pause_err) = self.pause_decommission_for_capacity(idx, err).await {
15552                return Err(decommission_capacity_blocked_error(format!(
15553                    "failed to persist paused state for pool {idx}: {pause_err}"
15554                )));
15555            }
15556            return Ok(());
15557        }
15558
15559        let (final_state, canceled, cmd_line) = {
15560            let pool_meta = self.pool_meta.read().await;
15561            let Some(pool) = pool_meta.pools.get(idx) else {
15562                error!(
15563                    event = EVENT_DECOMMISSION_STATE,
15564                    component = LOG_COMPONENT_ECSTORE,
15565                    subsystem = LOG_SUBSYSTEM_POOLS,
15566                    pool_index = idx,
15567                    state = "pool_metadata_missing",
15568                    "Decommission pool metadata missing"
15569                );
15570                return Err(Error::other(format!(
15571                    "failed to resolve decommission final state: pool metadata missing for idx {idx}"
15572                )));
15573            };
15574
15575            let (final_state, canceled) = if let Some(info) = &pool.decommission {
15576                (
15577                    determine_decommission_final_state(info.items_decommission_failed, info.canceled),
15578                    info.canceled,
15579                )
15580            } else {
15581                (DecommissionFinalState::Failed, false)
15582            };
15583            let cmd_line = pool.cmd_line.clone();
15584            (final_state, canceled, cmd_line)
15585        };
15586
15587        if let Err(err) = result {
15588            error!(
15589                event = EVENT_DECOMMISSION_STATE,
15590                component = LOG_COMPONENT_ECSTORE,
15591                subsystem = LOG_SUBSYSTEM_POOLS,
15592                pool_index = idx,
15593                state = "background_failed",
15594                error = ?err,
15595                "Decommission background routine failed"
15596            );
15597
15598            if should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
15599                warn!(
15600                    event = EVENT_DECOMMISSION_STATE,
15601                    component = LOG_COMPONENT_ECSTORE,
15602                    subsystem = LOG_SUBSYSTEM_POOLS,
15603                    pool_index = idx,
15604                    cmd_line = %cmd_line,
15605                    state = "cancelled_preserved",
15606                    "Decommission cancelled; preserving canceled state"
15607                );
15608                return Ok(());
15609            }
15610
15611            resolve_decommission_terminal_mark_after_error_result(
15612                self.decommission_failed_for_operation(idx, canceler).await,
15613                idx,
15614                &err,
15615            )?;
15616            warn!(
15617                event = EVENT_DECOMMISSION_STATE,
15618                component = LOG_COMPONENT_ECSTORE,
15619                subsystem = LOG_SUBSYSTEM_POOLS,
15620                pool_index = idx,
15621                state = "marked_failed",
15622                "Decommission marked failed"
15623            );
15624
15625            return Ok(());
15626        }
15627
15628        debug!(
15629            event = EVENT_DECOMMISSION_STATE,
15630            component = LOG_COMPONENT_ECSTORE,
15631            subsystem = LOG_SUBSYSTEM_POOLS,
15632            pool_index = idx,
15633            state = "background_complete",
15634            "Decommission background routine completed"
15635        );
15636
15637        if should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
15638            warn!(
15639                event = EVENT_DECOMMISSION_STATE,
15640                component = LOG_COMPONENT_ECSTORE,
15641                subsystem = LOG_SUBSYSTEM_POOLS,
15642                pool_index = idx,
15643                cmd_line = %cmd_line,
15644                state = "terminal_state_preserved",
15645                "Decommission terminal state preserved after cancellation"
15646            );
15647            return Ok(());
15648        }
15649
15650        match final_state {
15651            DecommissionFinalState::Complete => {
15652                debug!(
15653                    event = EVENT_DECOMMISSION_STATE,
15654                    component = LOG_COMPONENT_ECSTORE,
15655                    subsystem = LOG_SUBSYSTEM_POOLS,
15656                    pool_index = idx,
15657                    cmd_line = %cmd_line,
15658                    state = "verifying_completion",
15659                    "Decommission completion verification started"
15660                );
15661                let verified_unresolved_entries = match self.check_after_decommission(idx, &rx, generation).await {
15662                    Ok(verified_entries) => verified_entries,
15663                    Err(err) => {
15664                        if is_err_operation_canceled(&err) {
15665                            return Err(err);
15666                        }
15667                        resolve_decommission_terminal_mark_result(
15668                            self.decommission_failed_for_operation(idx, canceler).await,
15669                            "failed",
15670                            &cmd_line,
15671                        )?;
15672                        return Err(Error::other(format!(
15673                            "failed to finalize decommission for pool {cmd_line}: post-check failed: {err}"
15674                        )));
15675                    }
15676                };
15677                info!(
15678                    event = EVENT_DECOMMISSION_STATE,
15679                    component = LOG_COMPONENT_ECSTORE,
15680                    subsystem = LOG_SUBSYSTEM_POOLS,
15681                    pool_index = idx,
15682                    cmd_line = %cmd_line,
15683                    state = "marking_completed",
15684                    "Decommission marking completed state"
15685                );
15686                if let Err(err) = self
15687                    .complete_decommission_for_operation(idx, canceler, generation, verified_unresolved_entries)
15688                    .await
15689                {
15690                    resolve_decommission_terminal_mark_result(
15691                        self.decommission_failed_for_operation(idx, canceler).await,
15692                        "failed",
15693                        &cmd_line,
15694                    )?;
15695                    return Err(Error::other(format!("failed to finalize decommission for pool {cmd_line}: {err}")));
15696                }
15697            }
15698            DecommissionFinalState::Failed => {
15699                warn!(
15700                    event = EVENT_DECOMMISSION_STATE,
15701                    component = LOG_COMPONENT_ECSTORE,
15702                    subsystem = LOG_SUBSYSTEM_POOLS,
15703                    pool_index = idx,
15704                    cmd_line = %cmd_line,
15705                    state = "marking_failed",
15706                    "Decommission marking failed state"
15707                );
15708                resolve_decommission_terminal_mark_result(
15709                    self.decommission_failed_for_operation(idx, canceler).await,
15710                    "failed",
15711                    &cmd_line,
15712                )?;
15713            }
15714        }
15715
15716        info!(
15717            event = EVENT_DECOMMISSION_STATE,
15718            component = LOG_COMPONENT_ECSTORE,
15719            subsystem = LOG_SUBSYSTEM_POOLS,
15720            pool_index = idx,
15721            cmd_line = %cmd_line,
15722            state = "completed",
15723            "Decommission completed"
15724        );
15725        Ok(())
15726    }
15727
15728    #[tracing::instrument(skip(self))]
15729    pub async fn decommission_failed(&self, idx: usize) -> Result<()> {
15730        self.decommission_failed_with_owner(idx, None).await
15731    }
15732
15733    async fn decommission_failed_for_operation(&self, idx: usize, owner: &DecommissionCanceler) -> Result<()> {
15734        self.decommission_failed_with_owner(idx, Some(owner)).await
15735    }
15736
15737    async fn decommission_failed_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> {
15738        self.decommission_failed_with_owner_and_save(idx, owner, self.save_current_pool_meta(&[idx]))
15739            .await
15740    }
15741
15742    async fn decommission_failed_with_owner_and_save<SaveFuture>(
15743        &self,
15744        idx: usize,
15745        owner: Option<&DecommissionCanceler>,
15746        save_pool_meta: SaveFuture,
15747    ) -> Result<()>
15748    where
15749        SaveFuture: Future<Output = Result<()>>,
15750    {
15751        ensure_decommission_terminal_operation_supported(self.single_pool(), "mark decommission failed")?;
15752        let _start_guard = self.start_gate.lock().await;
15753        let movement_gate = self.ctx.data_movement_operation_gate();
15754        let _movement_guard = movement_gate.write().await;
15755
15756        // Lock order: movement gate, rebalance_meta, decommission_cancelers,
15757        // then pool_meta. Holding both state locks makes owner validation and
15758        // the terminal transition one atomic operation.
15759        let rebalance_meta = self.rebalance_meta.read().await.clone();
15760        let terminal_at = OffsetDateTime::now_utc();
15761        let (should_reload_pool_meta, previous_pool_meta, terminal_canceler) = {
15762            let cancelers = self.decommission_cancelers.read().await;
15763            let mut pool_meta = self.pool_meta.write().await;
15764            let previous_pool_meta = pool_meta.clone();
15765            let Some(changed) =
15766                update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, idx, owner, |pool_meta| {
15767                    pool_meta.decommission_failed_at(idx, terminal_at, rebalance_meta.as_ref())
15768                })
15769            else {
15770                return Ok(());
15771            };
15772            let terminal_canceler = if let Some(owner) = owner {
15773                Some(owner.clone())
15774            } else {
15775                cancelers.get(idx).and_then(Option::as_ref).cloned()
15776            };
15777            (changed, changed.then_some(previous_pool_meta), terminal_canceler)
15778        };
15779
15780        if should_reload_pool_meta && let Err(err) = save_pool_meta.await {
15781            if let Some(previous_pool_meta) = previous_pool_meta {
15782                let mut pool_meta = self.pool_meta.write().await;
15783                rollback_decommission_pool_meta(&mut pool_meta, &previous_pool_meta, &[idx]);
15784            }
15785            return Err(err);
15786        }
15787        if should_reload_pool_meta {
15788            {
15789                let mut pool_meta = self.pool_meta.write().await;
15790                pool_meta.mark_decommission_progress_saved();
15791            }
15792        }
15793        if let Some(canceler) = terminal_canceler.as_ref() {
15794            self.release_decommission_canceler_slot(idx, canceler).await;
15795        }
15796
15797        if should_reload_pool_meta {
15798            self.ctx.advance_data_movement_operation_epoch();
15799        }
15800        drop(_movement_guard);
15801
15802        if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
15803            let stage = format!("decommission_failed for pool {idx}");
15804            if let Some(err) = observe_decommission_terminal_reload_result(
15805                resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()),
15806                stage.as_str(),
15807            ) {
15808                if let Err(record_err) = self
15809                    .record_decommission_terminal_reload_failure(idx, stage.as_str(), err.clone())
15810                    .await
15811                {
15812                    warn!(
15813                        event = EVENT_DECOMMISSION_STATE,
15814                        component = LOG_COMPONENT_ECSTORE,
15815                        subsystem = LOG_SUBSYSTEM_POOLS,
15816                        pool_index = idx,
15817                        state = "terminal_reload_record_failed",
15818                        error = %record_err,
15819                        original_error = %err,
15820                        "Decommission terminal reload failure record failed"
15821                    );
15822                }
15823                warn!(
15824                    event = EVENT_DECOMMISSION_STATE,
15825                    component = LOG_COMPONENT_ECSTORE,
15826                    subsystem = LOG_SUBSYSTEM_POOLS,
15827                    pool_index = idx,
15828                    state = "terminal_reload_failed",
15829                    error = %err,
15830                    "Decommission terminal state saved but pool meta reload failed"
15831                );
15832            }
15833        }
15834
15835        Ok(())
15836    }
15837
15838    #[tracing::instrument(skip(self))]
15839    pub async fn complete_decommission(&self, idx: usize) -> Result<()> {
15840        self.complete_decommission_with_owner(idx, None, None, None).await
15841    }
15842
15843    async fn complete_decommission_for_operation(
15844        &self,
15845        idx: usize,
15846        owner: &DecommissionCanceler,
15847        verified_generation: OffsetDateTime,
15848        verified_unresolved_entries: Vec<DecommissionUnresolvedEntry>,
15849    ) -> Result<()> {
15850        self.complete_decommission_with_owner(idx, Some(owner), Some(verified_generation), Some(verified_unresolved_entries))
15851            .await
15852    }
15853
15854    async fn complete_decommission_with_owner(
15855        &self,
15856        idx: usize,
15857        owner: Option<&DecommissionCanceler>,
15858        verified_generation: Option<OffsetDateTime>,
15859        verified_unresolved_entries: Option<Vec<DecommissionUnresolvedEntry>>,
15860    ) -> Result<()> {
15861        ensure_decommission_terminal_operation_supported(self.single_pool(), "complete decommission")?;
15862        ensure_valid_decommission_pool_index(self.pools.len(), idx)?;
15863        if let Some(owner) = owner {
15864            let cancelers = self.decommission_cancelers.read().await;
15865            if !decommission_canceler_is_owned_by(cancelers.as_slice(), idx, owner) {
15866                owner.release();
15867                return Ok(());
15868            }
15869        }
15870        self.verify_decommission_durable_ilm_receipts(idx).await?;
15871        let _start_guard = self.start_gate.lock().await;
15872        let movement_gate = self.ctx.data_movement_operation_gate();
15873        let _movement_guard = movement_gate.write().await;
15874
15875        // Lock order: movement gate, rebalance_meta, decommission_cancelers,
15876        // then pool_meta. Holding both state locks makes owner validation and
15877        // the terminal transition one atomic operation.
15878        let rebalance_meta = self.rebalance_meta.read().await.clone();
15879        let terminal_at = OffsetDateTime::now_utc();
15880        let (should_reload_pool_meta, completed, previous_pool_meta, terminal_canceler) = {
15881            let cancelers = self.decommission_cancelers.read().await;
15882            let mut pool_meta = self.pool_meta.write().await;
15883            let previous_pool_meta = pool_meta.clone();
15884            let Some(changed) =
15885                update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, idx, owner, |pool_meta| {
15886                    reconcile_decommission_unresolved_entries_for_completion(
15887                        pool_meta,
15888                        idx,
15889                        verified_generation,
15890                        verified_unresolved_entries.as_deref(),
15891                    )?;
15892                    Ok::<bool, Error>(pool_meta.decommission_complete_at(idx, terminal_at, rebalance_meta.as_ref()))
15893                })
15894            else {
15895                return Ok(());
15896            };
15897            let changed = changed?;
15898            let completed = pool_meta
15899                .pools
15900                .get(idx)
15901                .and_then(|pool| pool.decommission.as_ref())
15902                .is_some_and(|decommission| decommission.complete);
15903            let terminal_canceler = if let Some(owner) = owner {
15904                Some(owner.clone())
15905            } else {
15906                cancelers.get(idx).and_then(Option::as_ref).cloned()
15907            };
15908            (changed, completed, changed.then_some(previous_pool_meta), terminal_canceler)
15909        };
15910
15911        if should_reload_pool_meta && let Err(err) = self.save_current_pool_meta(&[idx]).await {
15912            if let Some(previous_pool_meta) = previous_pool_meta {
15913                let mut pool_meta = self.pool_meta.write().await;
15914                rollback_decommission_pool_meta(&mut pool_meta, &previous_pool_meta, &[idx]);
15915            }
15916            return Err(err);
15917        }
15918        if should_reload_pool_meta {
15919            {
15920                let mut pool_meta = self.pool_meta.write().await;
15921                pool_meta.mark_decommission_progress_saved();
15922            }
15923        }
15924        if let Some(canceler) = terminal_canceler.as_ref() {
15925            self.release_decommission_canceler_slot(idx, canceler).await;
15926        }
15927
15928        if should_reload_pool_meta {
15929            self.ctx.advance_data_movement_operation_epoch();
15930        }
15931        drop(_movement_guard);
15932
15933        if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
15934            let stage = format!("complete_decommission for pool {idx}");
15935            if let Some(err) = observe_decommission_terminal_reload_result(
15936                resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()),
15937                stage.as_str(),
15938            ) {
15939                if let Err(record_err) = self
15940                    .record_decommission_terminal_reload_failure(idx, stage.as_str(), err.clone())
15941                    .await
15942                {
15943                    warn!(
15944                        event = EVENT_DECOMMISSION_STATE,
15945                        component = LOG_COMPONENT_ECSTORE,
15946                        subsystem = LOG_SUBSYSTEM_POOLS,
15947                        pool_index = idx,
15948                        state = "terminal_reload_record_failed",
15949                        error = %record_err,
15950                        original_error = %err,
15951                        "Decommission terminal reload failure record failed"
15952                    );
15953                }
15954                warn!(
15955                    event = EVENT_DECOMMISSION_STATE,
15956                    component = LOG_COMPONENT_ECSTORE,
15957                    subsystem = LOG_SUBSYSTEM_POOLS,
15958                    pool_index = idx,
15959                    state = "terminal_reload_failed",
15960                    error = %err,
15961                    "Decommission terminal state saved but pool meta reload failed"
15962                );
15963            }
15964        }
15965
15966        if completed && let Err(err) = self.cleanup_decommission_durable_ilm_receipts(idx).await {
15967            warn!(
15968                event = EVENT_DECOMMISSION_STATE,
15969                component = LOG_COMPONENT_ECSTORE,
15970                subsystem = LOG_SUBSYSTEM_POOLS,
15971                pool_index = idx,
15972                state = "receipt_cleanup_failed",
15973                error = %err,
15974                "Decommission durable ILM receipt cleanup failed"
15975            );
15976        }
15977
15978        Ok(())
15979    }
15980
15981    async fn decommission_pending_bucket(
15982        self: &Arc<Self>,
15983        rx: CancellationToken,
15984        idx: usize,
15985        generation: OffsetDateTime,
15986        bucket: DecomBucketInfo,
15987        entry_budget: Arc<Semaphore>,
15988        source_changed_exhaustions: Arc<AtomicUsize>,
15989    ) -> Result<()> {
15990        let is_decommissioned = {
15991            let pool_meta = self.pool_meta.read().await;
15992            ensure_decommission_generation(&pool_meta, idx, generation)?;
15993            resolve_decommission_bucket_state(&pool_meta, idx, &bucket)?
15994        };
15995
15996        if is_decommissioned {
15997            warn!("decommission: already done, moving on {}", bucket.to_string());
15998
15999            self.mark_decommission_bucket_done_and_save(idx, generation, &bucket).await?;
16000            return Ok(());
16001        }
16002
16003        warn!("decommission: currently on bucket {}", &bucket.name);
16004
16005        let pool = get_by_index(self.pools.as_slice(), idx, "load decommission background pool")?.clone();
16006        if let Err(err) = self
16007            .decommission_pool(rx.clone(), idx, pool, bucket.clone(), entry_budget, source_changed_exhaustions)
16008            .await
16009        {
16010            error!("decommission: decommission_pool err {:?}", &err);
16011            return Err(err);
16012        } else {
16013            warn!("decommission: decommission_pool done {}", &bucket.name);
16014        }
16015
16016        if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) {
16017            warn!("decommission: cancellation observed after decommission_pool {}", &bucket.name);
16018            return Err(err);
16019        }
16020
16021        self.mark_decommission_bucket_done_and_save(idx, generation, &bucket).await?;
16022
16023        warn!("decommission: decommission_pool bucket_done {}", &bucket.name);
16024
16025        Ok(())
16026    }
16027
16028    #[tracing::instrument(skip(self, rx))]
16029    async fn decommission_in_background(
16030        self: &Arc<Self>,
16031        rx: CancellationToken,
16032        idx: usize,
16033        generation: OffsetDateTime,
16034        entry_budget: Arc<Semaphore>,
16035    ) -> Result<()> {
16036        self.ensure_decommission_runtime_capacity_available(idx, generation).await?;
16037        let pending = {
16038            let pool_meta = self.pool_meta.read().await;
16039            pool_meta.pending_buckets(idx)
16040        };
16041        let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
16042        let bucket_concurrency = decommission_bucket_concurrency_limit();
16043        let (regular_buckets, meta_buckets) = split_decommission_buckets(pending);
16044        let store = Arc::clone(self);
16045        run_decommission_phases(rx.clone(), regular_buckets, meta_buckets, bucket_concurrency, move |bucket, rx| {
16046            let store = Arc::clone(&store);
16047            let entry_budget = entry_budget.clone();
16048            let source_changed_exhaustions = Arc::clone(&source_changed_exhaustions);
16049            Box::pin(async move {
16050                store
16051                    .decommission_pending_bucket(rx, idx, generation, bucket, entry_budget, source_changed_exhaustions)
16052                    .await
16053            })
16054        })
16055        .await
16056    }
16057
16058    #[tracing::instrument(skip(self))]
16059    pub async fn start_decommission(&self, indices: Vec<usize>) -> Result<()> {
16060        self.start_decommission_inner(indices, None).await.map(|_| ())
16061    }
16062
16063    async fn start_decommission_with_routines(
16064        &self,
16065        indices: Vec<usize>,
16066        rx: &CancellationToken,
16067        local_indices: &[usize],
16068    ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
16069        self.start_decommission_inner(indices, Some((rx, local_indices))).await
16070    }
16071
16072    async fn start_decommission_inner(
16073        &self,
16074        indices: Vec<usize>,
16075        reservation: Option<(&CancellationToken, &[usize])>,
16076    ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
16077        let indices = dedup_indices(&indices);
16078        validate_start_decommission_request(&indices, self.single_pool())?;
16079
16080        self.ensure_decommission_rebalance_idle_after_refresh().await?;
16081        #[cfg(test)]
16082        let endpoints = self.instance_endpoints().unwrap_or_else(|| self.endpoints());
16083        #[cfg(not(test))]
16084        let endpoints = self.endpoints();
16085        ensure_decommission_start_local_leader(&endpoints, &indices)?;
16086
16087        for idx in indices.iter().copied() {
16088            ensure_valid_decommission_pool_index(self.pools.len(), idx)?;
16089        }
16090
16091        {
16092            let pool_meta = self.pool_meta.read().await;
16093            ensure_decommission_start_pool_states(&pool_meta, &indices)?;
16094            ensure_decommission_ledger_persistence_supported(&pool_meta)?;
16095            ensure_decommission_capacity_writer_supported(&pool_meta)?;
16096        }
16097
16098        #[cfg(test)]
16099        observe_pool_activation_preflight_side_effect_attempt(PoolActivationStartKind::Decommission);
16100        let decom_buckets = self.get_buckets_to_decommission().await?;
16101
16102        let mut healed_buckets = HashSet::with_capacity(decom_buckets.len());
16103        for bk in decom_buckets.iter() {
16104            if healed_buckets.insert(bk.name.as_str()) {
16105                resolve_decommission_preflight_heal_result(&bk.name, self.heal_bucket(&bk.name, &HealOpts::default()).await)?;
16106            }
16107        }
16108
16109        let meta_bucket_opts = decommission_meta_bucket_options();
16110        for prefix in DECOMMISSION_META_PREFIXES {
16111            let bk = path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(prefix)]);
16112            if let Err(err) = self
16113                .make_bucket(bk.to_string_lossy().to_string().as_str(), &meta_bucket_opts)
16114                .await
16115                && !is_err_bucket_exists(&err)
16116            {
16117                error!("decommission: make bucket failed: {err}");
16118                return Err(err);
16119            }
16120        }
16121
16122        let _start_guard = self.start_gate.lock().await;
16123        self.ensure_decommission_rebalance_idle_after_refresh_under_start_gate()
16124            .await?;
16125
16126        let all_capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
16127        let target_fence_proof_available =
16128            crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof().is_some();
16129        // Signal cancellation before waiting for the movement writer so active
16130        // object operations can observe the signal and release read guards.
16131        self.cancel_decommission_routines(&indices).await;
16132        let movement_gate = self.ctx.data_movement_operation_gate();
16133        let _movement_guard = movement_gate.write().await;
16134
16135        let index_cancelers = if let Some((rx, local_indices)) = reservation {
16136            // Lock order matches terminal transitions: movement gate, then
16137            // decommission_cancelers, then pool_meta while start_gate excludes
16138            // another start.
16139            let mut cancelers = self.decommission_cancelers.write().await;
16140            let pool_meta = self.pool_meta.read().await;
16141            ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_capacity_infos, target_fence_proof_available)?;
16142            reserve_decommission_start_cancelers(&pool_meta, &indices, local_indices, rx, cancelers.as_mut_slice())?
16143        } else {
16144            let pool_meta = self.pool_meta.read().await;
16145            ensure_decommission_start_pool_states(&pool_meta, &indices)?;
16146            ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_capacity_infos, target_fence_proof_available)?;
16147            Vec::new()
16148        };
16149
16150        let previous_pool_meta = self
16151            .save_current_pool_meta_for_decommission_start(&indices, decom_buckets)
16152            .await?;
16153        self.ctx.advance_data_movement_operation_epoch();
16154        // The local durable transition is now fenced. Release the writer
16155        // before any peer RPC; remote reload must not block scanner admission.
16156        drop(_movement_guard);
16157
16158        if let Some(notification_sys) = runtime_sources::notification_sys()
16159            && let Err(err) = resolve_start_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await)
16160        {
16161            warn!(
16162                event = EVENT_DECOMMISSION_STATE,
16163                component = LOG_COMPONENT_ECSTORE,
16164                subsystem = LOG_SUBSYSTEM_POOLS,
16165                state = "start_failed",
16166                stage = "reload_pool_meta",
16167                error = %err,
16168                "Decommission start failed after pool metadata save"
16169            );
16170
16171            let rollback_result = self
16172                .rollback_decommission_start_after_reload_failure(&movement_gate, &previous_pool_meta, &indices)
16173                .await;
16174            if let Err(rollback_save_err) = rollback_result {
16175                error!(
16176                    event = EVENT_DECOMMISSION_STATE,
16177                    component = LOG_COMPONENT_ECSTORE,
16178                    subsystem = LOG_SUBSYSTEM_POOLS,
16179                    state = "rollback_failed",
16180                    stage = "save_pool_meta",
16181                    error = %rollback_save_err,
16182                    original_error = %err,
16183                    "Decommission rollback failed after pool metadata reload failure"
16184                );
16185                return Err(Error::other(format!(
16186                    "{err}; decommission start rollback save failed: {rollback_save_err}"
16187                )));
16188            }
16189
16190            if let Err(rollback_reload_err) = resolve_decommission_pool_meta_reload_result(
16191                notification_sys.reload_pool_meta().await,
16192                "start_decommission_rollback",
16193            ) {
16194                error!(
16195                    event = EVENT_DECOMMISSION_STATE,
16196                    component = LOG_COMPONENT_ECSTORE,
16197                    subsystem = LOG_SUBSYSTEM_POOLS,
16198                    state = "rollback_partial",
16199                    stage = "reload_pool_meta",
16200                    error = %rollback_reload_err,
16201                    original_error = %err,
16202                    "Decommission rollback metadata reload failed after local rollback save"
16203                );
16204                return Err(Error::other(format!(
16205                    "{err}; decommission start rollback saved locally but peer reload failed: {rollback_reload_err}"
16206                )));
16207            }
16208
16209            warn!(
16210                event = EVENT_DECOMMISSION_STATE,
16211                component = LOG_COMPONENT_ECSTORE,
16212                subsystem = LOG_SUBSYSTEM_POOLS,
16213                state = "rollback_success",
16214                original_error = %err,
16215                "Decommission start rolled back after pool metadata reload failure"
16216            );
16217            return Err(Error::other(format!("{err}; decommission start rollback succeeded")));
16218        }
16219
16220        Ok(index_cancelers)
16221    }
16222
16223    async fn get_buckets_to_decommission(&self) -> Result<Vec<DecomBucketInfo>> {
16224        let buckets = self.list_bucket(&BucketOptions::default()).await?;
16225
16226        let mut ret: Vec<DecomBucketInfo> = buckets
16227            .iter()
16228            .map(|v| DecomBucketInfo {
16229                name: v.name.clone(),
16230                ..Default::default()
16231            })
16232            .collect();
16233
16234        ret.extend(decommission_meta_buckets());
16235
16236        Ok(ret)
16237    }
16238
16239    async fn durable_ilm_receipt_run_token(&self, source_pool_idx: usize) -> Result<String> {
16240        let pool_meta = self.pool_meta.read().await;
16241        let pool = pool_meta
16242            .pools
16243            .get(source_pool_idx)
16244            .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), source_pool_idx))?;
16245        let start_time = pool
16246            .decommission
16247            .as_ref()
16248            .and_then(|info| info.start_time)
16249            .ok_or_else(|| Error::other(format!("decommission run identity is missing for pool {source_pool_idx}")))?;
16250        Ok(decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time))
16251    }
16252
16253    async fn durable_ilm_receipt_run_token_for_generation(
16254        &self,
16255        source_pool_idx: usize,
16256        generation: OffsetDateTime,
16257    ) -> Result<String> {
16258        let pool_meta = self.pool_meta.read().await;
16259        ensure_decommission_generation(&pool_meta, source_pool_idx, generation)?;
16260        let pool = pool_meta
16261            .pools
16262            .get(source_pool_idx)
16263            .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), source_pool_idx))?;
16264        Ok(decommission_durable_ilm_receipt_run_token(&pool.cmd_line, generation))
16265    }
16266
16267    async fn load_decommissioned_durable_ilm_target(
16268        &self,
16269        source_pool_idx: usize,
16270        path: &str,
16271        max_record_size: usize,
16272        record_context: &str,
16273    ) -> Result<Option<(usize, Vec<u8>)>> {
16274        let mut target = None::<(usize, Vec<u8>)>;
16275        let mut first_read_error = None;
16276        for (target_pool_idx, pool) in self.pools.iter().enumerate() {
16277            if target_pool_idx == source_pool_idx {
16278                continue;
16279            }
16280            match read_config_limited_preserve_empty(pool.clone(), path, max_record_size).await {
16281                Ok(data) => {
16282                    if let Some((existing_pool_idx, existing)) = target.as_ref()
16283                        && existing != &data
16284                    {
16285                        return Err(Error::other(format!(
16286                            "divergent target durable ILM records at path `{path}` {record_context} in pools {existing_pool_idx} and {target_pool_idx}"
16287                        )));
16288                    }
16289                    target = Some((target_pool_idx, data));
16290                }
16291                Err(err)
16292                    if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
16293                        || is_err_object_not_found(&err)
16294                        || is_err_version_not_found(&err) => {}
16295                Err(err) => {
16296                    first_read_error.get_or_insert_with(|| {
16297                        Error::other(format!(
16298                            "failed to read target durable ILM record at path `{path}` {record_context} from pool {target_pool_idx}: {err}"
16299                        ))
16300                    });
16301                }
16302            }
16303        }
16304
16305        if let Some(err) = first_read_error {
16306            return Err(err);
16307        }
16308        Ok(target)
16309    }
16310
16311    async fn list_decommission_durable_ilm_receipt_paths_in_pool(&self, pool_idx: usize, prefix: &str) -> Result<Vec<String>> {
16312        let pool = self
16313            .pools
16314            .get(pool_idx)
16315            .ok_or_else(|| invalid_decommission_pool_index_error(self.pools.len(), pool_idx))?;
16316        let mut receipts = Vec::new();
16317        let mut continuation = None;
16318        loop {
16319            let page = pool
16320                .clone()
16321                .list_objects_v2(RUSTFS_META_BUCKET, prefix, continuation, None, 1000, false, None, false)
16322                .await
16323                .map_err(|err| {
16324                    Error::other(format!(
16325                        "failed to list durable ILM decommission receipts under `{prefix}` in pool {pool_idx}: {err}"
16326                    ))
16327                })?;
16328            receipts.extend(page.objects.into_iter().map(|object| object.name));
16329            if !page.is_truncated {
16330                break;
16331            }
16332            continuation = Some(page.next_continuation_token.ok_or_else(|| {
16333                Error::other(format!(
16334                    "durable ILM decommission receipt listing under `{prefix}` in pool {pool_idx} was truncated without a continuation token"
16335                ))
16336            })?);
16337        }
16338        Ok(receipts)
16339    }
16340
16341    async fn list_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<Vec<(usize, String)>> {
16342        let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
16343        self.list_decommission_durable_ilm_receipts_for_run(source_pool_idx, &run_token)
16344            .await
16345    }
16346
16347    async fn list_decommission_durable_ilm_receipts_for_run(
16348        &self,
16349        source_pool_idx: usize,
16350        run_token: &str,
16351    ) -> Result<Vec<(usize, String)>> {
16352        let prefix = decommission_durable_ilm_receipt_run_prefix(run_token);
16353        let mut receipts = Vec::new();
16354        for pool_idx in 0..self.pools.len() {
16355            if pool_idx == source_pool_idx {
16356                continue;
16357            }
16358            for receipt_path in self
16359                .list_decommission_durable_ilm_receipt_paths_in_pool(pool_idx, &prefix)
16360                .await?
16361            {
16362                let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
16363                if locator.run_token != run_token {
16364                    return Err(Error::other(format!(
16365                        "durable ILM receipt path `{receipt_path}` has an unexpected run token"
16366                    )));
16367                }
16368                receipts.push((pool_idx, receipt_path));
16369            }
16370        }
16371        Ok(receipts)
16372    }
16373
16374    async fn list_decommission_durable_ilm_manifest_receipts(&self, source_pool_idx: usize) -> Result<Vec<String>> {
16375        let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
16376        self.list_decommission_durable_ilm_manifest_receipts_for_run(source_pool_idx, &run_token)
16377            .await
16378    }
16379
16380    async fn list_decommission_durable_ilm_manifest_receipts_for_run(
16381        &self,
16382        source_pool_idx: usize,
16383        run_token: &str,
16384    ) -> Result<Vec<String>> {
16385        let prefix = decommission_durable_ilm_receipt_run_prefix(run_token);
16386        let receipt_paths = self
16387            .list_decommission_durable_ilm_receipt_paths_in_pool(source_pool_idx, &prefix)
16388            .await?;
16389        for receipt_path in &receipt_paths {
16390            let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?;
16391            if locator.run_token != run_token {
16392                return Err(Error::other(format!(
16393                    "durable ILM expected manifest receipt path `{receipt_path}` has an unexpected run token"
16394                )));
16395            }
16396        }
16397        Ok(receipt_paths)
16398    }
16399
16400    #[cfg(all(test, feature = "test-util"))]
16401    async fn persist_decommission_durable_ilm_manifest(&self, source_pool_idx: usize) -> Result<()> {
16402        let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
16403        self.persist_decommission_durable_ilm_manifest_for_run(source_pool_idx, &run_token)
16404            .await
16405    }
16406
16407    async fn persist_decommission_durable_ilm_manifest_for_run(&self, source_pool_idx: usize, run_token: &str) -> Result<()> {
16408        let receipt_paths = self
16409            .list_decommission_durable_ilm_manifest_receipts_for_run(source_pool_idx, run_token)
16410            .await?;
16411        for receipt_path in &receipt_paths {
16412            self.read_decommission_durable_ilm_receipt(source_pool_idx, receipt_path)
16413                .await?;
16414        }
16415        let manifest = DecommissionDurableIlmManifest::new(run_token, &receipt_paths)?;
16416        let manifest_path = decommission_durable_ilm_manifest_path(run_token);
16417        let encoded = manifest.encode()?;
16418        let mut attempt = 1;
16419        loop {
16420            match read_config_limited_preserve_empty(
16421                self.pools[source_pool_idx].clone(),
16422                &manifest_path,
16423                DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE,
16424            )
16425            .await
16426            {
16427                Ok(existing) => {
16428                    DecommissionDurableIlmManifest::decode(&existing, run_token, &receipt_paths).map_err(|err| {
16429                        Error::other(format!(
16430                            "durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx} is invalid: {err}"
16431                        ))
16432                    })?;
16433                    return Ok(());
16434                }
16435                Err(err)
16436                    if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
16437                        || is_err_object_not_found(&err)
16438                        || is_err_version_not_found(&err) => {}
16439                Err(err) => {
16440                    return Err(Error::other(format!(
16441                        "failed to read durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}"
16442                    )));
16443                }
16444            }
16445            match save_config_with_opts(
16446                self.pools[source_pool_idx].clone(),
16447                &manifest_path,
16448                encoded.clone(),
16449                &ObjectOptions {
16450                    max_parity: true,
16451                    write_completion: crate::object_api::WriteCompletion::TailDrained,
16452                    http_preconditions: Some(HTTPPreconditions {
16453                        if_none_match: Some("*".to_string()),
16454                        ..Default::default()
16455                    }),
16456                    ..Default::default()
16457                },
16458            )
16459            .await
16460            {
16461                Ok(()) => return Ok(()),
16462                Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => {
16463                    attempt += 1;
16464                }
16465                Err(Error::PreconditionFailed) => {
16466                    return Err(Error::other(format!(
16467                        "failed to persist durable ILM expected manifest `{manifest_path}` after concurrent updates"
16468                    )));
16469                }
16470                Err(err) => {
16471                    return Err(Error::other(format!(
16472                        "failed to persist durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx}: {err}"
16473                    )));
16474                }
16475            }
16476        }
16477    }
16478
16479    async fn load_decommission_durable_ilm_manifest_for_run(
16480        &self,
16481        source_pool_idx: usize,
16482        run_token: &str,
16483    ) -> Result<HashMap<String, DecommissionDurableIlmReceipt>> {
16484        let receipt_paths = self
16485            .list_decommission_durable_ilm_manifest_receipts_for_run(source_pool_idx, run_token)
16486            .await?;
16487        let manifest_path = decommission_durable_ilm_manifest_path(run_token);
16488        let data = read_config_limited_preserve_empty(
16489            self.pools[source_pool_idx].clone(),
16490            &manifest_path,
16491            DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE,
16492        )
16493        .await
16494        .map_err(|err| {
16495            Error::other(format!(
16496                "failed to read durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}"
16497            ))
16498        })?;
16499        DecommissionDurableIlmManifest::decode(&data, run_token, &receipt_paths).map_err(|err| {
16500            Error::other(format!(
16501                "durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx} is invalid: {err}"
16502            ))
16503        })?;
16504
16505        let mut receipts = HashMap::with_capacity(receipt_paths.len());
16506        for receipt_path in receipt_paths {
16507            let receipt = self
16508                .read_decommission_durable_ilm_receipt(source_pool_idx, &receipt_path)
16509                .await?;
16510            if receipts.insert(receipt_path.clone(), receipt).is_some() {
16511                return Err(Error::other(format!(
16512                    "durable ILM expected manifest contains duplicate receipt path `{receipt_path}`"
16513                )));
16514            }
16515        }
16516        Ok(receipts)
16517    }
16518
16519    #[cfg(all(test, feature = "test-util"))]
16520    async fn persist_decommission_durable_ilm_receipt(
16521        &self,
16522        source_pool_idx: usize,
16523        target_pool_idx: usize,
16524        receipt: &DecommissionDurableIlmReceipt,
16525    ) -> Result<()> {
16526        let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
16527        self.persist_decommission_durable_ilm_receipt_for_run(target_pool_idx, receipt, &run_token)
16528            .await
16529    }
16530
16531    async fn persist_decommission_durable_ilm_receipt_for_run(
16532        &self,
16533        target_pool_idx: usize,
16534        receipt: &DecommissionDurableIlmReceipt,
16535        run_token: &str,
16536    ) -> Result<()> {
16537        let receipt_path = decommission_durable_ilm_receipt_path(run_token, &receipt.source_path, &receipt.id_kind, &receipt.id);
16538        let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
16539        let mut attempt = 1;
16540        loop {
16541            let (merged, http_preconditions) = match read_config_limited_preserve_empty_with_metadata(
16542                self.pools[target_pool_idx].clone(),
16543                &receipt_path,
16544                DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
16545            )
16546            .await
16547            {
16548                Ok((existing_data, metadata)) => {
16549                    let existing = DecommissionDurableIlmReceipt::decode(&existing_data).map_err(|err| {
16550                        Error::other(format!(
16551                            "durable ILM decommission receipt `{receipt_path}` in pool {target_pool_idx} for {} is invalid: {err}",
16552                            locator.context()
16553                        ))
16554                    })?;
16555                    Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &existing)?;
16556                    let merged = merge_decommission_durable_ilm_receipts(&existing, receipt)?;
16557                    if merged == existing {
16558                        return Ok(());
16559                    }
16560                    let etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| {
16561                        Error::other(format!(
16562                            "durable ILM decommission receipt `{receipt_path}` in pool {target_pool_idx} is missing an ETag"
16563                        ))
16564                    })?;
16565                    (
16566                        merged,
16567                        HTTPPreconditions {
16568                            if_match: Some(etag),
16569                            ..Default::default()
16570                        },
16571                    )
16572                }
16573                Err(err)
16574                    if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
16575                        || is_err_object_not_found(&err)
16576                        || is_err_version_not_found(&err) =>
16577                {
16578                    (
16579                        receipt.clone(),
16580                        HTTPPreconditions {
16581                            if_none_match: Some("*".to_string()),
16582                            ..Default::default()
16583                        },
16584                    )
16585                }
16586                Err(err) => {
16587                    return Err(Error::other(format!(
16588                        "failed to read durable ILM decommission receipt `{receipt_path}` from pool {target_pool_idx} for {}: {err}",
16589                        locator.context()
16590                    )));
16591                }
16592            };
16593            let encoded = merged.encode().map_err(|err| {
16594                Error::other(format!(
16595                    "failed to encode durable ILM decommission receipt `{receipt_path}` for source path `{}` {}: {err}",
16596                    receipt.source_path,
16597                    receipt.context()
16598                ))
16599            })?;
16600            match save_config_with_opts(
16601                self.pools[target_pool_idx].clone(),
16602                &receipt_path,
16603                encoded,
16604                &ObjectOptions {
16605                    max_parity: true,
16606                    write_completion: crate::object_api::WriteCompletion::TailDrained,
16607                    http_preconditions: Some(http_preconditions),
16608                    ..Default::default()
16609                },
16610            )
16611            .await
16612            {
16613                Ok(()) => return Ok(()),
16614                Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => {
16615                    attempt += 1;
16616                }
16617                Err(Error::PreconditionFailed) => {
16618                    return Err(Error::other(format!(
16619                        "failed to persist durable ILM decommission receipt `{receipt_path}` for {} after concurrent updates",
16620                        locator.context()
16621                    )));
16622                }
16623                Err(err) => {
16624                    return Err(Error::other(format!(
16625                        "failed to persist durable ILM decommission receipt `{receipt_path}` for {}: {err}",
16626                        locator.context()
16627                    )));
16628                }
16629            }
16630        }
16631    }
16632
16633    fn validate_decommission_durable_ilm_receipt_locator(
16634        receipt_path: &str,
16635        locator: &DecommissionDurableIlmReceiptLocator,
16636        receipt: &DecommissionDurableIlmReceipt,
16637    ) -> Result<()> {
16638        if locator.source_path != receipt.source_path || locator.id_kind != receipt.id_kind || locator.id != receipt.id {
16639            return Err(Error::other(format!(
16640                "durable ILM decommission receipt path `{receipt_path}` identity {} does not match receipt {}",
16641                locator.context(),
16642                receipt.context()
16643            )));
16644        }
16645        Ok(())
16646    }
16647
16648    async fn read_decommission_durable_ilm_receipt(
16649        &self,
16650        receipt_pool_idx: usize,
16651        receipt_path: &str,
16652    ) -> Result<DecommissionDurableIlmReceipt> {
16653        let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?;
16654        let data = read_config_limited_preserve_empty(
16655            self.pools[receipt_pool_idx].clone(),
16656            receipt_path,
16657            DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
16658        )
16659        .await
16660        .map_err(|err| {
16661            Error::other(format!(
16662                "failed to read durable ILM decommission receipt `{receipt_path}` from pool {receipt_pool_idx} for {}: {err}",
16663                locator.context()
16664            ))
16665        })?;
16666        let receipt = DecommissionDurableIlmReceipt::decode(&data).map_err(|err| {
16667            Error::other(format!(
16668                "durable ILM decommission receipt `{receipt_path}` in pool {receipt_pool_idx} for {} is invalid: {err}",
16669                locator.context()
16670            ))
16671        })?;
16672        Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?;
16673        Ok(receipt)
16674    }
16675
16676    async fn load_decommission_durable_ilm_terminal_receipt_for_run(
16677        &self,
16678        source_pool_idx: usize,
16679        path: &str,
16680        source_record: &ValidatedDurableIlmRecord,
16681        run_token: &str,
16682    ) -> Result<Option<DecommissionDurableIlmReceipt>> {
16683        let receipt_path = decommission_durable_ilm_receipt_path(run_token, path, source_record.id_kind, &source_record.id);
16684        let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
16685        let mut proof = None::<DecommissionDurableIlmReceipt>;
16686        let mut nonterminal_receipt_found = false;
16687        for pool_idx in 0..self.pools.len() {
16688            if pool_idx == source_pool_idx {
16689                continue;
16690            }
16691            let data = match read_config_limited_preserve_empty(
16692                self.pools[pool_idx].clone(),
16693                &receipt_path,
16694                DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
16695            )
16696            .await
16697            {
16698                Ok(data) => data,
16699                Err(err)
16700                    if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
16701                        || is_err_object_not_found(&err)
16702                        || is_err_version_not_found(&err) =>
16703                {
16704                    continue;
16705                }
16706                Err(err) => {
16707                    return Err(Error::other(format!(
16708                        "failed to read terminal durable ILM decommission receipt `{receipt_path}` from pool {pool_idx} for {}: {err}",
16709                        source_record.context()
16710                    )));
16711                }
16712            };
16713            let receipt = DecommissionDurableIlmReceipt::decode(&data).map_err(|err| {
16714                Error::other(format!(
16715                    "terminal durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} for {} is invalid: {err}",
16716                    source_record.context()
16717                ))
16718            })?;
16719            Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &receipt)?;
16720            if receipt.namespace != source_record.namespace
16721                || receipt.id_kind != source_record.id_kind
16722                || receipt.id != source_record.id
16723            {
16724                return Err(Error::other(format!(
16725                    "terminal durable ILM decommission receipt identity mismatch at path `{path}` {}; receipt {}",
16726                    source_record.context(),
16727                    receipt.context()
16728                )));
16729            }
16730            if let Some(terminal_checkpoint) = receipt.terminal_checkpoint.as_ref() {
16731                if !source_record.checkpoint.is_predecessor_of_terminal(terminal_checkpoint) {
16732                    return Err(Error::other_with_context(
16733                        "terminal durable ILM decommission receipt does not cover source",
16734                        format!("path `{path}` {}", source_record.context()),
16735                    ));
16736                }
16737                proof = Some(match proof {
16738                    Some(existing) => merge_decommission_durable_ilm_receipts(&existing, &receipt)?,
16739                    None => receipt,
16740                });
16741            } else {
16742                source_record
16743                    .checkpoint
16744                    .validate_successor(&receipt.checkpoint)
16745                    .map_err(|err| {
16746                        Error::other_with_context(
16747                            "durable ILM decommission receipt does not cover source",
16748                            format!("path `{path}` {}: {err}", source_record.context()),
16749                        )
16750                    })?;
16751                nonterminal_receipt_found = true;
16752            }
16753        }
16754        // A terminal receipt on one target must not hide another target copy
16755        // whose receipt was installed later and has not reached terminal yet.
16756        // Returning no proof makes recovery advance every outstanding copy
16757        // before source cleanup can treat the operation as complete.
16758        if nonterminal_receipt_found { Ok(None) } else { Ok(proof) }
16759    }
16760
16761    async fn verify_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> {
16762        let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
16763        self.verify_decommission_durable_ilm_receipts_for_run(source_pool_idx, &run_token)
16764            .await
16765    }
16766
16767    async fn verify_decommission_durable_ilm_receipts_for_run(&self, source_pool_idx: usize, run_token: &str) -> Result<()> {
16768        let expected_receipts = self
16769            .load_decommission_durable_ilm_manifest_for_run(source_pool_idx, run_token)
16770            .await?;
16771        let receipt_paths = self
16772            .list_decommission_durable_ilm_receipts_for_run(source_pool_idx, run_token)
16773            .await?;
16774        let present_receipt_paths = receipt_paths
16775            .iter()
16776            .map(|(_, receipt_path)| receipt_path.as_str())
16777            .collect::<HashSet<_>>();
16778        for (expected_path, expected) in &expected_receipts {
16779            if !present_receipt_paths.contains(expected_path.as_str()) {
16780                return Err(Error::other(format!(
16781                    "durable ILM decommission receipt is missing at `{expected_path}` for source path `{}` {}",
16782                    expected.source_path,
16783                    expected.context()
16784                )));
16785            }
16786        }
16787
16788        for (receipt_pool_idx, receipt_path) in receipt_paths {
16789            let expected = expected_receipts.get(&receipt_path).ok_or_else(|| {
16790                Error::other(format!(
16791                    "durable ILM decommission receipt `{receipt_path}` in pool {receipt_pool_idx} is absent from the expected manifest"
16792                ))
16793            })?;
16794            let receipt = self
16795                .read_decommission_durable_ilm_receipt(receipt_pool_idx, &receipt_path)
16796                .await?;
16797            if receipt.source_path != expected.source_path
16798                || receipt.namespace != expected.namespace
16799                || receipt.id_kind != expected.id_kind
16800                || receipt.id != expected.id
16801            {
16802                return Err(Error::other(format!(
16803                    "durable ILM decommission receipt identity mismatch at `{receipt_path}` for source path `{}` {}; decoded {}",
16804                    expected.source_path,
16805                    expected.context(),
16806                    receipt.context()
16807                )));
16808            }
16809            expected.checkpoint.validate_successor(&receipt.checkpoint).map_err(|err| {
16810                Error::other(format!(
16811                    "durable ILM decommission receipt generation mismatch at `{receipt_path}` for source path `{}` {}: {err}",
16812                    expected.source_path,
16813                    expected.context()
16814                ))
16815            })?;
16816            match (&expected.terminal_checkpoint, &receipt.terminal_checkpoint) {
16817                (Some(expected_terminal), Some(receipt_terminal)) => {
16818                    expected_terminal.validate_successor(receipt_terminal).map_err(|err| {
16819                        Error::other(format!(
16820                            "durable ILM decommission terminal receipt generation mismatch at `{receipt_path}` for source path `{}` {}: {err}",
16821                            expected.source_path,
16822                            expected.context()
16823                        ))
16824                    })?;
16825                }
16826                (Some(_), None) => {
16827                    return Err(Error::other(format!(
16828                        "durable ILM decommission terminal receipt is missing at `{receipt_path}` for source path `{}` {}",
16829                        expected.source_path,
16830                        expected.context()
16831                    )));
16832                }
16833                (None, _) => {}
16834            }
16835            let namespace = classify_durable_ilm_record(&receipt.source_path)?
16836                .ok_or_else(|| Error::other(format!("path `{}` is not a durable ILM record", receipt.source_path)))?;
16837            let target = self
16838                .load_decommissioned_durable_ilm_target(
16839                    source_pool_idx,
16840                    &receipt.source_path,
16841                    namespace.max_record_size,
16842                    &receipt.context(),
16843                )
16844                .await?;
16845            if let Some((_, target)) = target {
16846                let target_record = validate_durable_ilm_record(&receipt.source_path, &target).map_err(|err| {
16847                    Error::other(format!(
16848                        "target durable ILM record is invalid at path `{}` {}: {err}",
16849                        receipt.source_path,
16850                        receipt.context()
16851                    ))
16852                })?;
16853                let identity_matches = target_record.namespace == receipt.namespace
16854                    && target_record.id_kind == receipt.id_kind
16855                    && target_record.id == receipt.id;
16856                let reused_manual_scope = receipt.terminal_checkpoint.is_some()
16857                    && matches!(
16858                        (&receipt.checkpoint, &target_record.checkpoint),
16859                        (
16860                            DurableIlmRecordCheckpoint::ManualTransitionScope { .. },
16861                            DurableIlmRecordCheckpoint::ManualTransitionScope { .. }
16862                        )
16863                    );
16864                if !identity_matches && !reused_manual_scope {
16865                    return Err(Error::other(format!(
16866                        "target durable ILM record identity mismatch at path `{}` {}; decoded {}",
16867                        receipt.source_path,
16868                        receipt.context(),
16869                        target_record.context()
16870                    )));
16871                }
16872                if identity_matches {
16873                    receipt
16874                        .terminal_checkpoint
16875                        .as_ref()
16876                        .unwrap_or(&receipt.checkpoint)
16877                        .validate_successor(&target_record.checkpoint)
16878                        .map_err(|err| {
16879                            Error::other(format!(
16880                                "target durable ILM record generation mismatch at path `{}` {}: {err}",
16881                                receipt.source_path,
16882                                receipt.context()
16883                            ))
16884                        })?;
16885                }
16886            } else if receipt.terminal_checkpoint.is_none() {
16887                return Err(Error::other(format!(
16888                    "target durable ILM record is missing at path `{}` {} without a recovery terminal checkpoint",
16889                    receipt.source_path,
16890                    receipt.context()
16891                )));
16892            }
16893        }
16894        Ok(())
16895    }
16896
16897    async fn advance_durable_ilm_decommission_receipt(
16898        &self,
16899        pool_idx: usize,
16900        receipt_path: &str,
16901        record: &ValidatedDurableIlmRecord,
16902        fleet_topology_generation: Option<&str>,
16903        terminal: bool,
16904    ) -> Result<bool> {
16905        let stage = if terminal { "terminal" } else { "progress" };
16906        let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?;
16907        let mut attempt = 1;
16908        loop {
16909            let (receipt_data, metadata) = match read_config_limited_preserve_empty_with_metadata(
16910                self.pools[pool_idx].clone(),
16911                receipt_path,
16912                DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
16913            )
16914            .await
16915            {
16916                Ok(receipt) => receipt,
16917                Err(err)
16918                    if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
16919                        || is_err_object_not_found(&err)
16920                        || is_err_version_not_found(&err) =>
16921                {
16922                    return Ok(false);
16923                }
16924                Err(err) => {
16925                    return Err(Error::other(format!(
16926                        "failed to read durable ILM decommission receipt `{receipt_path}` from pool {pool_idx} for {}: {err}",
16927                        locator.context()
16928                    )));
16929                }
16930            };
16931            let mut receipt = DecommissionDurableIlmReceipt::decode(&receipt_data).map_err(|err| {
16932                Error::other(format!(
16933                    "durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} for {} is invalid: {err}",
16934                    locator.context()
16935                ))
16936            })?;
16937            Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?;
16938            if receipt.fleet_topology_generation.as_deref() != fleet_topology_generation {
16939                return Err(Error::other_with_context(
16940                    "durable ILM decommission receipt fleet topology mismatch",
16941                    format!("path `{}` {}", receipt.source_path, receipt.context()),
16942                ));
16943            }
16944            receipt.checkpoint.validate_successor(&record.checkpoint).map_err(|err| {
16945                Error::other(format!(
16946                    "{stage} durable ILM record generation mismatch at path `{}` {}: {err}",
16947                    receipt.source_path,
16948                    receipt.context()
16949                ))
16950            })?;
16951
16952            if terminal {
16953                if let Some(existing) = &receipt.terminal_checkpoint {
16954                    if existing == &record.checkpoint || record.checkpoint.validate_successor(existing).is_ok() {
16955                        return Ok(true);
16956                    }
16957                    existing.validate_successor(&record.checkpoint).map_err(|err| {
16958                        Error::other(format!(
16959                            "terminal durable ILM record checkpoint conflicts at path `{}` {}: {err}",
16960                            receipt.source_path,
16961                            receipt.context()
16962                        ))
16963                    })?;
16964                }
16965                receipt.terminal_checkpoint = Some(record.checkpoint.clone());
16966            } else {
16967                if let Some(existing) = &receipt.terminal_checkpoint {
16968                    if existing == &record.checkpoint {
16969                        return Ok(true);
16970                    }
16971                    existing.validate_successor(&record.checkpoint).map_err(|err| {
16972                        Error::other(format!(
16973                            "progress durable ILM record conflicts with terminal checkpoint at path `{}` {}: {err}",
16974                            receipt.source_path,
16975                            receipt.context()
16976                        ))
16977                    })?;
16978                    receipt.terminal_checkpoint = None;
16979                }
16980                if receipt.checkpoint == record.checkpoint {
16981                    return Ok(true);
16982                }
16983                receipt.checkpoint = record.checkpoint.clone();
16984            }
16985
16986            let etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| {
16987                Error::other(format!(
16988                    "durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} is missing an ETag"
16989                ))
16990            })?;
16991            let encoded = receipt.encode()?;
16992            match save_config_with_opts(
16993                self.pools[pool_idx].clone(),
16994                receipt_path,
16995                encoded,
16996                &ObjectOptions {
16997                    max_parity: true,
16998                    write_completion: crate::object_api::WriteCompletion::TailDrained,
16999                    http_preconditions: Some(HTTPPreconditions {
17000                        if_match: Some(etag),
17001                        ..Default::default()
17002                    }),
17003                    ..Default::default()
17004                },
17005            )
17006            .await
17007            {
17008                Ok(()) => return Ok(true),
17009                Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => {
17010                    attempt += 1;
17011                    continue;
17012                }
17013                Err(Error::PreconditionFailed) => {
17014                    return Err(Error::other(format!(
17015                        "failed to persist {stage} durable ILM decommission receipt `{receipt_path}` for {} after concurrent updates",
17016                        locator.context()
17017                    )));
17018                }
17019                Err(err) => {
17020                    return Err(Error::other(format!(
17021                        "failed to persist {stage} durable ILM decommission receipt `{receipt_path}` for {}: {err}",
17022                        locator.context()
17023                    )));
17024                }
17025            }
17026        }
17027    }
17028
17029    async fn advance_durable_ilm_decommission_receipts(
17030        &self,
17031        path: &str,
17032        data: &[u8],
17033        terminal: bool,
17034    ) -> Result<Option<Vec<usize>>> {
17035        let active_runs = {
17036            let pool_meta = self.pool_meta.read().await;
17037            pool_meta
17038                .pools
17039                .iter()
17040                .enumerate()
17041                .filter_map(|(pool_idx, pool)| {
17042                    pool.decommission
17043                        .as_ref()
17044                        .filter(|info| info.has_decommission_state() && !info.complete)
17045                        .and_then(|info| info.start_time)
17046                        .map(|start_time| (pool_idx, decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time)))
17047                })
17048                .collect::<Vec<_>>()
17049        };
17050        if active_runs.is_empty() {
17051            return Ok(None);
17052        }
17053
17054        let stage = if terminal { "terminal" } else { "progress" };
17055        let record = validate_durable_ilm_record(path, data)
17056            .map_err(|err| Error::other(format!("{stage} durable ILM record is invalid at path `{path}`: {err}")))?;
17057        let fleet_topology_generation = durable_ilm_v6_topology_generation(path, data)?;
17058        let active_source_pool_indices = active_runs.iter().map(|(pool_idx, _)| *pool_idx).collect::<Vec<_>>();
17059        let mut terminal_target_pool_indices = Vec::new();
17060        for (source_pool_idx, run_token) in active_runs {
17061            let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, record.id_kind, &record.id);
17062            let mut receipt_found = false;
17063            for pool_idx in 0..self.pools.len() {
17064                if pool_idx != source_pool_idx {
17065                    let found = self
17066                        .advance_durable_ilm_decommission_receipt(
17067                            pool_idx,
17068                            &receipt_path,
17069                            &record,
17070                            fleet_topology_generation.as_deref(),
17071                            terminal,
17072                        )
17073                        .await?;
17074                    receipt_found |= found;
17075                    if terminal
17076                        && found
17077                        && !active_source_pool_indices.contains(&pool_idx)
17078                        && !terminal_target_pool_indices.contains(&pool_idx)
17079                    {
17080                        terminal_target_pool_indices.push(pool_idx);
17081                    }
17082                }
17083            }
17084            if terminal && !receipt_found {
17085                return Err(Error::other(format!(
17086                    "terminal durable ILM record at path `{path}` {} is retained until its decommission receipt is committed",
17087                    record.context()
17088                )));
17089            }
17090        }
17091        Ok(Some(terminal_target_pool_indices))
17092    }
17093
17094    /// Resolve the exact receipt-bearing target copies on which a v6 dispatch
17095    /// manifest may advance while a decommission reservation is active.
17096    ///
17097    /// This is intentionally not a general capacity bypass. The target copy
17098    /// must still be covered by the active source reservation and its durable
17099    /// receipt, the ETag must be the caller's exact CAS generation, and the
17100    /// replacement must be a byte-non-growing adjacent manifest checkpoint.
17101    pub(crate) async fn decommission_durable_ilm_checkpoint_targets(
17102        &self,
17103        path: &str,
17104        next_data: &[u8],
17105        expected_etag: &str,
17106    ) -> Result<Option<Vec<DecommissionDurableIlmCheckpointTarget>>> {
17107        let active_runs = {
17108            let pool_meta = self.pool_meta.read().await;
17109            let mut active_runs = Vec::new();
17110            for (source_pool_index, pool) in pool_meta.pools.iter().enumerate() {
17111                let Some(info) = pool
17112                    .decommission
17113                    .as_ref()
17114                    .filter(|info| info.has_decommission_state() && !info.complete)
17115                else {
17116                    continue;
17117                };
17118                let Some(start_time) = info.start_time else {
17119                    continue;
17120                };
17121                let reservation = info.capacity_reservation.clone().ok_or_else(|| {
17122                    decommission_capacity_blocked_error(format!(
17123                        "active decommission source pool {source_pool_index} has no reservation for durable ILM checkpoint"
17124                    ))
17125                })?;
17126                if !reservation.active() {
17127                    return Err(decommission_capacity_blocked_error(format!(
17128                        "active decommission source pool {source_pool_index} has a released reservation for durable ILM checkpoint"
17129                    )));
17130                }
17131                active_runs.push((
17132                    source_pool_index,
17133                    decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time),
17134                    reservation,
17135                ));
17136            }
17137            active_runs
17138        };
17139        if active_runs.is_empty() {
17140            return Ok(None);
17141        }
17142
17143        let next_record = validate_durable_ilm_record(path, next_data)?;
17144        if next_record.namespace != TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.name {
17145            return Ok(None);
17146        }
17147        let next_fleet_topology_generation = durable_ilm_v6_topology_generation(path, next_data)?;
17148
17149        let mut targets = Vec::<DecommissionDurableIlmCheckpointTarget>::new();
17150        let mut missing_source_receipts = 0usize;
17151        let mut stale_target_etag_mismatch = false;
17152        for (source_pool_index, run_token, reservation) in active_runs {
17153            let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, next_record.id_kind, &next_record.id);
17154            let mut source_receipt_found = false;
17155            for allocation in &reservation.targets {
17156                let receipt_data = match read_config_limited_preserve_empty(
17157                    self.pools[allocation.pool_index].clone(),
17158                    &receipt_path,
17159                    DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
17160                )
17161                .await
17162                {
17163                    Ok(data) => data,
17164                    Err(err)
17165                        if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17166                            || is_err_object_not_found(&err)
17167                            || is_err_version_not_found(&err) =>
17168                    {
17169                        continue;
17170                    }
17171                    Err(err) => return Err(err),
17172                };
17173                let receipt = DecommissionDurableIlmReceipt::decode(&receipt_data)?;
17174                let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
17175                Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &receipt)?;
17176                if receipt.source_path != path
17177                    || receipt.namespace != next_record.namespace
17178                    || receipt.id_kind != next_record.id_kind
17179                    || receipt.id != next_record.id
17180                {
17181                    return Err(Error::other_with_context(
17182                        "durable ILM checkpoint receipt identity does not authorize source",
17183                        format!("path `{path}` {}", next_record.context()),
17184                    ));
17185                }
17186                if receipt.fleet_topology_generation != next_fleet_topology_generation {
17187                    return Err(Error::other_with_context(
17188                        "durable ILM checkpoint receipt fleet topology does not authorize source",
17189                        format!("path `{path}` {}", next_record.context()),
17190                    ));
17191                }
17192                receipt
17193                    .checkpoint
17194                    .validate_successor(&next_record.checkpoint)
17195                    .map_err(|err| {
17196                        Error::other_with_context(
17197                            "durable ILM checkpoint receipt is not a predecessor of the requested generation",
17198                            format!(
17199                                "target pool {}, path `{path}` {}; receipt checkpoint {:?}, requested checkpoint {:?}: {err}",
17200                                allocation.pool_index,
17201                                next_record.context(),
17202                                receipt.checkpoint,
17203                                next_record.checkpoint
17204                            ),
17205                        )
17206                    })?;
17207
17208                let (target_data, metadata) = read_config_limited_preserve_empty_with_metadata(
17209                    self.pools[allocation.pool_index].clone(),
17210                    path,
17211                    TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.max_record_size,
17212                )
17213                .await?;
17214                let target_record = validate_durable_ilm_record(path, &target_data)?;
17215                if target_record.namespace != next_record.namespace
17216                    || target_record.id_kind != next_record.id_kind
17217                    || target_record.id != next_record.id
17218                {
17219                    return Err(Error::other_with_context(
17220                        "durable ILM checkpoint target identity does not match source",
17221                        format!("path `{path}` {}", next_record.context()),
17222                    ));
17223                }
17224                let already_committed = target_data.as_slice() == next_data;
17225                let target_etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| {
17226                    Error::other_with_context("durable ILM checkpoint target is missing an ETag", format!("path `{path}`"))
17227                })?;
17228                if already_committed {
17229                    receipt
17230                        .checkpoint
17231                        .validate_successor(&target_record.checkpoint)
17232                        .map_err(|err| {
17233                            Error::other_with_context(
17234                                "durable ILM checkpoint receipt is not a predecessor of the committed target generation",
17235                                format!("path `{path}` {}: {err}", next_record.context()),
17236                            )
17237                        })?;
17238                } else {
17239                    if target_record.checkpoint != receipt.checkpoint {
17240                        return Err(Error::other_with_context(
17241                            "durable ILM checkpoint target is not the receipt generation",
17242                            format!("path `{path}` {}", next_record.context()),
17243                        ));
17244                    }
17245                    target_record
17246                        .checkpoint
17247                        .validate_successor(&next_record.checkpoint)
17248                        .map_err(|err| {
17249                            Error::other_with_context(
17250                                "durable ILM checkpoint target is not a predecessor of the requested generation",
17251                                format!("path `{path}` {}: {err}", next_record.context()),
17252                            )
17253                        })?;
17254                    if next_data.len() > target_data.len() {
17255                        return Err(decommission_capacity_blocked_error(format!(
17256                            "durable ILM checkpoint update at `{path}` grows from {} to {} bytes",
17257                            target_data.len(),
17258                            next_data.len()
17259                        )));
17260                    }
17261                    stale_target_etag_mismatch |= target_etag != expected_etag;
17262                }
17263
17264                source_receipt_found = true;
17265                if let Some(existing) = targets
17266                    .iter_mut()
17267                    .find(|target| target.target_pool_index == allocation.pool_index)
17268                {
17269                    if existing.already_committed != already_committed {
17270                        return Err(Error::other_with_context(
17271                            "durable ILM checkpoint target state changed during authorization",
17272                            format!("path `{path}`"),
17273                        ));
17274                    }
17275                    continue;
17276                }
17277                let base_owner = DecommissionCapacityOwner {
17278                    source_pool_index,
17279                    operation_id: reservation.operation_id,
17280                    generation: reservation.generation,
17281                    owner_nonce: reservation.owner_nonce,
17282                    mutation_id: None,
17283                };
17284                let mutation_id = decommission_capacity_mutation_id(
17285                    base_owner,
17286                    RUSTFS_META_BUCKET,
17287                    path,
17288                    Some(next_record.checkpoint.content_sha256()),
17289                    false,
17290                    None,
17291                );
17292                targets.push(DecommissionDurableIlmCheckpointTarget {
17293                    source_pool_index,
17294                    target_pool_index: allocation.pool_index,
17295                    capacity_owner: base_owner.with_mutation_id(mutation_id),
17296                    already_committed,
17297                    target_etag: Some(target_etag),
17298                });
17299            }
17300            if !source_receipt_found {
17301                missing_source_receipts = missing_source_receipts.saturating_add(1);
17302            }
17303        }
17304
17305        if missing_source_receipts > 0 {
17306            return Err(decommission_capacity_blocked_error(format!(
17307                "durable ILM checkpoint at `{path}` is missing receipt coverage for {missing_source_receipts} active source(s)"
17308            )));
17309        }
17310        if targets.is_empty() {
17311            return Err(decommission_capacity_blocked_error(format!(
17312                "durable ILM checkpoint at `{path}` has no receipt-bearing reservation target"
17313            )));
17314        }
17315        if stale_target_etag_mismatch && !targets.iter().any(|target| target.already_committed) {
17316            return Err(Error::PreconditionFailed);
17317        }
17318        // After a partial multi-target commit, an aggregate read may return
17319        // the ETag of the already-advanced target while another authorized
17320        // target still has the predecessor ETag. The exact committed bytes
17321        // plus every target's receipt/checkpoint proof authorize repairing
17322        // that predecessor with its own target-local CAS. Without an exact
17323        // committed target, retain the caller-ETag requirement above.
17324        targets.sort_unstable_by_key(|target| target.target_pool_index);
17325        Ok(Some(targets))
17326    }
17327
17328    pub(crate) async fn record_durable_ilm_decommission_progress(&self, path: &str, data: &[u8]) -> Result<()> {
17329        self.advance_durable_ilm_decommission_receipts(path, data, false)
17330            .await
17331            .map(|_| ())
17332    }
17333
17334    pub(crate) async fn record_durable_ilm_decommission_terminal(&self, path: &str, data: &[u8]) -> Result<()> {
17335        self.record_durable_ilm_decommission_terminal_target_pools(path, data)
17336            .await
17337            .map(|_| ())
17338    }
17339
17340    /// Record terminal proof and return its non-source receipt pools for targeted cleanup.
17341    pub(crate) async fn record_durable_ilm_decommission_terminal_target_pools(
17342        &self,
17343        path: &str,
17344        data: &[u8],
17345    ) -> Result<Option<Vec<usize>>> {
17346        self.advance_durable_ilm_decommission_receipts(path, data, true).await
17347    }
17348
17349    /// Return true only when `data` is the exact copy still owned by an active
17350    /// decommission source and a target-side terminal receipt authorizes that
17351    /// source's later verified cleanup. Lifecycle recovery may then regard the
17352    /// logical record as terminal without deleting the source checkpoint.
17353    pub(crate) async fn durable_ilm_terminal_receipt_covers_active_source(&self, path: &str, data: &[u8]) -> Result<bool> {
17354        let namespace = classify_durable_ilm_record(path)?
17355            .ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{path}`")))?;
17356        let source_record = validate_durable_ilm_record(path, data)?;
17357        let active_runs = {
17358            let pool_meta = self.pool_meta.read().await;
17359            pool_meta
17360                .pools
17361                .iter()
17362                .enumerate()
17363                .filter_map(|(source_pool_index, pool)| {
17364                    pool.decommission
17365                        .as_ref()
17366                        .filter(|info| info.has_decommission_state() && !info.complete)
17367                        .and_then(|info| info.start_time)
17368                        .map(|start_time| {
17369                            (source_pool_index, decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time))
17370                        })
17371                })
17372                .collect::<Vec<_>>()
17373        };
17374        let mut covered_active_source = false;
17375        for (source_pool_index, run_token) in active_runs {
17376            let source_data =
17377                match read_config_limited_preserve_empty(self.pools[source_pool_index].clone(), path, namespace.max_record_size)
17378                    .await
17379                {
17380                    Ok(source_data) => source_data,
17381                    Err(err)
17382                        if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17383                            || is_err_object_not_found(&err)
17384                            || is_err_version_not_found(&err) =>
17385                    {
17386                        continue;
17387                    }
17388                    Err(err) => return Err(err),
17389                };
17390            if source_data.as_slice() != data {
17391                continue;
17392            }
17393            if self
17394                .load_decommission_durable_ilm_terminal_receipt_for_run(source_pool_index, path, &source_record, &run_token)
17395                .await?
17396                .is_some()
17397            {
17398                covered_active_source = true;
17399            } else {
17400                // Every active source that still stores this exact generation
17401                // needs complete terminal receipt coverage. One covered source
17402                // cannot authorize lifecycle recovery to skip another.
17403                return Ok(false);
17404            }
17405        }
17406        Ok(covered_active_source)
17407    }
17408
17409    async fn cleanup_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> {
17410        for (pool_idx, receipt_path) in self.list_decommission_durable_ilm_receipts(source_pool_idx).await? {
17411            match delete_config(self.pools[pool_idx].clone(), &receipt_path).await {
17412                Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {}
17413                Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
17414                Err(err) => {
17415                    return Err(Error::other(format!(
17416                        "failed to clean durable ILM decommission receipt `{receipt_path}` from pool {pool_idx}: {err}"
17417                    )));
17418                }
17419            }
17420        }
17421        for receipt_path in self.list_decommission_durable_ilm_manifest_receipts(source_pool_idx).await? {
17422            match delete_config(self.pools[source_pool_idx].clone(), &receipt_path).await {
17423                Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {}
17424                Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
17425                Err(err) => {
17426                    return Err(Error::other(format!(
17427                        "failed to clean durable ILM expected manifest receipt `{receipt_path}` from source pool {source_pool_idx}: {err}"
17428                    )));
17429                }
17430            }
17431        }
17432        let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
17433        let manifest_path = decommission_durable_ilm_manifest_path(&run_token);
17434        match delete_config(self.pools[source_pool_idx].clone(), &manifest_path).await {
17435            Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {}
17436            Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
17437            Err(err) => {
17438                return Err(Error::other(format!(
17439                    "failed to clean durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}"
17440                )));
17441            }
17442        }
17443        Ok(())
17444    }
17445
17446    #[cfg(all(test, feature = "test-util"))]
17447    async fn verify_and_cleanup_decommissioned_durable_ilm_record(
17448        &self,
17449        source_pool_idx: usize,
17450        source_set: Arc<SetDisks>,
17451        path: &str,
17452    ) -> Result<()> {
17453        let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
17454        self.verify_and_cleanup_decommissioned_durable_ilm_record_for_run(source_pool_idx, source_set, path, &run_token)
17455            .await
17456    }
17457
17458    async fn verify_and_cleanup_decommissioned_durable_ilm_record_for_run(
17459        &self,
17460        source_pool_idx: usize,
17461        source_set: Arc<SetDisks>,
17462        path: &str,
17463        run_token: &str,
17464    ) -> Result<()> {
17465        let namespace = classify_durable_ilm_record(path)?
17466            .ok_or_else(|| Error::other(format!("path `{path}` is not a durable ILM record")))?;
17467        let source_versions = source_set
17468            .load_file_info_versions_exact(RUSTFS_META_BUCKET, path)
17469            .await
17470            .map_err(|err| Error::other(format!("failed to load source durable ILM versions at path `{path}`: {err}")))?
17471            .ok_or_else(|| Error::other(format!("source durable ILM record is missing at path `{path}`")))?;
17472        let source = read_config_limited_preserve_empty(source_set.clone(), path, namespace.max_record_size)
17473            .await
17474            .map_err(|err| Error::other(format!("failed to read source durable ILM record at path `{path}`: {err}")))?;
17475        let source_record = validate_durable_ilm_record(path, &source)
17476            .map_err(|err| Error::other(format!("source durable ILM record is invalid at path `{path}`: {err}")))?;
17477        let source_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &source)?;
17478        let target = self
17479            .load_decommissioned_durable_ilm_target(source_pool_idx, path, namespace.max_record_size, &source_record.context())
17480            .await?;
17481        let manifest_receipt = if let Some((target_pool_idx, target)) = target {
17482            let target_record = validate_decommission_durable_ilm_copy(path, &source_record, &target)?;
17483            let target_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &target)?;
17484            if target_fleet_topology_generation != source_fleet_topology_generation {
17485                return Err(Error::other_with_context(
17486                    "target durable ILM fleet topology generation differs from source",
17487                    format!("path `{path}` {}", source_record.context()),
17488                ));
17489            }
17490            let receipt = DecommissionDurableIlmReceipt::new(path, &target_record, target_fleet_topology_generation);
17491            self.persist_decommission_durable_ilm_receipt_for_run(target_pool_idx, &receipt, run_token)
17492                .await?;
17493            receipt
17494        } else {
17495            self.load_decommission_durable_ilm_terminal_receipt_for_run(source_pool_idx, path, &source_record, run_token)
17496                .await?
17497                .ok_or_else(|| {
17498                    Error::other(format!(
17499                        "target durable ILM record is missing at path `{path}` {} without a matching terminal receipt",
17500                        source_record.context()
17501                    ))
17502                })?
17503        };
17504        if manifest_receipt.fleet_topology_generation != source_fleet_topology_generation {
17505            return Err(Error::other_with_context(
17506                "terminal durable ILM receipt fleet topology generation does not cover source",
17507                format!("path `{path}` {}", source_record.context()),
17508            ));
17509        }
17510        let fleet_proof = if let Some(expected_generation) = source_fleet_topology_generation.as_deref() {
17511            let proof = acquire_tier_delete_journal_fleet_proof()
17512                .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable for source cleanup"))?;
17513            if tier_delete_journal_topology_generation(&proof) != expected_generation
17514                || !tier_delete_journal_fleet_proof_matches(&proof)
17515            {
17516                return Err(Error::other("tier delete journal v6 fleet generation changed before source cleanup"));
17517            }
17518            Some(proof)
17519        } else {
17520            None
17521        };
17522        self.persist_decommission_durable_ilm_receipt_for_run(source_pool_idx, &manifest_receipt, run_token)
17523            .await?;
17524
17525        if fleet_proof
17526            .as_ref()
17527            .is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof))
17528        {
17529            return Err(Error::other("tier delete journal v6 fleet proof changed before source cleanup"));
17530        }
17531
17532        let cleanup_result = data_movement::cleanup_source_entry_if_unchanged(
17533            source_set,
17534            RUSTFS_META_BUCKET,
17535            path,
17536            &source_versions,
17537            &[],
17538            data_movement::SourceCleanupBucketFence::default(),
17539            "decommission durable ILM final sweep",
17540        )
17541        .await
17542        .map_err(|err| {
17543            Error::other(format!(
17544                "source durable ILM cleanup failed at path `{path}` {}: {err}",
17545                source_record.context()
17546            ))
17547        });
17548        let cleanup_result = resolve_decommission_entry_cleanup_delete_result(cleanup_result, RUSTFS_META_BUCKET, path);
17549        if fleet_proof
17550            .as_ref()
17551            .is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof))
17552        {
17553            return Err(Error::other(
17554                "tier delete journal v6 fleet proof changed during source cleanup; exact source outcome requires verification",
17555            ));
17556        }
17557        cleanup_result
17558    }
17559
17560    #[cfg(all(test, feature = "test-util"))]
17561    pub(crate) async fn verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
17562        &self,
17563        source_pool_idx: usize,
17564        source_set: Arc<SetDisks>,
17565        path: &str,
17566    ) -> Result<()> {
17567        self.verify_and_cleanup_decommissioned_durable_ilm_record(source_pool_idx, source_set, path)
17568            .await
17569    }
17570
17571    #[cfg(all(test, feature = "test-util"))]
17572    pub(crate) async fn decommission_durable_ilm_receipt_count_for_test(&self, source_pool_idx: usize) -> Result<usize> {
17573        Ok(self.list_decommission_durable_ilm_receipts(source_pool_idx).await?.len())
17574    }
17575
17576    #[cfg(all(test, feature = "test-util"))]
17577    pub(crate) async fn decommission_durable_ilm_receipt_paths_for_test(
17578        &self,
17579        source_pool_idx: usize,
17580    ) -> Result<Vec<(usize, String)>> {
17581        self.list_decommission_durable_ilm_receipts(source_pool_idx).await
17582    }
17583
17584    #[cfg(all(test, feature = "test-util"))]
17585    pub(crate) async fn persist_decommission_durable_ilm_receipt_for_test(
17586        &self,
17587        source_pool_idx: usize,
17588        target_pool_idx: usize,
17589        source_path: &str,
17590        record: &ValidatedDurableIlmRecord,
17591        terminal: bool,
17592    ) -> Result<String> {
17593        let fleet_topology_generation = match read_config_limited_preserve_empty(
17594            self.pools[target_pool_idx].clone(),
17595            source_path,
17596            classify_durable_ilm_record(source_path)?
17597                .ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{source_path}`")))?
17598                .max_record_size,
17599        )
17600        .await
17601        {
17602            Ok(target_data) => durable_ilm_v6_topology_generation(source_path, &target_data)?,
17603            Err(err)
17604                if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17605                    || is_err_object_not_found(&err)
17606                    || is_err_version_not_found(&err) =>
17607            {
17608                None
17609            }
17610            Err(err) => return Err(err),
17611        };
17612        let mut receipt = DecommissionDurableIlmReceipt::new(source_path, record, fleet_topology_generation);
17613        if terminal {
17614            receipt.terminal_checkpoint = Some(record.checkpoint.clone());
17615        }
17616        self.persist_decommission_durable_ilm_receipt(source_pool_idx, target_pool_idx, &receipt)
17617            .await?;
17618        self.decommission_durable_ilm_receipt_path_for_test(source_pool_idx, source_path, record)
17619            .await
17620    }
17621
17622    #[cfg(all(test, feature = "test-util"))]
17623    pub(crate) async fn decommission_durable_ilm_receipt_path_for_test(
17624        &self,
17625        source_pool_idx: usize,
17626        source_path: &str,
17627        record: &ValidatedDurableIlmRecord,
17628    ) -> Result<String> {
17629        let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
17630        Ok(decommission_durable_ilm_receipt_path(&run_token, source_path, record.id_kind, &record.id))
17631    }
17632
17633    #[cfg(all(test, feature = "test-util"))]
17634    pub(crate) async fn persist_decommission_durable_ilm_manifest_for_test(&self, source_pool_idx: usize) -> Result<()> {
17635        self.persist_decommission_durable_ilm_manifest(source_pool_idx).await
17636    }
17637
17638    #[cfg(all(test, feature = "test-util"))]
17639    pub(crate) async fn cleanup_decommission_durable_ilm_receipts_for_test(&self, source_pool_idx: usize) -> Result<()> {
17640        self.cleanup_decommission_durable_ilm_receipts(source_pool_idx).await
17641    }
17642
17643    async fn check_after_decommission(
17644        self: &Arc<Self>,
17645        idx: usize,
17646        rx: &CancellationToken,
17647        generation: OffsetDateTime,
17648    ) -> Result<Vec<DecommissionUnresolvedEntry>> {
17649        let run_token = self.durable_ilm_receipt_run_token_for_generation(idx, generation).await?;
17650        let operation_gate = self.ctx.data_movement_operation_gate();
17651        self.run_guarded_decommission_side_effect(rx, &operation_gate, || {
17652            self.check_after_decommission_unfenced(idx, generation, run_token)
17653        })
17654        .await
17655    }
17656
17657    async fn check_after_decommission_unfenced(
17658        self: &Arc<Self>,
17659        idx: usize,
17660        generation: OffsetDateTime,
17661        run_token: String,
17662    ) -> Result<Vec<DecommissionUnresolvedEntry>> {
17663        let unresolved_entries = {
17664            let pool_meta = self.pool_meta.read().await;
17665            ensure_decommission_generation(&pool_meta, idx, generation)?;
17666            let info = pool_meta.pools[idx]
17667                .decommission
17668                .as_ref()
17669                .ok_or_else(|| decommission_metadata_not_initialized_error("verify unresolved decommission entries"))?;
17670            if info
17671                .unresolved_entries
17672                .iter()
17673                .any(|entry| entry.pool_index != idx || entry.source_generation != generation)
17674            {
17675                return Err(Error::other(format!(
17676                    "failed to verify decommission for pool {idx}: unresolved listing ledger contains a different pool or generation"
17677                )));
17678            }
17679            info.unresolved_entries.clone()
17680        };
17681        let unresolved_entries_by_identity = Arc::new(
17682            unresolved_entries
17683                .iter()
17684                .map(|entry| (decommission_unresolved_entry_identity(entry), entry.clone()))
17685                .collect::<HashMap<_, _>>(),
17686        );
17687        let resolved_unresolved_entries = Arc::new(tokio::sync::Mutex::new(Vec::new()));
17688        let buckets = self.get_buckets_to_decommission().await?;
17689        let pool = self.pools[idx].clone();
17690
17691        self.ensure_decommission_multipart_uploads_drained(idx, &pool, &buckets)
17692            .await?;
17693
17694        for (set_index, set) in pool.disk_set.iter().enumerate() {
17695            let require_all_disks = unresolved_entries.iter().any(|entry| entry.set_index == set_index);
17696            for bucket_info in &buckets {
17697                let mut lifecycle_config = None;
17698                let mut object_lock_config = None;
17699                let mut replication_configured = false;
17700                if bucket_info.name != RUSTFS_META_BUCKET {
17701                    let expiry_configs = get_expiry_configs(self, &bucket_info.name).await?;
17702                    lifecycle_config = expiry_configs.lifecycle.map(|config| (*config).clone());
17703                    object_lock_config = expiry_configs.object_lock.map(|config| (*config).clone());
17704                    replication_configured = resolve_decommission_optional_bucket_config_result(
17705                        &bucket_info.name,
17706                        "replication",
17707                        metadata_sys::get_replication_config_in(&self.ctx, &bucket_info.name).await,
17708                    )?
17709                    .is_some();
17710                }
17711
17712                let versions_found = Arc::new(AtomicUsize::new(0));
17713                let entry_error = Arc::new(tokio::sync::Mutex::new(None::<Error>));
17714                let first_remaining_path = Arc::new(tokio::sync::Mutex::new(None::<String>));
17715                let callback_rx = CancellationToken::new();
17716                let versions_found_cb = versions_found.clone();
17717                let entry_error_cb = entry_error.clone();
17718                let first_remaining_path_cb = first_remaining_path.clone();
17719                let bucket_name = bucket_info.name.clone();
17720                let lifecycle_config_cb = lifecycle_config.clone();
17721                let object_lock_config_cb = object_lock_config.clone();
17722                let replication_configured_cb = replication_configured;
17723                let store = Arc::clone(self);
17724                let source_set = set.clone();
17725                let callback_rx_cb = callback_rx.clone();
17726                let unresolved_entries_by_identity_cb = unresolved_entries_by_identity.clone();
17727                let resolved_unresolved_entries_cb = resolved_unresolved_entries.clone();
17728                let run_token_cb = run_token.clone();
17729
17730                let callback: ListCallback = Arc::new(move |entry: MetaCacheEntry| {
17731                    let versions_found = versions_found_cb.clone();
17732                    let entry_error = entry_error_cb.clone();
17733                    let first_remaining_path = first_remaining_path_cb.clone();
17734                    let bucket_name = bucket_name.clone();
17735                    let lifecycle_config = lifecycle_config_cb.clone();
17736                    let object_lock_config = object_lock_config_cb.clone();
17737                    let replication_configured = replication_configured_cb;
17738                    let store = Arc::clone(&store);
17739                    let source_set = source_set.clone();
17740                    let callback_rx = callback_rx_cb.clone();
17741                    let unresolved_entries_by_identity = unresolved_entries_by_identity_cb.clone();
17742                    let resolved_unresolved_entries = resolved_unresolved_entries_cb.clone();
17743                    let run_token = run_token_cb.clone();
17744                    Box::pin(async move {
17745                        if callback_rx.is_cancelled() {
17746                            return;
17747                        }
17748
17749                        if !entry.is_object() {
17750                            return;
17751                        }
17752
17753                        if is_decommission_set_local_usage_cache(&bucket_name, &entry.name) {
17754                            return;
17755                        }
17756
17757                        let durable_ilm_record = if bucket_name == RUSTFS_META_BUCKET {
17758                            match classify_durable_ilm_record(&entry.name) {
17759                                Ok(record) => record,
17760                                Err(err) => {
17761                                    let mut first_err = entry_error.lock().await;
17762                                    if first_err.is_none() {
17763                                        *first_err = Some(with_decommission_entry_context(
17764                                            "check_after_decommission.durable_ilm_namespace",
17765                                            &bucket_name,
17766                                            &entry.name,
17767                                            err,
17768                                        ));
17769                                        callback_rx.cancel();
17770                                    }
17771                                    return;
17772                                }
17773                            }
17774                        } else {
17775                            None
17776                        };
17777
17778                        if durable_ilm_record.is_some() {
17779                            if let Err(err) = store
17780                                .verify_and_cleanup_decommissioned_durable_ilm_record_for_run(
17781                                    idx,
17782                                    source_set,
17783                                    &entry.name,
17784                                    &run_token,
17785                                )
17786                                .await
17787                            {
17788                                let mut first_err = entry_error.lock().await;
17789                                if first_err.is_none() {
17790                                    *first_err = Some(err);
17791                                    callback_rx.cancel();
17792                                }
17793                            }
17794                            return;
17795                        }
17796
17797                        let mut fivs = match load_decommission_entry_exact_versions(
17798                            &source_set,
17799                            &entry,
17800                            &bucket_name,
17801                            "check_after_decommission.file_info_versions",
17802                        )
17803                        .await
17804                        {
17805                            Ok(fivs) => fivs,
17806                            Err(err) => {
17807                                let mut first_err = entry_error.lock().await;
17808                                if first_err.is_none() {
17809                                    *first_err = Some(err);
17810                                    callback_rx.cancel();
17811                                }
17812                                return;
17813                            }
17814                        };
17815
17816                        fivs.versions
17817                            .sort_by_key(|version| (version.mod_time.is_none(), std::cmp::Reverse(version.mod_time)));
17818
17819                        let mut remaining = 0;
17820                        let mut expired = 0;
17821                        for version in fivs.versions.iter().chain(fivs.free_versions.iter()) {
17822                            if version.tier_free_version() {
17823                                remaining += 1;
17824                                debug!(
17825                                    event = EVENT_DECOMMISSION_ENTRY,
17826                                    component = LOG_COMPONENT_ECSTORE,
17827                                    subsystem = LOG_SUBSYSTEM_POOLS,
17828                                    pool_index = idx,
17829                                    bucket = %bucket_name,
17830                                    object = %entry.name,
17831                                    version_id = ?version.version_id,
17832                                    reason = DECOMMISSION_FREE_VERSION_SWEEP_REASON,
17833                                    state = "free_version_retained",
17834                                    "Decommission final sweep retained a free version"
17835                                );
17836                                continue;
17837                            }
17838                            let skip_lifecycle = match should_skip_lifecycle_for_data_movement(
17839                                Arc::clone(&store),
17840                                &bucket_name,
17841                                version,
17842                                lifecycle_config.as_ref(),
17843                                object_lock_config.as_ref(),
17844                                false,
17845                                &LcEventSrc::Decom,
17846                                None,
17847                            )
17848                            .await
17849                            {
17850                                Ok(skip_lifecycle) => skip_lifecycle,
17851                                Err(err) => {
17852                                    let mut first_err = entry_error.lock().await;
17853                                    if first_err.is_none() {
17854                                        *first_err = Some(err);
17855                                        callback_rx.cancel();
17856                                    }
17857                                    return;
17858                                }
17859                            };
17860                            if skip_lifecycle {
17861                                expired += 1;
17862                                continue;
17863                            }
17864                            let remaining_versions = decommission_remaining_version_count(&fivs.versions, expired);
17865                            if should_skip_decommission_delete_marker(version, remaining_versions, replication_configured) {
17866                                continue;
17867                            }
17868                            remaining += 1;
17869                        }
17870
17871                        if remaining > 0 {
17872                            let mut first_path = first_remaining_path.lock().await;
17873                            if first_path.is_none() {
17874                                *first_path = Some(format!("{bucket_name}/{}", entry.name));
17875                            }
17876                        } else {
17877                            let identity = (set_index, bucket_name.clone(), entry.name.clone());
17878                            if let Some(unresolved_entry) = unresolved_entries_by_identity.get(&identity) {
17879                                let mut resolved = resolved_unresolved_entries.lock().await;
17880                                if !resolved.contains(unresolved_entry) {
17881                                    resolved.push(unresolved_entry.clone());
17882                                }
17883                            }
17884                        }
17885
17886                        versions_found.fetch_add(remaining, Ordering::Relaxed);
17887                    })
17888                });
17889
17890                let list_result = set
17891                    .list_objects_to_decommission(
17892                        self.clone(),
17893                        callback_rx,
17894                        bucket_info.clone(),
17895                        callback,
17896                        entry_error.clone(),
17897                        idx,
17898                        set_index,
17899                        generation,
17900                        require_all_disks,
17901                    )
17902                    .await;
17903                let entry_error = entry_error.lock().await.clone();
17904                resolve_decommission_check_after_list_result(list_result, entry_error)?;
17905
17906                let versions_found = versions_found.load(Ordering::Relaxed);
17907                if versions_found > 0 {
17908                    let first_remaining_path = first_remaining_path
17909                        .lock()
17910                        .await
17911                        .clone()
17912                        .unwrap_or_else(|| format!("{}/<unknown>", bucket_info.name));
17913                    return Err(Error::other(format!(
17914                        "at least {versions_found} object(s)/version(s) were found in bucket `{}` after decommissioning; first remaining path `{first_remaining_path}`",
17915                        bucket_info.name,
17916                    )));
17917                }
17918            }
17919        }
17920
17921        let mut verified_unresolved_entries = resolved_unresolved_entries.lock().await.clone();
17922        for entry in &unresolved_entries {
17923            let set = pool.disk_set.get(entry.set_index).ok_or_else(|| {
17924                Error::other(format!(
17925                    "failed to verify decommission for pool {idx}: unresolved listing entry references missing set {}",
17926                    entry.set_index
17927                ))
17928            })?;
17929            if set.decommission_unresolved_entry_absent_on_all_disks(idx, entry).await?
17930                && !verified_unresolved_entries.contains(entry)
17931            {
17932                verified_unresolved_entries.push(entry.clone());
17933            }
17934            if !verified_unresolved_entries.contains(entry) {
17935                return Err(Error::other(format!(
17936                    "failed to verify decommission for pool {idx}: unresolved listing entry {}/{} in set {} was neither re-observed as resolved nor absent on every source disk",
17937                    entry.bucket, entry.object, entry.set_index
17938                )));
17939            }
17940        }
17941
17942        self.persist_decommission_durable_ilm_manifest_for_run(idx, &run_token)
17943            .await?;
17944        self.verify_decommission_durable_ilm_receipts_for_run(idx, &run_token).await?;
17945
17946        Ok(verified_unresolved_entries)
17947    }
17948
17949    async fn ensure_decommission_multipart_uploads_drained(
17950        &self,
17951        idx: usize,
17952        pool: &Sets,
17953        buckets: &[DecomBucketInfo],
17954    ) -> Result<()> {
17955        let mut bucket_names = buckets
17956            .iter()
17957            .filter(|bucket| bucket.name != RUSTFS_META_BUCKET)
17958            .map(|bucket| bucket.name.as_str())
17959            .collect::<Vec<_>>();
17960        bucket_names.sort_unstable();
17961        bucket_names.dedup();
17962
17963        // Take one bucket fence at a time so cross-bucket COPY cannot form an
17964        // ABBA cycle. Suspension prevents new source uploads after each fence.
17965        for bucket in bucket_names {
17966            let lifecycle_guard = self.acquire_bucket_lifecycle_write_lock(bucket).await?;
17967            if lifecycle_guard.is_lock_lost() {
17968                return Err(Error::other(format!(
17969                    "decommission multipart drain lost the bucket lifecycle fence for `{bucket}`"
17970                )));
17971            }
17972            for set in &pool.disk_set {
17973                if let Some(upload_path) = set.first_multipart_upload_path_for_decommission(bucket).await? {
17974                    return Err(Error::other(format!(
17975                        "pool {idx} still contains multipart upload `{upload_path}` for bucket `{bucket}`; resolve it before retrying decommission"
17976                    )));
17977                }
17978            }
17979        }
17980
17981        Ok(())
17982    }
17983
17984    #[cfg(test)]
17985    pub(crate) async fn ensure_decommission_multipart_uploads_drained_for_test(self: &Arc<Self>, idx: usize) -> Result<()> {
17986        let buckets = self.get_buckets_to_decommission().await?;
17987        let pool = self.pools[idx].clone();
17988        self.ensure_decommission_multipart_uploads_drained(idx, pool.as_ref(), &buckets)
17989            .await
17990    }
17991    #[cfg(all(test, feature = "test-util"))]
17992    pub(crate) async fn check_after_decommission_for_test(self: &Arc<Self>, idx: usize) -> Result<()> {
17993        let generation = self.active_decommission_generation(idx).await?;
17994        self.check_after_decommission(idx, &CancellationToken::new(), generation)
17995            .await
17996            .map(|_| ())
17997    }
17998
17999    #[tracing::instrument(skip(self, rd))]
18000    async fn decommission_object(
18001        self: Arc<Self>,
18002        pool_idx: usize,
18003        bucket: String,
18004        rd: GetObjectReader,
18005        expected_bucket_incarnation_id: Option<uuid::Uuid>,
18006        capacity_owner: Option<DecommissionCapacityOwner>,
18007    ) -> Result<()> {
18008        warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name);
18009        let object_name = rd.object_info.name.clone();
18010        let mut migration = tokio::task::JoinSet::new();
18011        migration.spawn(data_movement::migrate_decommission_object(
18012            self,
18013            pool_idx,
18014            bucket.clone(),
18015            rd,
18016            expected_bucket_incarnation_id,
18017            "decommission_object",
18018            capacity_owner,
18019        ));
18020        let result = migration
18021            .join_next()
18022            .await
18023            .ok_or_else(|| Error::other("decommission migration task was not started"))?
18024            .map_err(|err| Error::other(format!("decommission migration task join error: {err}")))?;
18025        if result.is_ok() {
18026            warn!("decommission_object: migrated {} {}", &bucket, &object_name);
18027        }
18028        result
18029    }
18030}
18031
18032#[cfg(test)]
18033async fn persist_v3_pool_meta_for_test(store: &Arc<ECStore>) {
18034    let mut meta = PoolMeta::default();
18035    meta.load_no_lock_from_replicas(store.pools.clone())
18036        .await
18037        .expect("the baseline pool metadata should be readable before V3 migration");
18038    meta.version = POOL_META_GENERATION_VERSION;
18039    let mut write_state = store.pool_meta_save_gate.lock().await.clone();
18040    temp_env::async_with_vars(
18041        [
18042            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("true")),
18043            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("true")),
18044        ],
18045        async {
18046            meta.save_no_lock_observing(store.pools.clone(), &mut write_state)
18047                .await
18048                .expect("the baseline metadata should be upgraded to a durable V3 replica");
18049        },
18050    )
18051    .await;
18052
18053    let mut loaded = PoolMeta::default();
18054    loaded
18055        .load_no_lock_from_replicas(store.pools.clone())
18056        .await
18057        .expect("the upgraded V3 pool metadata should be readable");
18058    assert_eq!(loaded.version, POOL_META_GENERATION_VERSION);
18059    store.pool_meta.write().await.version = POOL_META_GENERATION_VERSION;
18060}
18061
18062#[cfg(test)]
18063#[allow(clippy::items_after_test_module)]
18064mod tests {
18065    use super::*;
18066    use crate::bucket::replication::{ReplicationState, ReplicationStatusType};
18067    use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
18068    use crate::storage_api_contracts::multipart::MultipartOperations as _;
18069    use serde::Serialize;
18070
18071    #[test]
18072    fn decommission_set_local_usage_cache_classification_is_exact() {
18073        for object in [
18074            "buckets/.usage-cache.bin",
18075            "buckets/.usage-cache.bin.bkp",
18076            "buckets/photos/.usage-cache.bin",
18077            "buckets/photos/.usage-cache.bin.bkp",
18078        ] {
18079            assert!(is_decommission_set_local_usage_cache(RUSTFS_META_BUCKET, object), "{object}");
18080            assert!(!is_decommission_set_local_usage_cache("user-bucket", object), "{object}");
18081        }
18082        for object in [
18083            "buckets/.usage.v2.json",
18084            "buckets/.usage.v2.json.bkp",
18085            "buckets/.usage-cache.bin.extra",
18086            "buckets/.usage-cache.bin.bkp.extra",
18087            "buckets/prefix.usage-cache.bin",
18088            "buckets/photos/.usage-cache.bin.bkp.bkp",
18089            "buckets/photos/nested/.usage-cache.bin",
18090            "buckets//.usage-cache.bin",
18091            "buckets/../.usage-cache.bin",
18092            "buckets/./.usage-cache.bin",
18093            "buckets/.usage-cache.bin/child",
18094            "config/.usage-cache.bin",
18095            "buckets-other/.usage-cache.bin",
18096            ".usage-cache.bin",
18097        ] {
18098            assert!(!is_decommission_set_local_usage_cache(RUSTFS_META_BUCKET, object), "{object}");
18099        }
18100    }
18101
18102    #[tokio::test]
18103    #[serial_test::serial]
18104    async fn decommission_keeps_set_local_usage_caches_out_of_target_capacity() {
18105        use crate::object_api::PutObjReader;
18106        use tokio::io::AsyncReadExt as _;
18107
18108        // Keep the scenario's large setup and migration futures off the test
18109        // future so ordinary metadata I/O retains the default thread stack.
18110        let (_temp_dirs, store, _other_store) =
18111            Box::pin(crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None)).await;
18112        let user_bucket = "decommission-usage-cache-control";
18113        Box::pin(store.make_bucket(user_bucket, &MakeBucketOptions::default()))
18114            .await
18115            .expect("create the ordinary-object control bucket");
18116        let incarnation = Box::pin(store.bucket_incarnation_id(user_bucket))
18117            .await
18118            .expect("control bucket incarnation");
18119        let source_time = OffsetDateTime::now_utc();
18120        let cache_objects = [
18121            "buckets/.usage-cache.bin",
18122            "buckets/.usage-cache.bin.bkp",
18123            "buckets/photos/.usage-cache.bin",
18124            "buckets/photos/.usage-cache.bin.bkp",
18125        ];
18126        let conflict_object = "buckets/.usage-cache.bin.conflict";
18127        let source_read_failure_object = "buckets/.usage-cache.bin.read-error";
18128        let source_body = b"source set cache";
18129        let target_body = b"independent older target set cache";
18130        for object in cache_objects.into_iter().chain([conflict_object]) {
18131            for (pool_index, body, mod_time) in [
18132                (0, source_body.as_slice(), source_time),
18133                (1, target_body.as_slice(), source_time - Duration::seconds(1)),
18134            ] {
18135                // Scanner cache persistence writes directly to its own set.
18136                store.pools[pool_index]
18137                    .get_disks_by_key(object)
18138                    .put_object(
18139                        RUSTFS_META_BUCKET,
18140                        object,
18141                        &mut PutObjReader::from_vec(body.to_vec()),
18142                        &ObjectOptions {
18143                            mod_time: Some(mod_time),
18144                            ..Default::default()
18145                        },
18146                    )
18147                    .await
18148                    .expect("seed distinct native set-local objects");
18149            }
18150        }
18151        let controls = [
18152            (RUSTFS_META_BUCKET, "buckets/.usage.v2.json"),
18153            (RUSTFS_META_BUCKET, "buckets/photos/.usage-cache.bin.extra"),
18154            (user_bucket, "ordinary-object"),
18155            (user_bucket, "buckets/.usage-cache.bin"),
18156        ];
18157        for (bucket, object) in controls {
18158            store.pools[0]
18159                .put_object(
18160                    bucket,
18161                    object,
18162                    &mut PutObjReader::from_vec(b"ordinary object contents".to_vec()),
18163                    &ObjectOptions {
18164                        expected_bucket_incarnation_id: (bucket == user_bucket).then_some(incarnation),
18165                        ..Default::default()
18166                    },
18167                )
18168                .await
18169                .expect("seed a control that must migrate");
18170        }
18171        store.pools[0]
18172            .put_object(
18173                RUSTFS_META_BUCKET,
18174                source_read_failure_object,
18175                &mut PutObjReader::from_vec(source_body.to_vec()),
18176                &ObjectOptions::default(),
18177            )
18178            .await
18179            .expect("seed a similarly named object whose source read will fail");
18180
18181        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
18182        set_decommission_capacity_info_overrides_for_test(
18183            store.id,
18184            vec![vec![
18185                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 16_384, 16_384),
18186                DecommissionPoolCapacityInfo::for_test(1, layout, 131_072, 131_072, 0),
18187            ]],
18188        );
18189        Box::pin(store.save_current_pool_meta_for_decommission_start(&[0], Vec::new()))
18190            .await
18191            .expect("activate the decommission capacity reservation");
18192
18193        for object in cache_objects {
18194            Box::pin(store.decommission_entry_for_test(
18195                0,
18196                MetaCacheEntry {
18197                    name: object.to_string(),
18198                    ..Default::default()
18199                },
18200                RUSTFS_META_BUCKET.to_string(),
18201                store.pools[0].get_disks_by_key(object),
18202            ))
18203            .await
18204            .expect("set-local cache must not enter cross-pool migration");
18205            for (pool_index, expected) in [(0, source_body.as_slice()), (1, target_body.as_slice())] {
18206                let mut reader = store.pools[pool_index]
18207                    .get_disks_by_key(object)
18208                    .get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
18209                    .await
18210                    .expect("each set must retain its own cache");
18211                let mut actual = Vec::new();
18212                reader.stream.read_to_end(&mut actual).await.expect("read retained cache");
18213                assert_eq!(actual, expected, "pool {pool_index}, {object}");
18214            }
18215            let meta = store.pool_meta.read().await;
18216            let info = meta.pools[0].decommission.as_ref().expect("decommission progress");
18217            assert_eq!((info.items_decommissioned, info.items_decommission_failed), (0, 0));
18218            assert_eq!((info.bytes_done, info.bytes_failed), (0, 0));
18219            let reservation = info.capacity_reservation.as_ref().expect("capacity reservation");
18220            assert_eq!(reservation.pending_target_physical_bytes, 0);
18221            assert_eq!(reservation.consumed_target_physical_bytes, 0);
18222            assert!(reservation.targets.iter().all(|target| target.pending_mutation_id.is_none()));
18223        }
18224        let mut persisted = PoolMeta::default();
18225        Box::pin(persisted.load_no_lock_from_replicas(store.pools.clone()))
18226            .await
18227            .expect("reload durable capacity intents after cache entries");
18228        let reservation = persisted.pools[0]
18229            .decommission
18230            .as_ref()
18231            .and_then(|info| info.capacity_reservation.as_ref())
18232            .expect("durable reservation");
18233        assert_eq!(reservation.pending_target_physical_bytes, 0);
18234        assert!(reservation.targets.iter().all(|target| target.pending_mutation_id.is_none()));
18235
18236        let injected_reads = Arc::new(AtomicUsize::new(0));
18237        let observed_reads = Arc::clone(&injected_reads);
18238        let read_fault = DecommissionTestFaultGuard::install(Arc::new(move |stage, bucket, object, _, success| {
18239            if stage == "object_read" && bucket == RUSTFS_META_BUCKET && object == source_read_failure_object && success {
18240                observed_reads.fetch_add(1, Ordering::SeqCst);
18241                return true;
18242            }
18243            false
18244        }));
18245        Box::pin(store.decommission_entry_for_test(
18246            0,
18247            MetaCacheEntry {
18248                name: source_read_failure_object.to_string(),
18249                ..Default::default()
18250            },
18251            RUSTFS_META_BUCKET.to_string(),
18252            store.pools[0].get_disks_by_key(source_read_failure_object),
18253        ))
18254        .await
18255        .expect("entry must record the non-NotFound source read failure");
18256        drop(read_fault);
18257        assert_eq!(injected_reads.load(Ordering::SeqCst), DECOMMISSION_VERSION_COPY_ATTEMPTS);
18258        {
18259            let meta = store.pool_meta.read().await;
18260            let info = meta.pools[0].decommission.as_ref().expect("source read failure progress");
18261            assert_eq!((info.items_decommissioned, info.items_decommission_failed), (0, 1));
18262            assert_eq!(info.bytes_failed, source_body.len());
18263            assert_eq!(
18264                info.capacity_reservation
18265                    .as_ref()
18266                    .expect("reservation")
18267                    .pending_target_physical_bytes,
18268                0
18269            );
18270        }
18271        let mut retained = store.pools[0]
18272            .get_object_reader(
18273                RUSTFS_META_BUCKET,
18274                source_read_failure_object,
18275                None,
18276                HeaderMap::new(),
18277                &ObjectOptions::default(),
18278            )
18279            .await
18280            .expect("source read failure must retain the source");
18281        let mut retained_body = Vec::new();
18282        retained
18283            .stream
18284            .read_to_end(&mut retained_body)
18285            .await
18286            .expect("read retained source");
18287        assert_eq!(retained_body, source_body);
18288        drop(retained);
18289        let target_err = store.pools[1]
18290            .get_object_info(RUSTFS_META_BUCKET, source_read_failure_object, &ObjectOptions::default())
18291            .await
18292            .expect_err("failed source read must not create a target object");
18293        assert!(is_err_object_not_found(&target_err), "unexpected target state: {target_err:?}");
18294
18295        for (bucket, object) in controls {
18296            Box::pin(store.decommission_entry_for_test(
18297                0,
18298                MetaCacheEntry {
18299                    name: object.to_string(),
18300                    ..Default::default()
18301                },
18302                bucket.to_string(),
18303                store.pools[0].get_disks_by_key(object),
18304            ))
18305            .await
18306            .expect("ordinary and similarly named objects must migrate");
18307            let mut reader = store.pools[1]
18308                .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
18309                .await
18310                .expect("control must exist on the target");
18311            let mut actual = Vec::new();
18312            reader.stream.read_to_end(&mut actual).await.expect("read migrated control");
18313            assert_eq!(actual, b"ordinary object contents", "{bucket}/{object}");
18314            let err = store.pools[0]
18315                .get_object_info(bucket, object, &ObjectOptions::default())
18316                .await
18317                .expect_err("migrated control must be removed from the source");
18318            assert!(is_err_object_not_found(&err), "{bucket}/{object}: {err:?}");
18319        }
18320
18321        Box::pin(store.decommission_entry_for_test(
18322            0,
18323            MetaCacheEntry {
18324                name: conflict_object.to_string(),
18325                ..Default::default()
18326            },
18327            RUSTFS_META_BUCKET.to_string(),
18328            store.pools[0].get_disks_by_key(conflict_object),
18329        ))
18330        .await
18331        .expect("entry must record a real conditional-copy failure");
18332        let meta = store.pool_meta.read().await;
18333        let info = meta.pools[0].decommission.as_ref().expect("final progress");
18334        assert_eq!(info.items_decommissioned, controls.len());
18335        assert_eq!(
18336            info.items_decommission_failed, 2,
18337            "similar names must not hide read or migration failures"
18338        );
18339        assert_eq!(info.bytes_failed, source_body.len() * 2);
18340        drop(meta);
18341        for (pool_index, expected) in [(0, source_body.as_slice()), (1, target_body.as_slice())] {
18342            let mut reader = store.pools[pool_index]
18343                .get_object_reader(RUSTFS_META_BUCKET, conflict_object, None, HeaderMap::new(), &ObjectOptions::default())
18344                .await
18345                .expect("failed migration must preserve both objects");
18346            let mut actual = Vec::new();
18347            reader.stream.read_to_end(&mut actual).await.expect("read conflict object");
18348            assert_eq!(actual, expected);
18349        }
18350    }
18351
18352    #[tokio::test]
18353    #[serial_test::serial]
18354    async fn decommission_final_sweep_excludes_only_set_local_usage_caches() {
18355        use crate::object_api::PutObjReader;
18356
18357        let (_temp_dirs, store, _other_store) =
18358            crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
18359        for object in [
18360            "buckets/.usage-cache.bin",
18361            "buckets/.usage-cache.bin.bkp",
18362            "buckets/photos/.usage-cache.bin",
18363            "buckets/photos/.usage-cache.bin.bkp",
18364        ] {
18365            store.pools[0]
18366                .get_disks_by_key(object)
18367                .put_object(
18368                    RUSTFS_META_BUCKET,
18369                    object,
18370                    &mut PutObjReader::from_vec(b"set-local cache".to_vec()),
18371                    &ObjectOptions::default(),
18372                )
18373                .await
18374                .expect("seed each supported set-local cache path");
18375        }
18376        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
18377        set_decommission_capacity_info_overrides_for_test(
18378            store.id,
18379            vec![vec![
18380                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 16_384, 16_384),
18381                DecommissionPoolCapacityInfo::for_test(1, layout, 131_072, 131_072, 0),
18382            ]],
18383        );
18384        store
18385            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
18386            .await
18387            .expect("activate the final-sweep generation");
18388        let generation = store.active_decommission_generation(0).await.expect("active generation");
18389        store
18390            .check_after_decommission(0, &CancellationToken::new(), generation)
18391            .await
18392            .expect("the four set-local cache forms must not block the final sweep");
18393
18394        for object in [
18395            "buckets/.usage-cache.bin.extra",
18396            "buckets/photos/.usage-cache.bin.bkp.extra",
18397            "buckets/.usage.v2.json",
18398        ] {
18399            let source_set = store.pools[0].get_disks_by_key(object);
18400            source_set
18401                .put_object(
18402                    RUSTFS_META_BUCKET,
18403                    object,
18404                    &mut PutObjReader::from_vec(b"unmigrated ordinary metadata".to_vec()),
18405                    &ObjectOptions::default(),
18406                )
18407                .await
18408                .expect("seed ordinary metadata that must prevent completion");
18409            let err = store
18410                .check_after_decommission(0, &CancellationToken::new(), generation)
18411                .await
18412                .expect_err("a remaining similar name or global usage snapshot must block completion");
18413            assert!(err.to_string().contains("after decommissioning"), "unexpected final-sweep error: {err:?}");
18414            assert!(err.to_string().contains(object), "the final sweep must identify {object}: {err:?}");
18415            source_set
18416                .delete_object(RUSTFS_META_BUCKET, object, ObjectOptions::default())
18417                .await
18418                .expect("remove only the ordinary-metadata control before the next sweep");
18419        }
18420        store
18421            .check_after_decommission(0, &CancellationToken::new(), generation)
18422            .await
18423            .expect("only the four set-local caches remain after removing the controls");
18424    }
18425
18426    #[test]
18427    fn pool_activation_fleet_proof_error_classifier_matches_only_retryable_proof_failures() {
18428        assert!(is_pool_activation_fleet_proof_error(&Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED)));
18429        assert!(is_pool_activation_fleet_proof_error(&Error::other(POOL_ACTIVATION_FLEET_PROOF_EXPIRED)));
18430        let wrapped = format!("rebalance meta save failed during start_rebalance: {POOL_ACTIVATION_FLEET_PROOF_EXPIRED}");
18431        assert!(is_pool_activation_fleet_proof_error(&Error::other(wrapped)));
18432        assert!(!is_pool_activation_fleet_proof_error(&Error::ConfigNotFound));
18433    }
18434
18435    #[tokio::test]
18436    #[serial_test::serial]
18437    async fn decommission_v1_start_preflights_reject_before_metadata_writes() {
18438        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
18439        let mut v1_meta = store.pool_meta.read().await.clone();
18440        v1_meta.version = POOL_META_V1_VERSION;
18441        let v1_data = pool_meta_v1_replica_test_data(&v1_meta);
18442        for pool in &store.pools {
18443            save_config_with_opts(
18444                pool.clone(),
18445                POOL_META_NAME,
18446                v1_data.clone(),
18447                &ObjectOptions {
18448                    max_parity: true,
18449                    ..Default::default()
18450                },
18451            )
18452            .await
18453            .expect("the V1 baseline should be persisted to every pool");
18454        }
18455        let baseline = load_pool_meta_replicas(store.pools.clone(), true)
18456            .await
18457            .expect("baseline pool metadata should be readable");
18458        assert_eq!(baseline.meta.version, POOL_META_V1_VERSION);
18459        *store.pool_meta.write().await = baseline.meta.clone();
18460        let start_probe = PoolActivationStartProbe::install(PoolActivationStartKind::Decommission);
18461        let err = store
18462            .start_decommission(vec![0])
18463            .await
18464            .expect_err("the initial V1 start preflight must reject before side effects");
18465        assert!(matches!(err, Error::InvalidArgument(..)));
18466        assert!(
18467            !start_probe.preflight_side_effect_was_attempted(),
18468            "V1 rejection must precede bucket listing, healing, and metadata-bucket creation"
18469        );
18470        assert!(
18471            !start_probe.activation_was_attempted(),
18472            "V1 rejection must not enter the authoritative activation save"
18473        );
18474        let after_early_rejection = load_pool_meta_replicas(store.pools.clone(), true)
18475            .await
18476            .expect("early rejection must leave durable pool metadata readable");
18477        assert_eq!(after_early_rejection.canonical, baseline.canonical);
18478        assert!(
18479            store
18480                .pool_meta
18481                .read()
18482                .await
18483                .pools
18484                .iter()
18485                .all(|pool| pool.decommission.is_none())
18486        );
18487        assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
18488        store
18489            .ensure_pool_meta_side_effects_safe("V1 start preflight")
18490            .await
18491            .expect("a deterministic start rejection must not latch recovery");
18492        drop(start_probe);
18493
18494        let err = store
18495            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
18496            .await
18497            .expect_err("the authoritative V1 start preflight must reject before saving");
18498        assert!(matches!(err, Error::InvalidArgument(..)));
18499
18500        let after_authoritative_rejection = load_pool_meta_replicas(store.pools.clone(), true)
18501            .await
18502            .expect("authoritative rejection must leave durable pool metadata readable");
18503        assert_eq!(after_authoritative_rejection.canonical, baseline.canonical);
18504        assert!(
18505            after_authoritative_rejection
18506                .meta
18507                .pools
18508                .iter()
18509                .all(|pool| pool.decommission.is_none())
18510        );
18511        assert!(
18512            store
18513                .pool_meta
18514                .read()
18515                .await
18516                .pools
18517                .iter()
18518                .all(|pool| pool.decommission.is_none())
18519        );
18520        assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
18521        store
18522            .ensure_pool_meta_side_effects_safe("authoritative V1 start preflight")
18523            .await
18524            .expect("an authoritative capability rejection must not latch recovery");
18525    }
18526
18527    #[tokio::test]
18528    #[serial_test::serial]
18529    async fn decommission_activation_fence_loss_after_durable_save_blocks_publication() {
18530        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
18531        crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
18532        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
18533        set_decommission_capacity_info_overrides_for_test(
18534            store.id,
18535            vec![vec![
18536                DecommissionPoolCapacityInfo::for_test(0, layout, 60, 60, 0),
18537                DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100),
18538            ]],
18539        );
18540        let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
18541        let start_store = Arc::clone(&store);
18542        let mut start_task = tokio::spawn(async move {
18543            start_store
18544                .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
18545                .await
18546        });
18547
18548        tokio::select! {
18549            result = &mut start_task => panic!("activation finished before the durable-save barrier: {result:?}"),
18550            () = barrier.wait_until_paused() => {}
18551        }
18552        barrier.release_after_fence_loss();
18553        let err = tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
18554            .await
18555            .expect("decommission activation should stop after losing its fence")
18556            .expect("decommission activation task should not panic")
18557            .expect_err("post-durable-save fence loss must reject in-memory publication");
18558        assert!(err.to_string().contains("activation lock lost"));
18559
18560        let local = store.pool_meta.read().await;
18561        assert!(
18562            !pool_meta_has_active_decommission(&local),
18563            "the activation must not publish after its durable fence is lost"
18564        );
18565        drop(local);
18566        store
18567            .ensure_pool_meta_side_effects_safe("post-durable-save activation fence loss")
18568            .await
18569            .expect_err("the undisarmed transaction must latch the sticky pool metadata gate");
18570
18571        let mut persisted = PoolMeta::default();
18572        persisted
18573            .load_no_lock_from_replicas(vec![store.pools[0].clone()])
18574            .await
18575            .expect("the durable replica should remain readable for recovery");
18576        assert!(
18577            pool_meta_has_active_decommission(&persisted),
18578            "the test must lose the fence only after one durable replica commit"
18579        );
18580    }
18581
18582    #[tokio::test]
18583    #[serial_test::serial]
18584    async fn decommission_activation_adopts_canonical_commit_after_replica_failure() {
18585        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
18586        crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
18587        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
18588        set_decommission_capacity_info_overrides_for_test(
18589            store.id,
18590            vec![vec![
18591                DecommissionPoolCapacityInfo::for_test(0, layout, 60, 60, 0),
18592                DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100),
18593            ]],
18594        );
18595        let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
18596        let start_store = Arc::clone(&store);
18597        let mut start_task = tokio::spawn(async move {
18598            start_store
18599                .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
18600                .await
18601        });
18602
18603        tokio::select! {
18604            result = &mut start_task => panic!("activation finished before the durable-save barrier: {result:?}"),
18605            () = barrier.wait_until_paused() => {}
18606        }
18607        let mut replica_disks = Vec::new();
18608        for set in &store.pools[1].disk_set {
18609            let mut disks = set.disks.write().await;
18610            let saved = std::mem::take(&mut *disks);
18611            *disks = vec![None; saved.len()];
18612            replica_disks.push(saved);
18613        }
18614        barrier.release_without_fence_loss();
18615
18616        tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
18617            .await
18618            .expect("decommission activation should finish after its canonical commit")
18619            .expect("decommission activation task should not panic")
18620            .expect("a replica save failure must not report the committed activation as failed");
18621
18622        for (set, disks) in store.pools[1].disk_set.iter().zip(replica_disks) {
18623            *set.disks.write().await = disks;
18624        }
18625
18626        let local = store.pool_meta.read().await;
18627        assert!(pool_meta_has_active_decommission(&local));
18628        drop(local);
18629
18630        let mut canonical = PoolMeta::default();
18631        canonical
18632            .load_no_lock_from_replicas(vec![store.pools[0].clone()])
18633            .await
18634            .expect("the canonical committed decommission metadata should remain readable");
18635        assert!(pool_meta_has_active_decommission(&canonical));
18636
18637        let mut replica = PoolMeta::default();
18638        replica
18639            .load_no_lock_from_replicas(vec![store.pools[1].clone()])
18640            .await
18641            .expect("the stale replica metadata should remain readable after disks recover");
18642        assert!(!pool_meta_has_active_decommission(&replica));
18643
18644        let mut reloaded = PoolMeta::default();
18645        let replica_state = reloaded
18646            .load_no_lock_from_replicas(store.pools.clone())
18647            .await
18648            .expect("the canonical commit should remain reloadable after a replica recovers stale");
18649        assert!(pool_meta_has_active_decommission(&reloaded));
18650        assert!(replica_state.needs_repair);
18651        assert!(replica_state.repair_write_safe);
18652
18653        let worker_cancel = CancellationToken::new();
18654        let index_cancelers = store
18655            .reserve_decommission_routines(&worker_cancel, &[0])
18656            .await
18657            .expect("the committed activation should admit its decommission worker");
18658        drop(spawn_decommission_index_cancelers(
18659            Arc::clone(&store),
18660            worker_cancel.clone(),
18661            index_cancelers,
18662            Arc::new(Semaphore::new(decommission_entry_concurrency_limit())),
18663        ));
18664        let admitted_cancel = store.decommission_cancelers.read().await[0]
18665            .clone()
18666            .expect("the admitted decommission worker should have a cancellation token");
18667        assert!(!admitted_cancel.is_cancelled());
18668        worker_cancel.cancel();
18669        assert!(admitted_cancel.is_cancelled());
18670    }
18671
18672    #[tokio::test]
18673    #[serial_test::serial(pool_meta_version_env)]
18674    async fn decommission_capacity_reservation_serializes_two_nodes_without_overselling() {
18675        let (_temp_dirs, first_node, second_node) =
18676            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
18677        persist_v3_pool_meta_for_test(&first_node).await;
18678        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
18679        let capacity_snapshot = vec![
18680            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
18681            DecommissionPoolCapacityInfo::for_test(1, layout, 0, 30, 30),
18682            DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0),
18683        ];
18684        set_decommission_capacity_info_overrides_for_test(first_node.id, vec![capacity_snapshot.clone()]);
18685        set_decommission_capacity_info_overrides_for_test(second_node.id, vec![capacity_snapshot]);
18686
18687        let barrier =
18688            PutObjectCommitBarrier::install(RUSTFS_META_BUCKET, POOL_META_NAME, PutObjectCommitPause::BeforeQuotaRename);
18689        let first_store = Arc::clone(&first_node);
18690        let first = tokio::spawn(async move {
18691            first_store
18692                .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
18693                .await
18694        });
18695        barrier.wait_until_paused().await;
18696
18697        let second_probe = PoolActivationStartProbe::install(PoolActivationStartKind::Decommission);
18698        let second_store = Arc::clone(&second_node);
18699        let second = tokio::spawn(async move {
18700            second_store
18701                .save_current_pool_meta_for_decommission_start(&[1], Vec::new())
18702                .await
18703        });
18704        second_probe.wait_until_attempted().await;
18705        assert!(
18706            !second.is_finished(),
18707            "the competing node must wait behind the distributed activation fence"
18708        );
18709
18710        drop(barrier);
18711        first
18712            .await
18713            .expect("first-node activation should not panic")
18714            .expect("first-node reservation should fit");
18715        let mut started_v3 = PoolMeta::default();
18716        started_v3
18717            .load_no_lock_from_replicas(first_node.pools.clone())
18718            .await
18719            .expect("the started capacity reservation should reload from V3 replicas");
18720        assert_eq!(started_v3.version, POOL_META_GENERATION_VERSION);
18721        let err = second
18722            .await
18723            .expect("second-node activation should not panic")
18724            .expect_err("the second reservation must observe and reject the committed first reservation");
18725        assert!(err.to_string().contains("requires 60 bytes, but 40 bytes are available"));
18726
18727        let mut persisted = PoolMeta::default();
18728        persisted
18729            .load_no_lock_from_replicas(first_node.pools.clone())
18730            .await
18731            .expect("the winning capacity reservation should remain readable");
18732        let reservation = persisted.pools[0]
18733            .decommission
18734            .as_ref()
18735            .and_then(|info| info.capacity_reservation.as_ref())
18736            .expect("the winning operation must retain its durable reservation");
18737        assert_eq!(
18738            reservation.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
18739            "an all-v4 proof must persist the per-target lock model through V3 replicas"
18740        );
18741        assert_eq!(reservation.peak_physical_bytes, 60);
18742        assert!(persisted.pools[1].decommission.is_none());
18743    }
18744
18745    #[tokio::test]
18746    #[serial_test::serial]
18747    async fn decommission_start_without_dedicated_v4_proof_persists_the_legacy_lock_model() {
18748        let (_temp_dirs, store, _other_store) =
18749            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
18750        crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
18751        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
18752        set_decommission_capacity_info_overrides_for_test(
18753            store.id,
18754            vec![vec![
18755                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
18756                DecommissionPoolCapacityInfo::for_test(1, layout, 0, 10, 10),
18757                DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
18758            ]],
18759        );
18760        let _proof_guard = crate::services::notification_sys::without_decommission_target_fence_fleet_proof_for_test();
18761
18762        store
18763            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
18764            .await
18765            .expect("a mixed-version fleet should retain the compatible global lock model");
18766        let reservation = store.pool_meta.read().await.pools[0]
18767            .decommission
18768            .as_ref()
18769            .and_then(|info| info.capacity_reservation.as_ref())
18770            .expect("proofless start should persist a capacity reservation")
18771            .clone();
18772        assert_eq!(reservation.model_version, DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION);
18773    }
18774
18775    #[tokio::test]
18776    #[serial_test::serial]
18777    async fn decommission_cancel_replans_target_fences_after_a_concurrent_v2_start() {
18778        let (_temp_dirs, store, _other_store) =
18779            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
18780        crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
18781        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
18782        set_decommission_capacity_info_overrides_for_test(
18783            store.id,
18784            vec![vec![
18785                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
18786                DecommissionPoolCapacityInfo::for_test(1, layout, 0, 10, 10),
18787                DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
18788            ]],
18789        );
18790        let barrier = super::DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
18791        barrier.pause_cancel_before_start();
18792        let cancel_store = Arc::clone(&store);
18793        let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
18794        barrier.wait_until_cancel_before_start().await;
18795
18796        store
18797            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
18798            .await
18799            .expect("the concurrent start should persist a v2 reservation");
18800        let target_pool_index = store.pool_meta.read().await.pools[0]
18801            .decommission
18802            .as_ref()
18803            .and_then(|info| info.capacity_reservation.as_ref())
18804            .map(|reservation| {
18805                assert_eq!(reservation.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION);
18806                reservation.targets[0].pool_index
18807            })
18808            .expect("the concurrent start should publish its target cohort");
18809        let target_lock = store.pools[0]
18810            .new_ns_lock(
18811                RUSTFS_META_BUCKET,
18812                &format!("{}/{target_pool_index}", super::DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX),
18813            )
18814            .await
18815            .expect("create the cancel/start target-fence probe");
18816        let target_guard = target_lock
18817            .get_write_lock(std::time::Duration::from_secs(30))
18818            .await
18819            .expect("hold the newly started target fence");
18820
18821        barrier.release_cancel_before_start();
18822        tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel)
18823            .await
18824            .expect_err("cancel must wait for the target fence selected by the concurrent start");
18825        assert!(
18826            store.pool_meta.read().await.pools[0]
18827                .decommission
18828                .as_ref()
18829                .and_then(|info| info.capacity_reservation.as_ref())
18830                .is_some_and(DecommissionCapacityReservation::active),
18831            "cancel must not release the new reservation before fencing its target"
18832        );
18833
18834        drop(target_guard);
18835        tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
18836            .await
18837            .expect("cancel should finish after the target fence is released")
18838            .expect("cancel task should not panic")
18839            .expect("cancel should publish the terminal state");
18840        drop(barrier);
18841
18842        let pool_meta = store.pool_meta.read().await;
18843        let info = pool_meta.pools[0]
18844            .decommission
18845            .as_ref()
18846            .expect("canceled decommission metadata should remain present");
18847        assert!(info.canceled);
18848        let reservation = info
18849            .capacity_reservation
18850            .as_ref()
18851            .expect("canceled capacity accounting should remain inspectable");
18852        assert!(!reservation.active());
18853        assert_eq!(reservation.pending_target_physical_bytes, 0);
18854        assert_eq!(reservation.inflight_target_physical_bytes, 0);
18855    }
18856
18857    async fn start_target_fenced_cancel_test(
18858        store: &Arc<ECStore>,
18859        first_target_free: usize,
18860    ) -> (DecommissionCapacityTerminalFencePlan, DecommissionCanceler) {
18861        crate::services::rebalance::promote_test_pool_meta_to_v2(store).await;
18862        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
18863        set_decommission_capacity_info_overrides_for_test(
18864            store.id,
18865            vec![vec![
18866                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
18867                DecommissionPoolCapacityInfo::for_test(1, layout, first_target_free, 10, 10 - first_target_free),
18868                DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
18869            ]],
18870        );
18871        store
18872            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
18873            .await
18874            .expect("persist an active target-fenced decommission");
18875        let plan = decommission_capacity_terminal_fence_plan(&*store.pool_meta.read().await, 0)
18876            .expect("active decommission should have a valid fence plan")
18877            .expect("active decommission should retain its reservation");
18878        assert_eq!(plan.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION);
18879        let canceler = DecommissionCanceler::new(CancellationToken::new());
18880        *store.decommission_cancelers.write().await = vec![Some(canceler.clone()), None, None];
18881        (plan, canceler)
18882    }
18883
18884    #[tokio::test]
18885    #[serial_test::serial]
18886    async fn decommission_cancel_waits_for_target_contention_past_one_lock_timeout() {
18887        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("5"))], async {
18888            let (_temp_dirs, store, _other_store) =
18889                crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
18890            let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
18891            assert_eq!(plan.target_pool_indices, vec![2]);
18892            let target_lock = store.pools[0]
18893                .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
18894                .await
18895                .expect("create the active migration target gate");
18896            let target_guard = target_lock
18897                .get_write_lock(std::time::Duration::from_secs(30))
18898                .await
18899                .expect("hold the target gate across the first cancel acquisition timeout");
18900            let movement_gate = store.ctx.data_movement_operation_gate();
18901            let movement_guard = movement_gate.read().await;
18902            let cancel_store = Arc::clone(&store);
18903            let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
18904
18905            tokio::time::timeout(get_lock_acquire_timeout() + std::time::Duration::from_secs(1), &mut cancel)
18906                .await
18907                .expect_err("one target-lock timeout must not end a legitimate cancel while its retry budget remains");
18908            assert!(!canceler.is_cancelled(), "cancel must not signal its worker before durable fencing");
18909            let mut durable = PoolMeta::default();
18910            durable
18911                .load_no_lock_from_replicas(store.pools.clone())
18912                .await
18913                .expect("the active reservation must remain readable while cancel waits");
18914            assert_eq!(
18915                decommission_capacity_terminal_fence_plan(&durable, 0).expect("read the durable active plan"),
18916                Some(plan),
18917                "a failed target acquisition must not release or replace the durable reservation"
18918            );
18919            assert!(!durable.pools[0].decommission.as_ref().expect("active decommission").canceled);
18920
18921            drop(target_guard);
18922            tokio::time::timeout(std::time::Duration::from_secs(30), canceler.token().cancelled())
18923                .await
18924                .expect("cancel should persist and signal its worker after target contention is released");
18925            durable
18926                .load_no_lock_from_replicas(store.pools.clone())
18927                .await
18928                .expect("reload the committed cancellation from native replicas");
18929            let info = durable.pools[0]
18930                .decommission
18931                .as_ref()
18932                .expect("canceled state must remain durable");
18933            assert!(info.canceled);
18934            assert!(!info.complete && !info.failed);
18935            let reservation = info
18936                .capacity_reservation
18937                .as_ref()
18938                .expect("terminal capacity accounting must remain inspectable");
18939            assert!(!reservation.active());
18940            assert_eq!(reservation.pending_target_physical_bytes, 0);
18941            assert_eq!(reservation.inflight_target_physical_bytes, 0);
18942            tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel)
18943                .await
18944                .expect_err("the runtime-fenced cancel must wait for in-flight movement after signaling");
18945            drop(movement_guard);
18946            tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
18947                .await
18948                .expect("cancel should return after in-flight movement quiesces")
18949                .expect("cancel task should not panic")
18950                .expect("the same cancel request should finish its durable terminal transition");
18951        })
18952        .await;
18953    }
18954
18955    #[tokio::test]
18956    #[serial_test::serial]
18957    async fn decommission_cancel_target_contention_exhausts_bounded_attempts_without_committing() {
18958        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
18959            let (_temp_dirs, store, _other_store) =
18960                crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
18961            let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
18962            let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
18963            let target_lock = store.pools[0]
18964                .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
18965                .await
18966                .expect("create the persistently contended target gate");
18967            let target_guard = target_lock
18968                .get_write_lock(std::time::Duration::from_secs(30))
18969                .await
18970                .expect("hold the target gate for every cancel attempt");
18971
18972            let err = tokio::time::timeout(std::time::Duration::from_secs(30), store.decommission_cancel(0))
18973                .await
18974                .expect("target contention must not retry indefinitely")
18975                .expect_err("exhausting the acquisition budget must not report cancellation success");
18976            match err {
18977                Error::Lock(rustfs_lock::LockError::Timeout { resource, timeout }) => {
18978                    assert_eq!(resource, ".rustfs.sys/decommission/capacity-target/2@latest");
18979                    assert_eq!(timeout, std::time::Duration::from_secs(1));
18980                }
18981                other => panic!("expected the final typed target-lock timeout, got {other:?}"),
18982            }
18983            assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 3);
18984            assert!(!canceler.is_cancelled());
18985            assert!(canceler.is_active());
18986            let mut durable = PoolMeta::default();
18987            durable
18988                .load_no_lock_from_replicas(store.pools.clone())
18989                .await
18990                .expect("reload the reservation after the canceled request exhausted its budget");
18991            assert_eq!(
18992                decommission_capacity_terminal_fence_plan(&durable, 0).expect("the active plan should remain valid"),
18993                Some(plan)
18994            );
18995            assert!(!durable.pools[0].decommission.as_ref().expect("active decommission").canceled);
18996            drop(target_guard);
18997        })
18998        .await;
18999    }
19000
19001    #[tokio::test]
19002    #[serial_test::serial]
19003    async fn decommission_cancel_target_retry_releases_partial_cohort_and_rejects_remote_replacement() {
19004        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
19005            let (_temp_dirs, store, other_store) =
19006                crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19007            let (plan, _old_canceler) = start_target_fenced_cancel_test(&store, 8).await;
19008            assert_eq!(plan.target_pool_indices, vec![1, 2]);
19009            let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19010            barrier.pause_cancel_target_timeout();
19011            let target_lock = store.pools[0]
19012                .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
19013                .await
19014                .expect("create the second target gate");
19015            let target_guard = target_lock
19016                .get_write_lock(std::time::Duration::from_secs(30))
19017                .await
19018                .expect("block cancellation after it acquires the first target");
19019            let cancel_store = Arc::clone(&store);
19020            let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
19021            barrier.wait_until_cancel_target_timeout().await;
19022
19023            let first_target_lock = store.pools[0]
19024                .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/1"))
19025                .await
19026                .expect("create the partial-cohort release probe");
19027            let first_target_guard = first_target_lock
19028                .get_write_lock(std::time::Duration::from_secs(1))
19029                .await
19030                .expect("cancel must release its first target before retrying the blocked second target");
19031            drop(first_target_guard);
19032            drop(target_guard);
19033
19034            other_store
19035                .reload_pool_meta()
19036                .await
19037                .expect("load the active generation on the remote node");
19038            other_store
19039                .decommission_cancel(0)
19040                .await
19041                .expect("the remote node should cancel the old generation");
19042            other_store
19043                .clear_decommission(0)
19044                .await
19045                .expect("the remote node should clear the old generation");
19046            let (replacement_plan, replacement_canceler) = start_target_fenced_cancel_test(&other_store, 8).await;
19047            assert_ne!(replacement_plan.operation_id, plan.operation_id);
19048
19049            barrier.release_cancel_target_timeout();
19050            let err = tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
19051                .await
19052                .expect("the stale request must finish without retrying a replacement operation")
19053                .expect("the stale cancel task should not panic")
19054                .expect_err("the original request must not cancel a remotely replaced generation");
19055            assert!(err.to_string().contains("changed while retrying terminal fences"));
19056            assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
19057            assert!(!replacement_canceler.is_cancelled());
19058            let mut durable = PoolMeta::default();
19059            durable
19060                .load_no_lock_from_replicas(store.pools.clone())
19061                .await
19062                .expect("reload the replacement after the stale cancel returns");
19063            assert_eq!(
19064                decommission_capacity_terminal_fence_plan(&durable, 0).expect("the replacement must retain its valid plan"),
19065                Some(replacement_plan)
19066            );
19067            assert!(
19068                !durable.pools[0]
19069                    .decommission
19070                    .as_ref()
19071                    .expect("replacement decommission")
19072                    .canceled
19073            );
19074        })
19075        .await;
19076    }
19077
19078    #[tokio::test]
19079    #[serial_test::serial]
19080    async fn decommission_cancel_target_retry_survives_caller_abort_and_settles_inflight_mutation() {
19081        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
19082            let (_temp_dirs, store, _other_store) =
19083                crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19084            let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
19085            let owner = DecommissionCapacityOwner {
19086                source_pool_index: 0,
19087                operation_id: plan.operation_id,
19088                generation: plan.generation,
19089                owner_nonce: plan.owner_nonce,
19090                mutation_id: Some(uuid::Uuid::new_v4()),
19091            };
19092            let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
19093            let (release_tx, release_rx) = tokio::sync::oneshot::channel();
19094            let mutation_store = Arc::clone(&store);
19095            let mutation = tokio::spawn(async move {
19096                mutation_store
19097                    .run_decommission_capacity_admitted_mutation(2, Some(owner), Some(1), || async {
19098                        entered_tx
19099                            .send(())
19100                            .expect("the test should observe the admitted target mutation");
19101                        release_rx.await.expect("the test should release the in-flight mutation");
19102                        Ok(())
19103                    })
19104                    .await
19105            });
19106            tokio::time::timeout(std::time::Duration::from_secs(30), entered_rx)
19107                .await
19108                .expect("target mutation should reach its controlled I/O phase")
19109                .expect("target admission should succeed before cancel starts");
19110            let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19111            let cancel_store = Arc::clone(&store);
19112            let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
19113            barrier.wait_until_cancel_target_timeout().await;
19114            assert!(!canceler.is_cancelled(), "in-flight capacity settlement must precede the cancel signal");
19115            cancel.abort();
19116            assert!(cancel.await.expect_err("the RPC waiter should be aborted").is_cancelled());
19117
19118            release_tx
19119                .send(())
19120                .expect("the mutation must still be alive after the caller disconnects");
19121            tokio::time::timeout(std::time::Duration::from_secs(30), mutation)
19122                .await
19123                .expect("the in-flight mutation must be able to settle without a metadata lock cycle")
19124                .expect("the mutation task should not panic")
19125                .expect("the admitted mutation must settle before the target gate is released");
19126            tokio::time::timeout(std::time::Duration::from_secs(30), async {
19127                loop {
19128                    if store.pool_meta.read().await.pools[0]
19129                        .decommission
19130                        .as_ref()
19131                        .is_some_and(|info| info.canceled)
19132                    {
19133                        break;
19134                    }
19135                    tokio::time::sleep(std::time::Duration::from_millis(20)).await;
19136                }
19137            })
19138            .await
19139            .expect("the detached cancel transaction must finish after target contention clears");
19140            assert!(canceler.is_cancelled());
19141            let mut durable = PoolMeta::default();
19142            durable
19143                .load_no_lock_from_replicas(store.pools.clone())
19144                .await
19145                .expect("reload cancellation after the RPC waiter was dropped");
19146            let info = durable.pools[0].decommission.as_ref().expect("durable canceled decommission");
19147            assert!(info.canceled && !info.failed && !info.complete);
19148            let reservation = info
19149                .capacity_reservation
19150                .as_ref()
19151                .expect("inspect settled capacity accounting");
19152            assert!(!reservation.active());
19153            assert_eq!(reservation.pending_target_physical_bytes, 0);
19154            assert_eq!(reservation.inflight_target_physical_bytes, 0);
19155        })
19156        .await;
19157    }
19158
19159    #[tokio::test]
19160    #[serial_test::serial]
19161    async fn decommission_cancel_target_retry_does_not_replay_a_failed_durable_save() {
19162        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
19163            let (_temp_dirs, store, _other_store) =
19164                crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19165            let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
19166            let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19167            barrier.pause_cancel_target_timeout();
19168            let target_lock = store.pools[0]
19169                .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
19170                .await
19171                .expect("create the target gate preceding the failed save");
19172            let target_guard = target_lock
19173                .get_write_lock(std::time::Duration::from_secs(30))
19174                .await
19175                .expect("force one target acquisition retry before persistence");
19176            let save_calls = Arc::new(AtomicUsize::new(0));
19177            let calls = Arc::clone(&save_calls);
19178            let cancel_store = Arc::clone(&store);
19179            let owner = canceler.clone();
19180            let cancel = tokio::spawn(async move {
19181                cancel_store
19182                    .decommission_cancel_transaction(0, Some(owner), true, move |_, _| async move {
19183                        calls.fetch_add(1, Ordering::AcqRel);
19184                        Err(Error::Timeout)
19185                    })
19186                    .await
19187            });
19188            barrier.wait_until_cancel_target_timeout().await;
19189            assert_eq!(save_calls.load(Ordering::Acquire), 0, "persistence must wait for every target fence");
19190            drop(target_guard);
19191            barrier.release_cancel_target_timeout();
19192            let err = tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
19193                .await
19194                .expect("a persistence failure must end the cancellation transaction")
19195                .expect("the failed-save task should not panic")
19196                .expect_err("the injected durable-save failure must reach the caller");
19197            assert!(matches!(err, Error::Timeout));
19198            assert_eq!(save_calls.load(Ordering::Acquire), 1);
19199            assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
19200            assert!(!canceler.is_cancelled());
19201            assert!(canceler.is_active());
19202            store
19203                .ensure_pool_meta_side_effects_safe("verify ambiguous cancellation save blocks further writes")
19204                .await
19205                .expect_err("a failed durable save must retain the existing recovery gate");
19206            let mut durable = PoolMeta::default();
19207            durable
19208                .load_no_lock_from_replicas(store.pools.clone())
19209                .await
19210                .expect("the original reservation should remain readable after the injected save failure");
19211            assert_eq!(
19212                decommission_capacity_terminal_fence_plan(&durable, 0).expect("the old reservation should remain valid"),
19213                Some(plan)
19214            );
19215        })
19216        .await;
19217    }
19218
19219    #[tokio::test]
19220    #[serial_test::serial]
19221    async fn decommission_cancel_target_retry_converges_after_successive_contenders() {
19222        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
19223            let (_temp_dirs, store, _other_store) =
19224                crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19225            let (_plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
19226            let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19227            barrier.pause_cancel_target_timeout();
19228            let target_lock = store.pools[0]
19229                .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
19230                .await
19231                .expect("create the repeatedly contended target gate");
19232            let first_guard = target_lock
19233                .get_write_lock(std::time::Duration::from_secs(30))
19234                .await
19235                .expect("the first contender should own the target gate");
19236            let cancel_store = Arc::clone(&store);
19237            let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
19238            barrier.wait_until_cancel_target_timeout().await;
19239            assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
19240            drop(first_guard);
19241            let second_guard = target_lock
19242                .get_write_lock(std::time::Duration::from_secs(30))
19243                .await
19244                .expect("a second contender should be able to acquire between cancel attempts");
19245            barrier.release_cancel_target_timeout();
19246            barrier.pause_cancel_target_timeout();
19247            barrier.wait_until_cancel_target_timeout().await;
19248            assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 2);
19249            assert!(!canceler.is_cancelled());
19250            drop(second_guard);
19251            barrier.release_cancel_target_timeout();
19252
19253            tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
19254                .await
19255                .expect("cancel should converge after the repeated contention ends")
19256                .expect("the cancel task should not panic")
19257                .expect("the third target acquisition should permit a durable cancellation");
19258            assert!(canceler.is_cancelled());
19259            let mut durable = PoolMeta::default();
19260            durable
19261                .load_no_lock_from_replicas(store.pools.clone())
19262                .await
19263                .expect("the successful retried cancellation must survive a native metadata reload");
19264            assert!(
19265                durable.pools[0]
19266                    .decommission
19267                    .as_ref()
19268                    .expect("canceled decommission")
19269                    .canceled
19270            );
19271            assert!(
19272                decommission_capacity_terminal_fence_plan(&durable, 0)
19273                    .expect("valid terminal metadata")
19274                    .is_none()
19275            );
19276        })
19277        .await;
19278    }
19279
19280    #[tokio::test]
19281    #[serial_test::serial]
19282    async fn stale_node_cancel_cannot_replace_a_new_durable_v2_operation() {
19283        let (_temp_dirs, first_node, stale_node) =
19284            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19285        crate::services::rebalance::promote_test_pool_meta_to_v2(&first_node).await;
19286        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19287        set_decommission_capacity_info_overrides_for_test(
19288            first_node.id,
19289            vec![vec![
19290                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
19291                DecommissionPoolCapacityInfo::for_test(1, layout, 0, 10, 10),
19292                DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
19293            ]],
19294        );
19295
19296        first_node
19297            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19298            .await
19299            .expect("start the operation retained by the stale node");
19300        let stale_operation = first_node.pool_meta.read().await.clone();
19301        let stale_plan = decommission_capacity_terminal_fence_plan(&stale_operation, 0)
19302            .expect("the stale operation should have a valid terminal fence plan")
19303            .expect("the stale operation should have an active reservation");
19304
19305        first_node
19306            .decommission_cancel(0)
19307            .await
19308            .expect("cancel the first durable operation");
19309        first_node
19310            .clear_decommission(0)
19311            .await
19312            .expect("clear the first terminal operation");
19313        first_node
19314            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19315            .await
19316            .expect("start the replacement durable operation");
19317        let replacement_plan = {
19318            let pool_meta = first_node.pool_meta.read().await;
19319            decommission_capacity_terminal_fence_plan(&pool_meta, 0)
19320                .expect("the replacement operation should have a valid terminal fence plan")
19321                .expect("the replacement operation should have an active reservation")
19322        };
19323        assert_ne!(replacement_plan.operation_id, stale_plan.operation_id);
19324
19325        *stale_node.pool_meta.write().await = stale_operation;
19326        let err = stale_node
19327            .decommission_cancel(0)
19328            .await
19329            .expect_err("a stale local operation must not be merged over the durable replacement");
19330        assert!(err.to_string().contains("differs from the durable terminal fence plan"));
19331
19332        let mut durable = PoolMeta::default();
19333        durable
19334            .load_no_lock_from_replicas(first_node.pools.clone())
19335            .await
19336            .expect("the replacement operation should remain readable from durable replicas");
19337        assert_eq!(
19338            decommission_capacity_terminal_fence_plan(&durable, 0)
19339                .expect("the durable replacement should retain a valid terminal fence plan")
19340                .expect("the durable replacement reservation should remain active"),
19341            replacement_plan
19342        );
19343        let replacement = durable.pools[0]
19344            .decommission
19345            .as_ref()
19346            .expect("the durable replacement metadata should remain present");
19347        assert!(!replacement.canceled);
19348        assert!(
19349            replacement
19350                .capacity_reservation
19351                .as_ref()
19352                .is_some_and(DecommissionCapacityReservation::active)
19353        );
19354    }
19355
19356    #[tokio::test]
19357    #[serial_test::serial]
19358    async fn decommission_capacity_publication_allows_immediate_bucket_completion() {
19359        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
19360        crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
19361        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19362        let capacity = vec![
19363            DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80),
19364            DecommissionPoolCapacityInfo::for_test(1, layout, 200, 200, 0),
19365        ];
19366        set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity.clone(), capacity.clone(), capacity]);
19367        let bucket = DecomBucketInfo {
19368            name: "capacity-publication-bucket".to_string(),
19369            prefix: String::new(),
19370        };
19371        store
19372            .save_current_pool_meta_for_decommission_start(&[0], vec![bucket.clone()])
19373            .await
19374            .expect("activate a reservation and queue the regression bucket");
19375        let (owner, initial_update, generation) = {
19376            let mut meta = store.pool_meta.write().await;
19377            meta.count_item(0, 17, false);
19378            let reservation = meta.pools[0]
19379                .decommission
19380                .as_ref()
19381                .and_then(|info| info.capacity_reservation.as_ref())
19382                .expect("the source must own an active reservation");
19383            (
19384                DecommissionCapacityOwner {
19385                    source_pool_index: 0,
19386                    operation_id: reservation.operation_id,
19387                    generation: reservation.generation,
19388                    owner_nonce: reservation.owner_nonce,
19389                    mutation_id: Some(uuid::Uuid::new_v4()),
19390                },
19391                meta.pools[0].last_update,
19392                meta.pools[0]
19393                    .decommission
19394                    .as_ref()
19395                    .and_then(|info| info.start_time)
19396                    .expect("active worker generation"),
19397            )
19398        };
19399        store
19400            .run_decommission_capacity_admitted_mutation(1, Some(owner), Some(17), || async {
19401                let meta = store.pool_meta.read().await;
19402                assert!(
19403                    meta.pools[0].last_update > initial_update,
19404                    "durable capacity intent must publish its revision before target I/O"
19405                );
19406                assert_eq!(
19407                    meta.pools[0]
19408                        .decommission
19409                        .as_ref()
19410                        .expect("active source")
19411                        .items_decommissioned,
19412                    1
19413                );
19414                Ok(())
19415            })
19416            .await
19417            .expect("capacity intent and consumption must commit without a progress checkpoint");
19418        let persisted = load_pool_meta_replicas(store.pools.clone(), true)
19419            .await
19420            .expect("reload capacity consumption before bucket completion");
19421        {
19422            let meta = store.pool_meta.read().await;
19423            assert_eq!(meta.pools[0].last_update, persisted.meta.pools[0].last_update);
19424            let info = meta.pools[0].decommission.as_ref().expect("active source");
19425            assert_eq!(info.items_decommissioned, 1, "capacity publication must retain uncheckpointed progress");
19426            assert_eq!(
19427                info.progress_save_item_baseline, 0,
19428                "capacity-only saves do not checkpoint object counters"
19429            );
19430        }
19431        assert!(
19432            store
19433                .mark_decommission_bucket_done_and_save(0, generation, &bucket)
19434                .await
19435                .expect("bucket completion immediately after capacity persistence must not reject its own revision")
19436        );
19437        let persisted = load_pool_meta_replicas(store.pools.clone(), true)
19438            .await
19439            .expect("bucket completion must survive a metadata reload");
19440        let info = persisted.meta.pools[0]
19441            .decommission
19442            .as_ref()
19443            .expect("durable source progress");
19444        assert!(info.queued_buckets.is_empty());
19445        assert_eq!(info.decommissioned_buckets, vec![bucket.to_string()]);
19446        assert_eq!((info.items_decommissioned, info.bytes_done), (1, 17));
19447        let reservation = info.capacity_reservation.as_ref().expect("durable capacity consumption");
19448        assert_eq!(reservation.pending_target_physical_bytes, 0);
19449        assert_eq!(reservation.consumed_target_physical_bytes, 17);
19450    }
19451
19452    #[tokio::test]
19453    #[serial_test::serial]
19454    async fn decommission_bucket_completion_publishes_only_committed_progress() {
19455        for fault_stage in [
19456            "bucket_completion_before_save",
19457            "bucket_completion_after_save",
19458            "count_during_save",
19459        ] {
19460            let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
19461            let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19462            set_decommission_capacity_info_overrides_for_test(
19463                store.id,
19464                vec![vec![
19465                    DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80),
19466                    DecommissionPoolCapacityInfo::for_test(1, layout, 200, 200, 0),
19467                ]],
19468            );
19469            let bucket = DecomBucketInfo {
19470                name: format!("bucket-completion-{fault_stage}"),
19471                prefix: String::new(),
19472            };
19473            store
19474                .save_current_pool_meta_for_decommission_start(&[0], vec![bucket.clone()])
19475                .await
19476                .expect("activate the bucket completion fixture");
19477            let generation = {
19478                let mut meta = store.pool_meta.write().await;
19479                meta.count_item(0, 17, false);
19480                meta.pools[0]
19481                    .decommission
19482                    .as_ref()
19483                    .and_then(|info| info.start_time)
19484                    .expect("active generation")
19485            };
19486            let hook_store = Arc::clone(&store);
19487            let observed = Arc::new(AtomicBool::new(false));
19488            let hook_observed = Arc::clone(&observed);
19489            let expected_bucket = bucket.name.clone();
19490            let fault = DecommissionTestFaultGuard::install(Arc::new(move |stage, name, _, _, success| {
19491                if name != expected_bucket || !success {
19492                    return false;
19493                }
19494                if stage == fault_stage {
19495                    hook_observed.store(true, Ordering::SeqCst);
19496                    return true;
19497                }
19498                if fault_stage == "count_during_save" && stage == "bucket_completion_after_save" {
19499                    hook_observed.store(true, Ordering::SeqCst);
19500                    hook_store
19501                        .pool_meta
19502                        .try_write()
19503                        .expect("publication must not hold the local metadata lock over I/O")
19504                        .count_item(0, 23, false);
19505                }
19506                false
19507            }));
19508            let result = store.mark_decommission_bucket_done_and_save(0, generation, &bucket).await;
19509            assert!(
19510                observed.load(Ordering::SeqCst),
19511                "{fault_stage}: the production save boundary was not reached"
19512            );
19513            drop(fault);
19514            let persisted = load_pool_meta_replicas(store.pools.clone(), true)
19515                .await
19516                .expect("durable metadata remains readable");
19517            let durable = persisted.meta.pools[0].decommission.as_ref().expect("durable progress");
19518            let local = store.pool_meta.read().await;
19519            let live = local.pools[0].decommission.as_ref().expect("live progress");
19520            if fault_stage == "count_during_save" {
19521                assert!(result.expect("successful bucket completion"));
19522                assert_eq!((live.items_decommissioned, live.bytes_done, live.progress_save_item_baseline), (2, 40, 1));
19523                assert_eq!((durable.items_decommissioned, durable.bytes_done), (1, 17));
19524                assert!(live.queued_buckets.is_empty());
19525                assert_eq!(live.decommissioned_buckets, vec![bucket.to_string()]);
19526            } else {
19527                let err = result.expect_err("injected failure must reach the caller");
19528                assert!(err.to_string().contains(fault_stage));
19529                assert_eq!(live.queued_buckets, vec![bucket.to_string()]);
19530                assert!(live.decommissioned_buckets.is_empty());
19531                assert_eq!(live.progress_save_item_baseline, 0);
19532                if fault_stage == "bucket_completion_before_save" {
19533                    assert_eq!(durable.queued_buckets, vec![bucket.to_string()]);
19534                    assert!(durable.decommissioned_buckets.is_empty());
19535                } else {
19536                    assert!(durable.queued_buckets.is_empty());
19537                    assert_eq!(durable.decommissioned_buckets, vec![bucket.to_string()]);
19538                    assert_eq!((durable.items_decommissioned, durable.bytes_done), (1, 17));
19539                }
19540            }
19541            drop(local);
19542            if fault_stage == "bucket_completion_after_save" {
19543                store
19544                    .ensure_pool_meta_side_effects_safe("ambiguous bucket completion")
19545                    .await
19546                    .expect_err("commit without publication must require recovery before another write");
19547            } else {
19548                store
19549                    .ensure_pool_meta_side_effects_safe("bucket completion")
19550                    .await
19551                    .expect("no ambiguous commit");
19552            }
19553        }
19554    }
19555
19556    #[tokio::test]
19557    #[serial_test::serial]
19558    async fn legacy_capacity_model_holds_the_global_pool_meta_fence_through_target_io() {
19559        let (_temp_dirs, store, _other_store) =
19560            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19561        crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
19562        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19563        set_decommission_capacity_info_overrides_for_test(
19564            store.id,
19565            vec![vec![
19566                DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80),
19567                DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
19568                DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0),
19569            ]],
19570        );
19571        let _proof_guard = crate::services::notification_sys::without_decommission_target_fence_fleet_proof_for_test();
19572        store
19573            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19574            .await
19575            .expect("activate a legacy reservation spanning two targets");
19576        let base_owner = {
19577            let pool_meta = store.pool_meta.read().await;
19578            let reservation = pool_meta.pools[0]
19579                .decommission
19580                .as_ref()
19581                .and_then(|info| info.capacity_reservation.as_ref())
19582                .expect("legacy capacity reservation should exist");
19583            assert_eq!(reservation.model_version, DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION);
19584            assert_eq!(reservation.targets.iter().map(|target| target.pool_index).collect::<Vec<_>>(), vec![1, 2]);
19585            DecommissionCapacityOwner {
19586                source_pool_index: 0,
19587                operation_id: reservation.operation_id,
19588                generation: reservation.generation,
19589                owner_nonce: reservation.owner_nonce,
19590                mutation_id: None,
19591            }
19592        };
19593
19594        let (first_entered_tx, first_entered_rx) = tokio::sync::oneshot::channel();
19595        let (first_release_tx, first_release_rx) = tokio::sync::oneshot::channel();
19596        let first_store = Arc::clone(&store);
19597        let first_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
19598        let first = tokio::spawn(async move {
19599            first_store
19600                .run_decommission_capacity_admitted_mutation_with_capacity_lease(1, Some(first_owner), Some(1), |_| async {
19601                    first_entered_tx.send(()).expect("first legacy mutation should be observed");
19602                    first_release_rx.await.expect("first legacy mutation should be released");
19603                    Ok(())
19604                })
19605                .await
19606        });
19607        tokio::time::timeout(std::time::Duration::from_secs(30), first_entered_rx)
19608            .await
19609            .expect("first legacy mutation should enter")
19610            .expect("first legacy mutation should report entry");
19611        assert!(
19612            store.pool_meta_save_gate.try_lock().is_err(),
19613            "legacy target I/O must retain the local pool metadata save gate"
19614        );
19615        let pool_meta_lock = store.pools[0]
19616            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
19617            .await
19618            .expect("create the legacy pool metadata lock probe");
19619        assert!(
19620            pool_meta_lock
19621                .get_write_lock_quiet(std::time::Duration::from_millis(300))
19622                .await
19623                .is_err(),
19624            "legacy target I/O must retain the distributed pool metadata write fence"
19625        );
19626
19627        let (second_entered_tx, mut second_entered_rx) = tokio::sync::oneshot::channel();
19628        let (second_release_tx, second_release_rx) = tokio::sync::oneshot::channel();
19629        let second_store = Arc::clone(&store);
19630        let second_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
19631        let second = tokio::spawn(async move {
19632            second_store
19633                .run_decommission_capacity_admitted_mutation_with_capacity_lease(2, Some(second_owner), Some(1), |_| async {
19634                    second_entered_tx.send(()).expect("second legacy mutation should be observed");
19635                    second_release_rx.await.expect("second legacy mutation should be released");
19636                    Ok(())
19637                })
19638                .await
19639        });
19640        tokio::time::timeout(std::time::Duration::from_millis(500), &mut second_entered_rx)
19641            .await
19642            .expect_err("a different target must still wait behind the legacy global fence");
19643
19644        first_release_tx.send(()).expect("release first legacy mutation");
19645        first
19646            .await
19647            .expect("first legacy mutation task should not panic")
19648            .expect("first legacy mutation should finalize");
19649        tokio::time::timeout(std::time::Duration::from_secs(30), &mut second_entered_rx)
19650            .await
19651            .expect("second legacy mutation should enter after the first finalizes")
19652            .expect("second legacy mutation should report entry");
19653        assert!(
19654            pool_meta_lock
19655                .get_write_lock_quiet(std::time::Duration::from_millis(300))
19656                .await
19657                .is_err(),
19658            "the second legacy target I/O must also retain the global fence"
19659        );
19660        second_release_tx.send(()).expect("release second legacy mutation");
19661        second
19662            .await
19663            .expect("second legacy mutation task should not panic")
19664            .expect("second legacy mutation should finalize");
19665
19666        let pool_meta_guard = pool_meta_lock
19667            .get_write_lock(std::time::Duration::from_secs(30))
19668            .await
19669            .expect("the global pool metadata fence should release after both target tails");
19670        drop(pool_meta_guard);
19671        let pool_meta = store.pool_meta.read().await;
19672        let reservation = pool_meta.pools[0]
19673            .decommission
19674            .as_ref()
19675            .and_then(|info| info.capacity_reservation.as_ref())
19676            .expect("legacy reservation should remain active");
19677        assert_eq!(reservation.pending_target_physical_bytes, 0);
19678        assert_eq!(reservation.consumed_target_physical_bytes, 2);
19679    }
19680
19681    #[tokio::test]
19682    #[serial_test::serial]
19683    async fn decommission_capacity_mutations_overlap_across_targets_but_serialize_per_target() {
19684        let (_temp_dirs, store, _other_store) =
19685            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19686        crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
19687        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19688        let capacity_snapshot = vec![
19689            DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80),
19690            DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
19691            DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0),
19692        ];
19693        set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot]);
19694        store
19695            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19696            .await
19697            .expect("activate a reservation spanning two targets");
19698
19699        let base_owner = {
19700            let pool_meta = store.pool_meta.read().await;
19701            let reservation = pool_meta.pools[0]
19702                .decommission
19703                .as_ref()
19704                .and_then(|info| info.capacity_reservation.as_ref())
19705                .expect("active capacity reservation should exist");
19706            assert_eq!(
19707                reservation.targets.iter().map(|target| target.pool_index).collect::<Vec<_>>(),
19708                vec![1, 2],
19709                "the fixture must reserve both target pools"
19710            );
19711            DecommissionCapacityOwner {
19712                source_pool_index: 0,
19713                operation_id: reservation.operation_id,
19714                generation: reservation.generation,
19715                owner_nonce: reservation.owner_nonce,
19716                mutation_id: None,
19717            }
19718        };
19719
19720        let (first_entered_tx, first_entered_rx) = tokio::sync::oneshot::channel();
19721        let (first_release_tx, first_release_rx) = tokio::sync::oneshot::channel();
19722        let first_store = Arc::clone(&store);
19723        let first_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
19724        let first = tokio::spawn(async move {
19725            first_store
19726                .run_decommission_capacity_admitted_mutation_with_capacity_lease(1, Some(first_owner), Some(1), |_| async {
19727                    first_entered_tx.send(()).expect("first target mutation should be observed");
19728                    first_release_rx.await.expect("first target mutation should be released");
19729                    Ok(())
19730                })
19731                .await
19732        });
19733        tokio::time::timeout(std::time::Duration::from_secs(30), first_entered_rx)
19734            .await
19735            .expect("first target mutation should enter without hanging")
19736            .expect("first target mutation should report entry");
19737
19738        let (second_entered_tx, second_entered_rx) = tokio::sync::oneshot::channel();
19739        let (second_release_tx, second_release_rx) = tokio::sync::oneshot::channel();
19740        let second_store = Arc::clone(&store);
19741        let second_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
19742        let second = tokio::spawn(async move {
19743            second_store
19744                .run_decommission_capacity_admitted_mutation_with_capacity_lease(2, Some(second_owner), Some(1), |_| async {
19745                    second_entered_tx.send(()).expect("second target mutation should be observed");
19746                    second_release_rx.await.expect("second target mutation should be released");
19747                    Ok(())
19748                })
19749                .await
19750        });
19751        tokio::time::timeout(std::time::Duration::from_secs(30), second_entered_rx)
19752            .await
19753            .expect("a different target mutation should overlap instead of waiting for the first")
19754            .expect("second target mutation should report entry");
19755
19756        let save_guard = store
19757            .pool_meta_save_gate
19758            .try_lock()
19759            .expect("target I/O must not retain the local pool metadata save gate");
19760        drop(save_guard);
19761        let pool_meta_lock = store.pools[0]
19762            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
19763            .await
19764            .expect("create a pool metadata lock probe");
19765        let pool_meta_guard = pool_meta_lock
19766            .get_write_lock_quiet(std::time::Duration::from_secs(1))
19767            .await
19768            .expect("target I/O must not retain the distributed pool metadata write lock");
19769        drop(pool_meta_guard);
19770
19771        let blocked_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
19772        let blocked = store
19773            .run_decommission_capacity_admitted_mutation(1, Some(blocked_owner), Some(1), || async { Ok(()) })
19774            .await
19775            .expect_err("a second mutation on the same target must wait behind its target gate");
19776        assert!(is_decommission_capacity_blocked_error(&blocked));
19777        assert!(
19778            is_decommission_capacity_target_gate_busy(&blocked),
19779            "same-target serialization must remain distinguishable from durable intent conflicts"
19780        );
19781        assert!(
19782            decommission_capacity_retry_kind(&blocked, 0).is_none(),
19783            "target contention must be retried inside the current version instead of replaying the entry"
19784        );
19785
19786        let cancel_store = Arc::clone(&store);
19787        let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
19788        tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel)
19789            .await
19790            .expect_err("terminal reservation release must wait for prepared target mutations");
19791        assert!(
19792            store.pool_meta.read().await.pools[0]
19793                .decommission
19794                .as_ref()
19795                .and_then(|info| info.capacity_reservation.as_ref())
19796                .is_some_and(DecommissionCapacityReservation::active),
19797            "cancellation must not release the reservation while target mutations are active"
19798        );
19799
19800        first_release_tx.send(()).expect("release first target mutation");
19801        second_release_tx.send(()).expect("release second target mutation");
19802        first
19803            .await
19804            .expect("first target mutation task should not panic")
19805            .expect("first target mutation should finalize");
19806        second
19807            .await
19808            .expect("second target mutation task should not panic")
19809            .expect("second target mutation should finalize");
19810        cancel
19811            .await
19812            .expect("decommission cancellation task should not panic")
19813            .expect("decommission cancellation should finish after target mutations");
19814
19815        let pool_meta = store.pool_meta.read().await;
19816        let reservation = pool_meta.pools[0]
19817            .decommission
19818            .as_ref()
19819            .and_then(|info| info.capacity_reservation.as_ref())
19820            .expect("released capacity reservation should remain available for accounting");
19821        assert!(!reservation.active());
19822        assert_eq!(reservation.release_reason.as_deref(), Some(DECOMMISSION_CAPACITY_RELEASE_CANCELED));
19823        assert_eq!(reservation.pending_target_physical_bytes, 0);
19824        assert_eq!(reservation.consumed_target_physical_bytes, 2);
19825        assert_eq!(
19826            reservation
19827                .targets
19828                .iter()
19829                .map(|target| (target.pool_index, target.consumed_physical_bytes))
19830                .collect::<Vec<_>>(),
19831            vec![(1, 1), (2, 1)]
19832        );
19833    }
19834
19835    #[tokio::test]
19836    #[serial_test::serial]
19837    async fn ordinary_write_capacity_fence_serializes_with_decommission_activation() {
19838        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
19839        let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
19840        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
19841        let write_store = Arc::clone(&store);
19842        let ordinary_write = tokio::spawn(async move {
19843            write_store
19844                .run_decommission_capacity_admitted_mutation(1, None, None, || async move {
19845                    entered_tx.send(()).expect("ordinary write admission should be observed");
19846                    release_rx.await.expect("ordinary write should be released");
19847                    Ok(())
19848                })
19849                .await
19850        });
19851        entered_rx
19852            .await
19853            .expect("ordinary write should hold the shared capacity fence");
19854
19855        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19856        set_decommission_capacity_info_overrides_for_test(
19857            store.id,
19858            vec![vec![
19859                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
19860                DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1),
19861            ]],
19862        );
19863        let activation_store = Arc::clone(&store);
19864        let activation = tokio::spawn(async move {
19865            activation_store
19866                .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19867                .await
19868        });
19869        tokio::task::yield_now().await;
19870        assert!(
19871            !activation.is_finished(),
19872            "activation must wait until the already-admitted ordinary write leaves the distributed capacity boundary"
19873        );
19874
19875        release_tx.send(()).expect("ordinary write should be released");
19876        ordinary_write
19877            .await
19878            .expect("ordinary write task should join")
19879            .expect("the pre-activation ordinary write should complete");
19880        let err = activation
19881            .await
19882            .expect("activation task should join")
19883            .expect_err("activation must recheck and reject capacity consumed by the ordinary write");
19884        assert!(err.to_string().contains("requires 60 bytes, but 59 bytes are available"));
19885    }
19886
19887    async fn single_pool_capacity_admission_test_store() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
19888        let (temp_dirs, store) =
19889            crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta::default()).await;
19890        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
19891        (temp_dirs, store)
19892    }
19893
19894    #[tokio::test]
19895    #[serial_test::serial]
19896    async fn single_pool_public_writes_skip_decommission_capacity_admission() {
19897        let (_temp_dirs, store) = single_pool_capacity_admission_test_store().await;
19898        let bucket = format!("single-pool-capacity-skip-{}", uuid::Uuid::new_v4());
19899        store
19900            .make_bucket(&bucket, &MakeBucketOptions::default())
19901            .await
19902            .expect("create single-pool bucket before blocking pool metadata writes");
19903        let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
19904        store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
19905        let write_gate = store.pool_meta_write_gate_status().await;
19906        assert!(!write_gate.writes_ready);
19907        assert!(write_gate.write_blocked);
19908        assert!(!write_gate.transaction_aborted);
19909
19910        let object = "ordinary-put.bin";
19911        let mut put_data = crate::object_api::PutObjReader::from_vec(b"ordinary single-pool body".to_vec());
19912        store
19913            .put_object(&bucket, object, &mut put_data, &ObjectOptions::default())
19914            .await
19915            .expect("single-pool ordinary PUT must not enter decommission capacity admission");
19916        store
19917            .get_object_info(&bucket, object, &ObjectOptions::default())
19918            .await
19919            .expect("single-pool ordinary PUT must remain readable");
19920
19921        let multipart_object = "ordinary-multipart.bin";
19922        let upload = store
19923            .new_multipart_upload(
19924                &bucket,
19925                multipart_object,
19926                &ObjectOptions {
19927                    expected_bucket_incarnation_id: Some(incarnation),
19928                    ..Default::default()
19929                },
19930            )
19931            .await
19932            .expect("single-pool MPU creation must not enter decommission capacity admission");
19933        let mut part_data = crate::object_api::PutObjReader::from_vec(b"single-pool multipart body".to_vec());
19934        let part = store
19935            .put_object_part(
19936                &bucket,
19937                multipart_object,
19938                &upload.upload_id,
19939                1,
19940                &mut part_data,
19941                &ObjectOptions {
19942                    expected_bucket_incarnation_id: Some(incarnation),
19943                    ..Default::default()
19944                },
19945            )
19946            .await
19947            .expect("single-pool UploadPart must not enter decommission capacity admission");
19948        store
19949            .clone()
19950            .complete_multipart_upload(
19951                &bucket,
19952                multipart_object,
19953                &upload.upload_id,
19954                vec![crate::storage_api_contracts::multipart::CompletePart {
19955                    part_num: part.part_num,
19956                    etag: part.etag,
19957                    ..Default::default()
19958                }],
19959                &ObjectOptions {
19960                    expected_bucket_incarnation_id: Some(incarnation),
19961                    ..Default::default()
19962                },
19963            )
19964            .await
19965            .expect("single-pool CompleteMultipartUpload must not enter decommission capacity admission");
19966        store
19967            .get_object_info(&bucket, multipart_object, &ObjectOptions::default())
19968            .await
19969            .expect("single-pool completed MPU must remain readable");
19970    }
19971
19972    #[tokio::test]
19973    #[serial_test::serial]
19974    async fn multipart_mutations_locate_later_upload_before_reserved_pool_admission() {
19975        let (_temp_dirs, store, _other_store) =
19976            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19977        let bucket = format!("multipart-capacity-routing-{}", uuid::Uuid::new_v4());
19978        let object = "later-pool.bin";
19979        store
19980            .make_bucket(&bucket, &MakeBucketOptions::default())
19981            .await
19982            .expect("create multipart routing bucket");
19983        let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
19984        let bucket_guard = store
19985            .acquire_bucket_lifecycle_read_lock(&bucket)
19986            .await
19987            .expect("acquire multipart routing bucket lifecycle guard");
19988        let mut upload_opts = ObjectOptions {
19989            expected_bucket_incarnation_id: Some(incarnation),
19990            ..Default::default()
19991        };
19992        upload_opts.add_bucket_lifecycle_lock_guard(&bucket_guard);
19993        let upload = store.pools[1]
19994            .new_multipart_upload(&bucket, object, &upload_opts)
19995            .await
19996            .expect("seed an upload in the later pool");
19997
19998        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19999        set_decommission_capacity_info_overrides_for_test(
20000            store.id,
20001            vec![vec![
20002                DecommissionPoolCapacityInfo::for_test(0, layout, 20, 20, 0),
20003                DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
20004                DecommissionPoolCapacityInfo::for_test(2, layout, 0, 10, 10),
20005            ]],
20006        );
20007        store
20008            .save_current_pool_meta_for_decommission_start(&[2], Vec::new())
20009            .await
20010            .expect("reserve the first target pool");
20011        {
20012            let pool_meta = store.pool_meta.read().await;
20013            let targets = &pool_meta.pools[2]
20014                .decommission
20015                .as_ref()
20016                .and_then(|info| info.capacity_reservation.as_ref())
20017                .expect("source reservation should exist")
20018                .targets;
20019            assert_eq!(targets.len(), 1);
20020            assert_eq!(targets[0].pool_index, 0, "the first probed pool must be reserved");
20021        }
20022
20023        let mut data = crate::object_api::PutObjReader::from_vec(b"multipart body".to_vec());
20024        let part = store
20025            .put_object_part(&bucket, object, &upload.upload_id, 1, &mut data, &ObjectOptions::default())
20026            .await
20027            .expect("put-part must locate the later upload before capacity admission");
20028        store
20029            .clone()
20030            .complete_multipart_upload(
20031                &bucket,
20032                object,
20033                &upload.upload_id,
20034                vec![crate::storage_api_contracts::multipart::CompletePart {
20035                    part_num: part.part_num,
20036                    etag: part.etag,
20037                    ..Default::default()
20038                }],
20039                &ObjectOptions::default(),
20040            )
20041            .await
20042            .expect("complete must locate the later upload before capacity admission");
20043
20044        store.pools[1]
20045            .get_object_info(&bucket, object, &ObjectOptions::default())
20046            .await
20047            .expect("the later-pool multipart upload should commit in place");
20048        let first_pool_result = store.pools[0]
20049            .get_object_info(&bucket, object, &ObjectOptions::default())
20050            .await;
20051        assert!(
20052            first_pool_result
20053                .as_ref()
20054                .err()
20055                .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)),
20056            "the reserved first pool must not receive the multipart mutation"
20057        );
20058    }
20059
20060    #[tokio::test]
20061    #[serial_test::serial]
20062    async fn expired_capacity_owner_nonce_rejects_stale_put_and_multipart_without_consuming() {
20063        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
20064        let bucket = format!("stale-capacity-owner-{}", uuid::Uuid::new_v4());
20065        let multipart_object = "stale-multipart.bin";
20066        store
20067            .make_bucket(&bucket, &MakeBucketOptions::default())
20068            .await
20069            .expect("create stale owner bucket");
20070        let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
20071        let bucket_guard = store
20072            .acquire_bucket_lifecycle_read_lock(&bucket)
20073            .await
20074            .expect("acquire stale owner bucket lifecycle guard");
20075        let mut upload_opts = ObjectOptions {
20076            expected_bucket_incarnation_id: Some(incarnation),
20077            ..Default::default()
20078        };
20079        upload_opts.add_bucket_lifecycle_lock_guard(&bucket_guard);
20080        let upload = store.pools[1]
20081            .new_multipart_upload(&bucket, multipart_object, &upload_opts)
20082            .await
20083            .expect("seed multipart upload before reserving the target");
20084
20085        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20086        set_decommission_capacity_info_overrides_for_test(
20087            store.id,
20088            vec![vec![
20089                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
20090                DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
20091            ]],
20092        );
20093        store
20094            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20095            .await
20096            .expect("activate target reservation");
20097
20098        let stale_owner = {
20099            let pool_meta = store.pool_meta.read().await;
20100            let reservation = pool_meta.pools[0]
20101                .decommission
20102                .as_ref()
20103                .and_then(|info| info.capacity_reservation.as_ref())
20104                .expect("active reservation should exist");
20105            DecommissionCapacityOwner {
20106                source_pool_index: 0,
20107                operation_id: reservation.operation_id,
20108                generation: reservation.generation,
20109                owner_nonce: reservation.owner_nonce,
20110                mutation_id: None,
20111            }
20112        };
20113        {
20114            let mut pool_meta = store.pool_meta.write().await;
20115            let reservation = pool_meta.pools[0]
20116                .decommission
20117                .as_mut()
20118                .and_then(|info| info.capacity_reservation.as_mut())
20119                .expect("active reservation should remain mutable");
20120            reservation.expires_at = OffsetDateTime::now_utc() - Duration::seconds(1);
20121        }
20122        store
20123            .save_current_pool_meta(&[0])
20124            .await
20125            .expect("persist the expired lease before recovery");
20126        store
20127            .pause_decommission_for_capacity(0, &decommission_capacity_blocked_error("test lease recovery"))
20128            .await
20129            .expect("pause recovery should renew the lease with a new owner nonce");
20130
20131        let (current_owner, before_progress) = {
20132            let pool_meta = store.pool_meta.read().await;
20133            let reservation = pool_meta.pools[0]
20134                .decommission
20135                .as_ref()
20136                .and_then(|info| info.capacity_reservation.as_ref())
20137                .expect("renewed reservation should exist");
20138            (
20139                DecommissionCapacityOwner {
20140                    source_pool_index: 0,
20141                    operation_id: reservation.operation_id,
20142                    generation: reservation.generation,
20143                    owner_nonce: reservation.owner_nonce,
20144                    mutation_id: None,
20145                },
20146                (
20147                    reservation.consumed_target_physical_bytes,
20148                    reservation.inflight_target_physical_bytes,
20149                    reservation.pending_target_physical_bytes,
20150                    reservation.observed_target_physical_bytes,
20151                ),
20152            )
20153        };
20154        assert_ne!(stale_owner.owner_nonce, current_owner.owner_nonce);
20155        {
20156            let pool_meta = store.pool_meta.read().await;
20157            ensure_decommission_target_owner_admission(&pool_meta, current_owner, 1, 1, OffsetDateTime::now_utc())
20158                .expect("the renewed owner should retain its reservation");
20159            assert!(
20160                ensure_decommission_target_owner_admission(&pool_meta, stale_owner, 1, 1, OffsetDateTime::now_utc(),).is_err(),
20161                "the expired owner token must be rejected after nonce rotation"
20162            );
20163        }
20164
20165        let stale_put_object = "stale-put.bin";
20166        let mut stale_put_opts = ObjectOptions {
20167            data_movement: true,
20168            version_id: Some(uuid::Uuid::new_v4().to_string()),
20169            ..ObjectOptions::with_capacity_expected_data_bytes(Some(9))
20170        };
20171        stale_owner.apply_to(&mut stale_put_opts);
20172        let mut stale_put_data = crate::object_api::PutObjReader::from_vec(b"stale put".to_vec());
20173        let put_err = store
20174            .put_object_for_data_movement(&bucket, stale_put_object, &mut stale_put_data, &stale_put_opts, None)
20175            .await
20176            .expect_err("stale owner PUT must fail before selecting a target");
20177        assert!(is_decommission_capacity_blocked_error(&put_err));
20178
20179        let mut stale_part_opts = ObjectOptions {
20180            data_movement: true,
20181            part_number: Some(1),
20182            expected_bucket_incarnation_id: Some(incarnation),
20183            ..Default::default()
20184        };
20185        stale_owner.apply_to(&mut stale_part_opts);
20186        let mut stale_part_data = crate::object_api::PutObjReader::from_vec(b"stale part".to_vec());
20187        let part_err = store
20188            .put_object_part_for_data_movement(
20189                1,
20190                &bucket,
20191                multipart_object,
20192                &upload.upload_id,
20193                &mut stale_part_data,
20194                &stale_part_opts,
20195            )
20196            .await
20197            .expect_err("stale owner multipart PUT must fail before mutation");
20198        assert!(is_decommission_capacity_blocked_error(&part_err));
20199
20200        let after_progress = {
20201            let pool_meta = store.pool_meta.read().await;
20202            let reservation = pool_meta.pools[0]
20203                .decommission
20204                .as_ref()
20205                .and_then(|info| info.capacity_reservation.as_ref())
20206                .expect("renewed reservation should remain active");
20207            (
20208                reservation.consumed_target_physical_bytes,
20209                reservation.inflight_target_physical_bytes,
20210                reservation.pending_target_physical_bytes,
20211                reservation.observed_target_physical_bytes,
20212            )
20213        };
20214        assert_eq!(after_progress, before_progress, "stale mutations must not consume the capacity ledger");
20215        let parts = store.pools[1]
20216            .list_object_parts(
20217                &bucket,
20218                multipart_object,
20219                &upload.upload_id,
20220                None,
20221                1_000,
20222                &ObjectOptions {
20223                    expected_bucket_incarnation_id: Some(incarnation),
20224                    ..Default::default()
20225                },
20226            )
20227            .await
20228            .expect("the seeded upload should remain readable");
20229        assert!(parts.parts.is_empty(), "the stale multipart write must not stage a part");
20230        assert!(
20231            store.pools[1]
20232                .get_object_info(&bucket, stale_put_object, &ObjectOptions::default())
20233                .await
20234                .is_err(),
20235            "the stale PUT must not create a target object"
20236        );
20237    }
20238
20239    #[tokio::test]
20240    #[serial_test::serial]
20241    async fn decommission_activation_lock_recheck_rejects_sudden_space_drop_without_persisting() {
20242        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
20243        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20244        let source = DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30);
20245        let preflight = vec![source, DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0)];
20246        {
20247            let pool_meta = store.pool_meta.read().await;
20248            ensure_decommission_start_target_capacity(&pool_meta, &[0], &preflight, true)
20249                .expect("the pre-lock capacity snapshot should fit exactly");
20250        }
20251
20252        set_decommission_capacity_info_overrides_for_test(
20253            store.id,
20254            vec![vec![source, DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)]],
20255        );
20256        let err = store
20257            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20258            .await
20259            .expect_err("the capacity snapshot inside the activation lock must be authoritative");
20260        assert!(err.to_string().contains("requires 60 bytes, but 59 bytes are available"));
20261
20262        let local = store.pool_meta.read().await;
20263        assert!(local.pools[0].decommission.is_none());
20264        drop(local);
20265        let mut persisted = PoolMeta::default();
20266        persisted
20267            .load_no_lock_from_replicas(store.pools.clone())
20268            .await
20269            .expect("the rejected activation must leave baseline metadata readable");
20270        assert!(persisted.pools[0].decommission.is_none());
20271    }
20272
20273    #[tokio::test]
20274    #[serial_test::serial]
20275    async fn decommission_worker_pauses_on_runtime_capacity_shortage_without_source_side_effect() {
20276        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
20277        let bucket = format!("capacity-blocked-{}", uuid::Uuid::new_v4());
20278        let object = "object.bin";
20279        store
20280            .make_bucket(&bucket, &MakeBucketOptions::default())
20281            .await
20282            .expect("the production-path bucket should be created");
20283        let mut source = crate::object_api::PutObjReader::from_vec(b"source remains authoritative".to_vec());
20284        store.pools[0]
20285            .put_object(&bucket, object, &mut source, &ObjectOptions::default())
20286            .await
20287            .expect("the source object should be written before decommission starts");
20288
20289        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20290        let enough = vec![
20291            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
20292            DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
20293        ];
20294        set_decommission_capacity_info_overrides_for_test(store.id, vec![enough.clone()]);
20295        store
20296            .save_current_pool_meta_for_decommission_start(
20297                &[0],
20298                vec![DecomBucketInfo {
20299                    name: bucket.clone(),
20300                    prefix: String::new(),
20301                }],
20302            )
20303            .await
20304            .expect("the initial reservation should be activated");
20305
20306        let shortage = vec![enough[0], DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)];
20307        set_decommission_capacity_info_overrides_for_test(store.id, vec![shortage]);
20308        let canceler = DecommissionCanceler::new(CancellationToken::new());
20309        store.decommission_cancelers.write().await[0] = Some(canceler.clone());
20310        store
20311            .do_decommission_in_routine(canceler, 0, Arc::new(Semaphore::new(1)))
20312            .await
20313            .expect("runtime capacity shortage should pause the worker, not fail it");
20314
20315        store.pools[0]
20316            .get_object_info(&bucket, object, &ObjectOptions::default())
20317            .await
20318            .expect("capacity pause must not delete the source object");
20319        let target_result = store.pools[1]
20320            .get_object_info(&bucket, object, &ObjectOptions::default())
20321            .await;
20322        assert!(
20323            target_result
20324                .as_ref()
20325                .err()
20326                .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)),
20327            "capacity pause must happen before a target mutation"
20328        );
20329
20330        let local = store.pool_meta.read().await;
20331        let info = local.pools[0]
20332            .decommission
20333            .as_ref()
20334            .expect("the blocked decommission state should remain present");
20335        assert!(
20336            !info.complete && !info.failed && !info.canceled,
20337            "capacity-blocked state unexpectedly became terminal: complete={}, failed={}, canceled={}, blocked_reason={:?}",
20338            info.complete,
20339            info.failed,
20340            info.canceled,
20341            info.capacity_blocked_reason
20342        );
20343        assert_eq!(info.items_decommission_failed, 0);
20344        assert_eq!(info.bytes_failed, 0);
20345        assert!(info.capacity_blocked_reason.is_some());
20346        assert!(
20347            info.capacity_reservation
20348                .as_ref()
20349                .is_some_and(DecommissionCapacityReservation::active),
20350            "blocked work must retain and renew its reservation"
20351        );
20352        drop(local);
20353
20354        let mut persisted = PoolMeta::default();
20355        persisted
20356            .load_no_lock_from_replicas(store.pools.clone())
20357            .await
20358            .expect("the blocked state should be durable");
20359        let persisted_info = persisted.pools[0]
20360            .decommission
20361            .as_ref()
20362            .expect("the durable blocked state should remain nonterminal");
20363        assert!(!persisted_info.complete && !persisted_info.failed && !persisted_info.canceled);
20364        assert!(persisted_info.capacity_blocked_reason.is_some());
20365        assert!(
20366            persisted_info
20367                .capacity_reservation
20368                .as_ref()
20369                .is_some_and(DecommissionCapacityReservation::active)
20370        );
20371    }
20372
20373    #[tokio::test]
20374    #[serial_test::serial]
20375    async fn decommission_worker_rechecks_runtime_capacity_before_empty_background_completion() {
20376        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
20377        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20378        let enough = vec![
20379            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
20380            DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
20381        ];
20382        set_decommission_capacity_info_overrides_for_test(store.id, vec![enough.clone()]);
20383        store
20384            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20385            .await
20386            .expect("the initial reservation should be activated");
20387
20388        let shortage = vec![enough[0], DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)];
20389        set_decommission_capacity_info_overrides_for_test(store.id, vec![shortage]);
20390        let canceler = DecommissionCanceler::new(CancellationToken::new());
20391        store.decommission_cancelers.write().await[0] = Some(canceler.clone());
20392        store
20393            .do_decommission_in_routine(canceler, 0, Arc::new(Semaphore::new(1)))
20394            .await
20395            .expect("runtime capacity shortage should pause the worker before background completion");
20396
20397        let local = store.pool_meta.read().await;
20398        let info = local.pools[0]
20399            .decommission
20400            .as_ref()
20401            .expect("the blocked decommission state should remain present");
20402        assert!(!info.complete && !info.failed && !info.canceled);
20403        assert!(info.capacity_blocked_reason.is_some());
20404        assert!(
20405            info.capacity_reservation
20406                .as_ref()
20407                .is_some_and(DecommissionCapacityReservation::active)
20408        );
20409    }
20410
20411    fn pool_meta_replica_test_meta(cmd_line: &str) -> PoolMeta {
20412        PoolMeta {
20413            version: POOL_META_VERSION,
20414            pools: vec![PoolStatus {
20415                id: 0,
20416                cmd_line: cmd_line.to_string(),
20417                last_update: OffsetDateTime::UNIX_EPOCH,
20418                decommission: None,
20419            }],
20420            dont_save: false,
20421        }
20422    }
20423
20424    fn pool_meta_replica_test_data(cmd_line: &str) -> Vec<u8> {
20425        pool_meta_replica_test_meta(cmd_line)
20426            .encode_config_data()
20427            .expect("pool metadata should encode")
20428    }
20429
20430    fn pool_meta_v1_replica_test_data(meta: &PoolMeta) -> Vec<u8> {
20431        let mut data = Vec::new();
20432        data.write_u16::<LittleEndian>(POOL_META_FORMAT)
20433            .expect("pool metadata format should encode");
20434        data.write_u16::<LittleEndian>(POOL_META_V1_VERSION)
20435            .expect("pool metadata version should encode");
20436        PersistedPoolMetaV1::from(meta)
20437            .serialize(&mut Serializer::new(&mut data))
20438            .expect("legacy pool metadata should encode");
20439        data
20440    }
20441
20442    fn pool_meta_persisted_v1_replica_test_data(cmd_line: &str) -> Vec<u8> {
20443        pool_meta_v1_replica_test_data(&pool_meta_replica_test_meta(cmd_line))
20444    }
20445
20446    fn pool_meta_legacy_v1_replica_test_data(cmd_line: &str) -> Vec<u8> {
20447        let mut data = Vec::new();
20448        data.write_u16::<LittleEndian>(POOL_META_FORMAT)
20449            .expect("pool metadata format should encode");
20450        data.write_u16::<LittleEndian>(POOL_META_V1_VERSION)
20451            .expect("pool metadata version should encode");
20452        LegacyPoolMeta {
20453            version: POOL_META_V1_VERSION,
20454            pools: vec![LegacyPoolStatus {
20455                id: 0,
20456                cmd_line: cmd_line.to_string(),
20457                last_update: OffsetDateTime::UNIX_EPOCH,
20458                decommission: None,
20459            }],
20460            dont_save: false,
20461        }
20462        .serialize(&mut Serializer::new(&mut data))
20463        .expect("legacy v1 pool metadata should encode");
20464        data
20465    }
20466
20467    #[test]
20468    fn pool_meta_replica_selection_falls_back_from_corrupt_first_copy() {
20469        let selection = select_pool_meta_replica(vec![
20470            PoolMetaReplica::Corrupt("truncated".to_string()),
20471            decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
20472        ])
20473        .expect("a validated backup replica should be selected");
20474
20475        assert!(selection.replica_state.needs_repair);
20476        assert!(selection.replica_state.repair_write_safe);
20477        assert_eq!(selection.meta.pools.len(), 1);
20478        assert_eq!(selection.meta.pools[0].cmd_line, "pool-0");
20479    }
20480
20481    #[test]
20482    fn pool_meta_replica_selection_rejects_incompatible_copy() {
20483        let valid = pool_meta_replica_test_data("pool-0");
20484        let mut incompatible = valid.clone();
20485        LittleEndian::write_u16(&mut incompatible[2..4], POOL_META_VERSION + 1);
20486
20487        let err = select_pool_meta_replica(vec![decode_pool_meta_replica(valid), decode_pool_meta_replica(incompatible)])
20488            .expect_err("an incompatible replica must block fallback and repair writes");
20489
20490        assert!(err.to_string().contains("pool 1 is incompatible"));
20491        assert!(err.to_string().contains("without overwriting it"));
20492    }
20493
20494    #[test]
20495    fn pool_meta_replica_selection_prefers_valid_canonical_first_copy() {
20496        let selection = select_pool_meta_replica(vec![
20497            decode_pool_meta_replica(pool_meta_replica_test_data("pool-canonical")),
20498            decode_pool_meta_replica(pool_meta_replica_test_data("pool-stale")),
20499        ])
20500        .expect("pool zero is the durable commit record when a later replica is stale");
20501
20502        assert_eq!(selection.meta.pools[0].cmd_line, "pool-canonical");
20503        assert!(selection.replica_state.needs_repair);
20504        assert!(selection.replica_state.repair_write_safe);
20505    }
20506
20507    #[test]
20508    fn pool_meta_replica_selection_rejects_divergent_backups_without_canonical() {
20509        let err = select_pool_meta_replica(vec![
20510            PoolMetaReplica::Corrupt("canonical unavailable".to_string()),
20511            decode_pool_meta_replica(pool_meta_replica_test_data("pool-old")),
20512            decode_pool_meta_replica(pool_meta_replica_test_data("pool-new")),
20513        ])
20514        .expect_err("divergent backups have no safe ordering when the canonical copy is unavailable");
20515
20516        assert!(err.to_string().contains("valid replicas in pools 1 and 2 diverge"));
20517    }
20518
20519    #[test]
20520    fn pool_meta_replica_selection_normalizes_equivalent_legacy_copy() {
20521        let selection = select_pool_meta_replica(vec![
20522            decode_pool_meta_replica(pool_meta_legacy_v1_replica_test_data("pool-0")),
20523            decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
20524        ])
20525        .expect("equivalent legacy and current encodings should be compatible");
20526
20527        assert!(selection.replica_state.needs_repair);
20528        assert!(selection.replica_state.repair_write_safe);
20529        assert_eq!(selection.meta.pools[0].cmd_line, "pool-0");
20530    }
20531
20532    #[test]
20533    fn pool_meta_v1_replica_migrates_to_v2_canonical_form() {
20534        let mut source = pool_meta_replica_test_meta("pool-0");
20535        source.pools[0].decommission = Some(PoolDecommissionInfo {
20536            start_time: Some(OffsetDateTime::UNIX_EPOCH),
20537            queued_buckets: vec!["bucket-a".to_string()],
20538            ..Default::default()
20539        });
20540        let data = pool_meta_v1_replica_test_data(&source);
20541        let PoolMetaReplica::Valid {
20542            raw, canonical, meta, ..
20543        } = decode_pool_meta_replica(data)
20544        else {
20545            panic!("v1 pool metadata should remain readable");
20546        };
20547
20548        assert_eq!(LittleEndian::read_u16(&raw[2..4]), POOL_META_V1_VERSION);
20549        assert_eq!(LittleEndian::read_u16(&canonical[2..4]), POOL_META_VERSION);
20550        assert_eq!(meta.version, POOL_META_V1_VERSION);
20551        let info = meta.pools[0]
20552            .decommission
20553            .as_ref()
20554            .expect("v1 decommission state should migrate");
20555        assert_eq!(info.queued_buckets, vec!["bucket-a".to_string()]);
20556        assert!(info.unresolved_entries.is_empty());
20557    }
20558
20559    #[test]
20560    fn pool_meta_legacy_v1_replica_migrates_to_v2_canonical_form() {
20561        let data = pool_meta_legacy_v1_replica_test_data("pool-0");
20562        let PoolMetaReplica::Valid {
20563            raw, canonical, meta, ..
20564        } = decode_pool_meta_replica(data)
20565        else {
20566            panic!("legacy v1 pool metadata should remain readable");
20567        };
20568
20569        assert_eq!(LittleEndian::read_u16(&raw[2..4]), POOL_META_V1_VERSION);
20570        assert_eq!(LittleEndian::read_u16(&canonical[2..4]), POOL_META_VERSION);
20571        assert_eq!(meta.version, POOL_META_V1_VERSION);
20572        assert_eq!(meta.pools[0].cmd_line, "pool-0");
20573    }
20574
20575    #[test]
20576    fn pool_meta_replica_selection_rejects_v1_v2_divergence() {
20577        let err = select_pool_meta_replica(vec![
20578            decode_pool_meta_replica(pool_meta_persisted_v1_replica_test_data("pool-old")),
20579            decode_pool_meta_replica(pool_meta_replica_test_data("pool-new")),
20580        ])
20581        .expect_err("different v1 and v2 snapshots must remain fail-closed");
20582
20583        assert!(err.to_string().contains("valid replicas in pools 0 and 1 diverge"));
20584    }
20585
20586    #[test]
20587    fn pool_meta_replica_rejects_header_payload_version_mismatch() {
20588        let mut v1_with_v2_header = pool_meta_persisted_v1_replica_test_data("pool-0");
20589        LittleEndian::write_u16(&mut v1_with_v2_header[2..4], POOL_META_VERSION);
20590        assert!(matches!(decode_pool_meta_replica(v1_with_v2_header), PoolMetaReplica::Corrupt(_)));
20591
20592        let mut v2_with_v1_header = pool_meta_replica_test_data("pool-0");
20593        LittleEndian::write_u16(&mut v2_with_v1_header[2..4], POOL_META_V1_VERSION);
20594        assert!(matches!(
20595            decode_pool_meta_replica(v2_with_v1_header),
20596            PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
20597        ));
20598    }
20599
20600    #[test]
20601    fn pool_meta_v2_header_guards_v1_readers_from_tuple_extension() {
20602        let mut meta = pool_meta_replica_test_meta("pool-0");
20603        meta.pools[0].decommission = Some(PoolDecommissionInfo::default());
20604        let data = meta.encode_config_data().expect("v2 pool metadata should encode");
20605
20606        assert_eq!(LittleEndian::read_u16(&data[2..4]), POOL_META_VERSION);
20607        assert_ne!(POOL_META_VERSION, POOL_META_V1_VERSION);
20608        assert!(rmp_serde::from_slice::<PersistedPoolMetaV1>(&data[4..]).is_err());
20609    }
20610
20611    #[test]
20612    fn pool_meta_writer_stays_v1_until_v2_is_fleet_confirmed() {
20613        let mut meta = pool_meta_replica_test_meta("pool-0");
20614        meta.version = POOL_META_V1_VERSION;
20615
20616        let v1 = meta
20617            .encode_config_data_for_v2_gate(false)
20618            .expect("the default writer should preserve v1");
20619        let v2 = meta
20620            .encode_config_data_for_v2_gate(true)
20621            .expect("the confirmed writer should emit v2");
20622
20623        assert_eq!(LittleEndian::read_u16(&v1[2..4]), POOL_META_V1_VERSION);
20624        assert_eq!(LittleEndian::read_u16(&v2[2..4]), POOL_META_VERSION);
20625        assert!(rmp_serde::from_slice::<PersistedPoolMetaV1>(&v1[4..]).is_ok());
20626    }
20627
20628    #[test]
20629    fn pool_meta_v2_writer_requires_both_gates() {
20630        assert!(!pool_meta_v2_writer_enabled_for(false, false));
20631        assert!(!pool_meta_v2_writer_enabled_for(true, false));
20632        assert!(!pool_meta_v2_writer_enabled_for(false, true));
20633        assert!(pool_meta_v2_writer_enabled_for(true, true));
20634    }
20635
20636    #[test]
20637    fn pool_meta_v3_writer_requires_both_gates() {
20638        assert!(!pool_meta_v3_writer_enabled_for(false, false));
20639        assert!(!pool_meta_v3_writer_enabled_for(true, false));
20640        assert!(!pool_meta_v3_writer_enabled_for(false, true));
20641        assert!(pool_meta_v3_writer_enabled_for(true, true));
20642    }
20643
20644    #[test]
20645    fn decommission_ledger_persistence_requires_an_observed_or_confirmed_format() {
20646        for (version, v2_enabled, v3_enabled, expected) in [
20647            (POOL_META_V1_VERSION, false, false, false),
20648            (POOL_META_V1_VERSION, true, false, true),
20649            (POOL_META_V1_VERSION, false, true, true),
20650            (POOL_META_VERSION, false, false, true),
20651            (super::POOL_META_GENERATION_VERSION, false, false, true),
20652        ] {
20653            let result = super::ensure_decommission_ledger_persistence_supported_for(version, v2_enabled, v3_enabled);
20654            assert_eq!(
20655                result.is_ok(),
20656                expected,
20657                "unexpected capability result for pool metadata version {version}"
20658            );
20659        }
20660
20661        let half_confirmed_v2 = pool_meta_v2_writer_enabled_for(true, false);
20662        let half_confirmed_v3 = pool_meta_v3_writer_enabled_for(false, true);
20663        let err = super::ensure_decommission_ledger_persistence_supported_for(
20664            POOL_META_V1_VERSION,
20665            half_confirmed_v2,
20666            half_confirmed_v3,
20667        )
20668        .expect_err("half-enabled rollout gates must not admit decommission");
20669        assert!(matches!(err, Error::InvalidArgument(..)));
20670        assert!(err.to_string().contains("durable unresolved-entry recovery"));
20671        assert!(err.to_string().contains(rustfs_config::ENV_POOL_META_V3_WRITE));
20672        assert!(err.to_string().contains(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED));
20673    }
20674
20675    #[test]
20676    fn decommission_capacity_accepts_v3_only_writer_rollout() {
20677        assert!(decommission_capacity_writer_supported_for(
20678            POOL_META_V1_VERSION,
20679            false,
20680            pool_meta_v3_writer_enabled_for(true, true),
20681        ));
20682        assert!(!decommission_capacity_writer_supported_for(
20683            POOL_META_V1_VERSION,
20684            false,
20685            pool_meta_v3_writer_enabled_for(true, false),
20686        ));
20687    }
20688
20689    #[test]
20690    fn pool_meta_stale_write_rejection_metric_is_countable() {
20691        let recorder = metrics_util::debugging::DebuggingRecorder::new();
20692        let snapshotter = recorder.snapshotter();
20693        metrics::with_local_recorder(&recorder, || {
20694            record_pool_meta_stale_write_rejection("prepare_cas");
20695            record_pool_meta_stale_write_rejection("prepare_cas");
20696        });
20697
20698        let total = snapshotter
20699            .snapshot()
20700            .into_vec()
20701            .into_iter()
20702            .filter(|(composite, _, _, _)| composite.key().name() == METRIC_POOL_META_STALE_WRITE_REJECTIONS_TOTAL)
20703            .filter_map(|(_, _, _, value)| match value {
20704                metrics_util::debugging::DebugValue::Counter(value) => Some(value),
20705                _ => None,
20706            })
20707            .sum::<u64>();
20708        assert_eq!(total, 2);
20709    }
20710
20711    #[test]
20712    fn pool_meta_block_transition_is_counted_once_across_both_latches() {
20713        let recorder = metrics_util::debugging::DebuggingRecorder::new();
20714        let snapshotter = recorder.snapshotter();
20715        metrics::with_local_recorder(&recorder, || {
20716            let mut state = PoolMetaWriteState::default();
20717            let mut arm = state.arm_transaction();
20718            arm.phase = Some("commit_cas");
20719            drop(arm);
20720            let first = state
20721                .ensure_write_safe("first block")
20722                .unwrap_err()
20723                .pool_metadata_failure()
20724                .unwrap()
20725                .since;
20726            state.block_writes_after_fence_loss();
20727            state.block_writes();
20728            assert_eq!(
20729                state
20730                    .ensure_write_safe("later block")
20731                    .unwrap_err()
20732                    .pool_metadata_failure()
20733                    .unwrap()
20734                    .since,
20735                first
20736            );
20737        });
20738        let total = snapshotter
20739            .snapshot()
20740            .into_vec()
20741            .into_iter()
20742            .filter(|(key, _, _, _)| key.key().name() == "rustfs_pool_metadata_blocks_total")
20743            .filter_map(|(_, _, _, value)| match value {
20744                metrics_util::debugging::DebugValue::Counter(count) => Some(count),
20745                _ => None,
20746            })
20747            .sum::<u64>();
20748        assert_eq!(total, 1);
20749    }
20750
20751    fn pool_meta_v3_test_revision(cluster_id: uuid::Uuid, generation: u64, transaction_id: uuid::Uuid) -> PoolMetaRevision {
20752        PoolMetaRevision {
20753            version: POOL_META_GENERATION_VERSION,
20754            cluster_id: Some(cluster_id),
20755            epoch: POOL_META_INITIAL_EPOCH,
20756            generation,
20757            transaction_id: Some(transaction_id),
20758        }
20759    }
20760
20761    fn pool_meta_legacy_candidate(meta: PoolMeta) -> PoolMetaCommittedCandidate {
20762        PoolMetaCommittedCandidate {
20763            canonical: meta
20764                .encode_config_data_for_v2_gate(true)
20765                .expect("legacy pool metadata should encode"),
20766            revision: PoolMetaRevision::legacy(meta.version),
20767            meta,
20768        }
20769    }
20770
20771    #[test]
20772    fn pool_meta_v3_pending_prepare_recovers_previous_snapshot_after_restart() {
20773        let previous = pool_meta_replica_test_meta("pool-before");
20774        let mut next = pool_meta_replica_test_meta("pool-after");
20775        next.version = POOL_META_GENERATION_VERSION;
20776        let revision = pool_meta_v3_test_revision(uuid::Uuid::new_v4(), 1, uuid::Uuid::new_v4());
20777        let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&pool_meta_legacy_candidate(previous.clone())))
20778            .expect("pending generation should encode");
20779
20780        let selected = select_pool_meta_replica(vec![
20781            decode_pool_meta_replica(pending),
20782            decode_pool_meta_replica(
20783                previous
20784                    .encode_config_data_for_v2_gate(true)
20785                    .expect("previous snapshot should encode"),
20786            ),
20787        ])
20788        .expect("a prepare-only transaction should expose its committed predecessor");
20789
20790        assert_eq!(selected.meta.pools[0].cmd_line, "pool-before");
20791        assert_eq!(selected.revision.version, POOL_META_VERSION);
20792        assert!(selected.replica_state.needs_repair);
20793    }
20794
20795    #[test]
20796    fn pool_meta_v3_partial_commit_selects_new_generation_after_restart() {
20797        let previous = pool_meta_replica_test_meta("pool-before");
20798        let mut next = pool_meta_replica_test_meta("pool-after");
20799        next.version = POOL_META_GENERATION_VERSION;
20800        let revision = pool_meta_v3_test_revision(uuid::Uuid::new_v4(), 1, uuid::Uuid::new_v4());
20801        let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&pool_meta_legacy_candidate(previous)))
20802            .expect("pending generation should encode");
20803        let committed = encode_pool_meta_v3_envelope(&next, revision, true, None).expect("committed generation should encode");
20804
20805        let selected = select_pool_meta_replica(vec![decode_pool_meta_replica(committed), decode_pool_meta_replica(pending)])
20806            .expect("one committed replica should make the cross-pool transaction durable");
20807
20808        assert_eq!(selected.meta.pools[0].cmd_line, "pool-after");
20809        assert_eq!(selected.revision, revision);
20810        assert!(selected.replica_state.needs_repair);
20811    }
20812
20813    #[test]
20814    fn pool_meta_v3_uses_valid_committed_backup_but_rejects_same_generation_fork() {
20815        let cluster_id = uuid::Uuid::new_v4();
20816        let mut next = pool_meta_replica_test_meta("pool-after");
20817        next.version = POOL_META_GENERATION_VERSION;
20818        let revision = pool_meta_v3_test_revision(cluster_id, 7, uuid::Uuid::new_v4());
20819        let committed = encode_pool_meta_v3_envelope(&next, revision, true, None).expect("committed generation should encode");
20820        let selected = select_pool_meta_replica(vec![
20821            PoolMetaReplica::Corrupt("truncated".to_string()),
20822            decode_pool_meta_replica(committed.clone()),
20823        ])
20824        .expect("a corrupt first copy should fall back to a valid committed generation");
20825        assert_eq!(selected.revision, revision);
20826        assert!(selected.replica_state.needs_repair);
20827
20828        let fork_revision = pool_meta_v3_test_revision(cluster_id, 7, uuid::Uuid::new_v4());
20829        let fork = encode_pool_meta_v3_envelope(&next, fork_revision, true, None).expect("fork generation should encode");
20830        let err = select_pool_meta_replica(vec![decode_pool_meta_replica(committed), decode_pool_meta_replica(fork)])
20831            .expect_err("same-generation transactions must never be selected by pool order");
20832        assert!(err.to_string().contains("committed generation 7 diverges"));
20833    }
20834
20835    #[test]
20836    fn pool_meta_v3_migration_keeps_legacy_committed_until_commit_record_exists() {
20837        let previous = pool_meta_replica_test_meta("pool-before");
20838        let mut next = pool_meta_replica_test_meta("pool-after");
20839        next.version = POOL_META_GENERATION_VERSION;
20840        let revision = pool_meta_v3_test_revision(uuid::Uuid::new_v4(), 1, uuid::Uuid::new_v4());
20841        let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&pool_meta_legacy_candidate(previous.clone())))
20842            .expect("pending migration should encode");
20843        let legacy = previous
20844            .encode_config_data_for_v2_gate(true)
20845            .expect("legacy snapshot should encode");
20846
20847        let prepared = select_pool_meta_replica(vec![decode_pool_meta_replica(pending), decode_pool_meta_replica(legacy)])
20848            .expect("prepared migration should retain the legacy commit");
20849        assert_eq!(prepared.meta.version, POOL_META_VERSION);
20850        assert_eq!(prepared.meta.pools[0].cmd_line, "pool-before");
20851
20852        let committed = encode_pool_meta_v3_envelope(&next, revision, true, None).expect("committed migration should encode");
20853        let migrated = select_pool_meta_replica(vec![decode_pool_meta_replica(committed)])
20854            .expect("committed migration should establish the V3 floor");
20855        assert_eq!(migrated.meta.version, POOL_META_GENERATION_VERSION);
20856        assert_eq!(migrated.meta.pools[0].cmd_line, "pool-after");
20857    }
20858
20859    #[test]
20860    fn pool_meta_v3_unknown_fields_remain_incompatible_not_silently_ignored() {
20861        #[derive(Serialize)]
20862        struct FuturePoolMetaV3 {
20863            version: u16,
20864            cluster_id: String,
20865            epoch: u64,
20866            generation: u64,
20867            transaction_id: String,
20868            committed: bool,
20869            pools: Vec<PersistedPoolStatus>,
20870            previous: Option<PersistedPoolMetaV3Previous>,
20871            future_guard: u64,
20872        }
20873
20874        let mut data = Vec::new();
20875        data.write_u16::<LittleEndian>(POOL_META_FORMAT)
20876            .expect("pool metadata format should encode");
20877        data.write_u16::<LittleEndian>(POOL_META_GENERATION_VERSION)
20878            .expect("pool metadata version should encode");
20879        FuturePoolMetaV3 {
20880            version: POOL_META_GENERATION_VERSION,
20881            cluster_id: uuid::Uuid::new_v4().to_string(),
20882            epoch: POOL_META_INITIAL_EPOCH,
20883            generation: 1,
20884            transaction_id: uuid::Uuid::new_v4().to_string(),
20885            committed: true,
20886            pools: Vec::new(),
20887            previous: None,
20888            future_guard: 1,
20889        }
20890        .serialize(&mut Serializer::new(&mut data))
20891        .expect("future V3 payload should encode");
20892
20893        assert!(matches!(decode_pool_meta_replica(data), PoolMetaReplica::Incompatible(_)));
20894    }
20895
20896    #[test]
20897    fn pool_meta_identity_classifies_corrupt_incompatible_and_divergent_replicas() {
20898        let cluster_id = uuid::Uuid::new_v4();
20899        let identity = PersistedPoolMetaIdentity {
20900            version: POOL_META_IDENTITY_VERSION,
20901            cluster_id,
20902            epoch: POOL_META_INITIAL_EPOCH,
20903            initialized: true,
20904            fresh_bootstrap_nonce: None,
20905        };
20906        let selected = select_pool_meta_identity(
20907            vec![
20908                PoolMetaIdentityRead {
20909                    replica: PoolMetaIdentityReplica::Corrupt("truncated".to_string()),
20910                    cas: PoolMetaCasToken::Existing("corrupt".to_string()),
20911                },
20912                PoolMetaIdentityRead {
20913                    replica: PoolMetaIdentityReplica::Valid(identity),
20914                    cas: PoolMetaCasToken::Existing("valid".to_string()),
20915                },
20916            ],
20917            cluster_id,
20918        )
20919        .expect("a verified identity backup should survive a corrupt first replica");
20920        assert_eq!(selected.identity, Some(identity));
20921        assert!(selected.needs_repair);
20922
20923        let incompatible = select_pool_meta_identity(
20924            vec![PoolMetaIdentityRead {
20925                replica: PoolMetaIdentityReplica::Incompatible("future version".to_string()),
20926                cas: PoolMetaCasToken::Existing("future".to_string()),
20927            }],
20928            cluster_id,
20929        )
20930        .expect_err("an incompatible identity must fail closed");
20931        assert!(incompatible.to_string().contains("incompatible"));
20932
20933        let divergent = select_pool_meta_identity(
20934            vec![
20935                PoolMetaIdentityRead {
20936                    replica: PoolMetaIdentityReplica::Valid(identity),
20937                    cas: PoolMetaCasToken::Existing("epoch-1".to_string()),
20938                },
20939                PoolMetaIdentityRead {
20940                    replica: PoolMetaIdentityReplica::Valid(PersistedPoolMetaIdentity {
20941                        epoch: POOL_META_INITIAL_EPOCH + 1,
20942                        ..identity
20943                    }),
20944                    cas: PoolMetaCasToken::Existing("epoch-2".to_string()),
20945                },
20946            ],
20947            cluster_id,
20948        )
20949        .expect_err("identity epoch divergence must require recovery");
20950        assert!(divergent.to_string().contains("recovery required"));
20951    }
20952
20953    #[test]
20954    fn pool_meta_v2_floor_is_sticky_after_observation() {
20955        let meta = pool_meta_replica_test_meta("pool-0");
20956        let data = meta
20957            .encode_config_data_for_v2_gate(false)
20958            .expect("an observed v2 snapshot must not be downgraded");
20959
20960        assert_eq!(LittleEndian::read_u16(&data[2..4]), POOL_META_VERSION);
20961    }
20962
20963    #[test]
20964    fn pool_meta_v1_writer_rejects_unresolved_ledger() {
20965        let mut meta = pool_meta_replica_test_meta("pool-0");
20966        meta.version = POOL_META_V1_VERSION;
20967        meta.pools[0].decommission = Some(PoolDecommissionInfo {
20968            start_time: Some(OffsetDateTime::UNIX_EPOCH),
20969            unresolved_entries: vec![DecommissionUnresolvedEntry {
20970                bucket: "bucket-a".to_string(),
20971                object: "object-a".to_string(),
20972                pool_index: 0,
20973                set_index: 0,
20974                source_generation: OffsetDateTime::UNIX_EPOCH,
20975                candidate_count: 1,
20976                disk_error_count: 1,
20977                observed_at: OffsetDateTime::UNIX_EPOCH,
20978                reason: "metadata_resolution_failed".to_string(),
20979            }],
20980            ..Default::default()
20981        });
20982
20983        let err = meta
20984            .encode_config_data_for_v2_gate(false)
20985            .expect_err("v1 must not drop the unresolved ledger");
20986        assert!(err.to_string().contains("pool metadata V2 is required"));
20987    }
20988
20989    #[test]
20990    fn pool_meta_v1_writer_rejects_capacity_reservations() {
20991        let mut meta = pool_meta_replica_test_meta("pool-0");
20992        meta.version = POOL_META_V1_VERSION;
20993        let reservation = build_decommission_capacity_reservation(
20994            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 1, parity: 0 }, 0, 10, 10),
20995            DecommissionErasureLayout { data: 1, parity: 0 },
20996            uuid::Uuid::new_v4(),
20997            1,
20998            OffsetDateTime::UNIX_EPOCH,
20999        )
21000        .expect("test reservation should be valid");
21001        meta.pools[0].decommission = Some(PoolDecommissionInfo {
21002            start_time: Some(OffsetDateTime::UNIX_EPOCH),
21003            capacity_reservation: Some(reservation),
21004            ..Default::default()
21005        });
21006
21007        let err = meta
21008            .encode_config_data_for_v2_gate(false)
21009            .expect_err("v1 must not drop a distributed capacity reservation");
21010        assert!(
21011            err.to_string()
21012                .contains("pool metadata V2 is required to persist decommission capacity reservations")
21013        );
21014    }
21015
21016    #[test]
21017    fn pool_meta_replica_selection_preserves_observed_v2_floor() {
21018        let selection = select_pool_meta_replica(vec![
21019            decode_pool_meta_replica(pool_meta_persisted_v1_replica_test_data("pool-0")),
21020            decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
21021        ])
21022        .expect("equivalent v1 and v2 replicas should converge");
21023
21024        assert_eq!(selection.meta.version, POOL_META_VERSION);
21025        let data = selection
21026            .meta
21027            .encode_config_data_for_v2_gate(false)
21028            .expect("the selected v2 floor must remain writable");
21029        assert_eq!(LittleEndian::read_u16(&data[2..4]), POOL_META_VERSION);
21030    }
21031
21032    #[test]
21033    fn pool_meta_replica_selection_distinguishes_absent_from_unrecoverable() {
21034        assert!(matches!(decode_pool_meta_replica(Vec::new()), PoolMetaReplica::Corrupt(_)));
21035
21036        let empty = select_pool_meta_replica(vec![PoolMetaReplica::Missing, PoolMetaReplica::Missing])
21037            .expect("all missing replicas should preserve new-deployment behavior");
21038        assert!(empty.meta.pools.is_empty());
21039        assert!(!empty.replica_state.needs_repair);
21040        assert!(empty.replica_state.repair_write_safe);
21041
21042        let err = select_pool_meta_replica(vec![
21043            PoolMetaReplica::Missing,
21044            PoolMetaReplica::Unreadable(Arc::new(Error::other("read quorum unavailable"))),
21045        ])
21046        .expect_err("an unreadable replica must not be treated as a new deployment");
21047        assert!(err.to_string().contains("no valid committed replica is available"));
21048        assert!(err.to_string().contains("pool 1 is unreadable"));
21049    }
21050
21051    #[test]
21052    fn pool_meta_replica_selection_blocks_repair_for_unreadable_copy() {
21053        let selection = select_pool_meta_replica(vec![
21054            decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
21055            PoolMetaReplica::Unreadable(Arc::new(Error::other("read quorum unavailable"))),
21056        ])
21057        .expect("a validated replica should remain usable while another copy is unreadable");
21058
21059        assert!(selection.replica_state.needs_repair);
21060        assert!(!selection.replica_state.repair_write_safe);
21061    }
21062
21063    #[test]
21064    fn pool_meta_write_state_remains_blocked_after_unreadable_replica() {
21065        let mut write_state = PoolMetaWriteState::default();
21066        write_state.observe_replicas(PoolMetaReplicaState {
21067            needs_repair: true,
21068            repair_write_safe: false,
21069        });
21070        write_state.observe_replicas(PoolMetaReplicaState {
21071            needs_repair: false,
21072            repair_write_safe: true,
21073        });
21074
21075        let err = write_state
21076            .ensure_write_safe("pool metadata save failed")
21077            .expect_err("a later clean read must not clear the startup write block");
21078        assert!(
21079            err.to_string()
21080                .contains("restart after all replicas are readable and consistent")
21081        );
21082    }
21083
21084    #[test]
21085    fn pool_meta_read_probe_does_not_latch_writer_state() {
21086        let write_state = PoolMetaWriteState::default();
21087        select_pool_meta_replicas_for_read_probe(
21088            &write_state,
21089            vec![PoolMetaReplica::Unreadable(Arc::new(Error::Timeout))],
21090            "capacity probe",
21091        )
21092        .expect_err("an unreadable probe replica must fail the current admission");
21093
21094        assert!(
21095            write_state.ensure_write_safe("ordinary object write").is_ok(),
21096            "a read-only capacity probe must not permanently latch the pool metadata writer"
21097        );
21098    }
21099
21100    #[test]
21101    fn pool_meta_preflight_drop_does_not_arm_but_started_drop_retains_first_failure() {
21102        let state = PoolMetaWriteState::default();
21103        drop(state.arm_transaction());
21104        state.ensure_write_safe("preflight retry").unwrap();
21105        let mut first = state.arm_transaction();
21106        first.phase = Some("identity_cas");
21107        let err = first.failure(Error::Timeout);
21108        drop(first);
21109        let context = state.ensure_write_safe("retry").unwrap_err();
21110        let first_failure = context.pool_metadata_failure().unwrap();
21111        assert_eq!(first_failure.kind, crate::error::PoolMetadataFailure::TransactionUnknown);
21112        assert_eq!(first_failure.phase, "identity_cas");
21113        assert!(matches!(first_failure.source.as_deref(), Some(Error::Timeout)));
21114        assert!(matches!(err.pool_metadata_failure().unwrap().source.as_deref(), Some(Error::Timeout)));
21115        let mut late = state.arm_transaction();
21116        late.phase = Some("commit_cas");
21117        drop(late);
21118        let later = state.ensure_write_safe("retry").unwrap_err();
21119        assert_eq!(later.pool_metadata_failure().unwrap().since, first_failure.since);
21120        assert_eq!(later.pool_metadata_failure().unwrap().phase, "identity_cas");
21121        assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
21122    }
21123
21124    #[tokio::test]
21125    #[serial_test::serial]
21126    async fn pool_meta_writer_retries_after_identity_read_failure_without_restart() {
21127        let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
21128        let mut disks = Vec::new();
21129        for set in &store.pools[1].disk_set {
21130            let mut guard = set.disks.write().await;
21131            let disk_count = guard.len();
21132            let original = std::mem::replace(&mut *guard, vec![None; disk_count]);
21133            disks.push((set.clone(), original));
21134        }
21135        let mut state = store.pool_meta_save_gate.lock().await;
21136        let err = store
21137            .acquire_pool_meta_write_guard(&mut state, "writer preflight")
21138            .await
21139            .unwrap_err();
21140        assert_eq!(
21141            err.pool_metadata_failure().unwrap().kind,
21142            crate::error::PoolMetadataFailure::ReadUnavailable
21143        );
21144        state.ensure_write_safe("retry").unwrap();
21145        for (set, original) in disks {
21146            *set.disks.write().await = original;
21147        }
21148        let (_guard, _) = store.acquire_pool_meta_write_guard(&mut state, "writer retry").await.unwrap();
21149    }
21150
21151    #[tokio::test]
21152    #[serial_test::serial]
21153    async fn pool_meta_runtime_recovery_publishes_durable_state_and_drains_old_workers() {
21154        let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
21155        let before = store.pool_meta.read().await.clone();
21156        let mut requested = before.clone();
21157        requested.pools[0].last_update += Duration::seconds(1);
21158        requested.version = POOL_META_GENERATION_VERSION;
21159        {
21160            let mut state = store.pool_meta_save_gate.lock().await;
21161            let (fence, _) = store.acquire_pool_meta_write_guard(&mut state, "test save").await.unwrap();
21162            let outcome = requested
21163                .save_no_lock_armed(store.pools.clone(), &mut state, fence.lock_lost_signal(), &[0])
21164                .await
21165                .unwrap();
21166            assert_eq!(outcome.committed.pools[0].last_update, requested.pools[0].last_update);
21167            drop(outcome);
21168        }
21169        assert!(!store.pool_meta_writes_ready().await);
21170        let worker = DecommissionCanceler::new(CancellationToken::new());
21171        store.decommission_cancelers.write().await[0] = Some(worker.clone());
21172        assert!(store.recover_pool_meta_transaction().await.is_err());
21173        assert!(worker.is_cancelled());
21174        assert!(!store.pool_meta_writes_ready().await);
21175        assert_eq!(store.pool_meta.read().await.pools[0].last_update, before.pools[0].last_update);
21176        worker.release();
21177        store.pool_meta.write().await.pools[0].last_update = requested.pools[0].last_update + Duration::seconds(10);
21178        // An outcome may outlive the save mutex. Its Drop must precede recovery.
21179        let outstanding = store.pool_meta_save_gate.lock().await.arm_transaction();
21180        assert!(!store.recover_pool_meta_transaction().await.unwrap());
21181        drop(outstanding);
21182        assert!(store.recover_pool_meta_transaction().await.unwrap());
21183        assert!(store.pool_meta_writes_ready().await);
21184        let recovered = store.pool_meta_write_gate_status().await;
21185        assert!(recovered.writes_ready);
21186        assert!(!recovered.check_timed_out);
21187        assert!(!recovered.write_blocked);
21188        assert!(!recovered.transaction_aborted);
21189        assert_eq!((recovered.reason, recovered.phase, recovered.since_unix_secs), (None, None, None));
21190        assert_eq!(store.pool_meta.read().await.pools[0].last_update, requested.pools[0].last_update);
21191        assert!(!store.recover_pool_meta_transaction().await.unwrap());
21192        store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
21193        assert!(!store.recover_pool_meta_transaction().await.unwrap());
21194        assert!(!store.pool_meta_writes_ready().await);
21195    }
21196
21197    #[tokio::test]
21198    #[serial_test::serial]
21199    async fn pool_meta_readiness_is_bounded_while_save_gate_is_held() {
21200        let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
21201        let _state = store.pool_meta_save_gate.lock().await;
21202        let result = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_status())
21203            .await
21204            .unwrap();
21205        assert!(matches!(result, Err(Error::Timeout)));
21206        let snapshot = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_gate_status())
21207            .await
21208            .unwrap();
21209        assert!(!snapshot.writes_ready);
21210        assert!(snapshot.check_timed_out);
21211        assert!(!snapshot.write_blocked);
21212        assert!(!snapshot.transaction_aborted);
21213        assert_eq!((snapshot.reason, snapshot.phase, snapshot.since_unix_secs), (None, None, None));
21214    }
21215
21216    #[tokio::test]
21217    #[serial_test::serial]
21218    async fn pool_meta_cancelled_recovery_and_new_integrity_block_never_clear_original_gate() {
21219        let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
21220        let since = OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap();
21221        {
21222            let state = store.pool_meta_save_gate.lock().await;
21223            let mut arm = state.arm_transaction();
21224            arm.phase = Some("publication");
21225            drop(arm);
21226            // An old fixed timestamp detects polling-time resets without sleeping.
21227            state.transaction_failure.lock().unwrap().as_mut().unwrap().since = since;
21228            *state.block_started_at.lock().unwrap() = Some(since);
21229        }
21230        let original = store.pool_meta_write_gate_status().await;
21231        assert!(!original.writes_ready);
21232        assert!(!original.check_timed_out);
21233        assert!(!original.write_blocked);
21234        assert!(original.transaction_aborted);
21235        assert_eq!(original.reason, Some("transaction_unknown"));
21236        assert_eq!(original.phase, Some("publication"));
21237        assert_eq!(original.since_unix_secs, Some(since.unix_timestamp()));
21238        assert_eq!(store.pool_meta_write_gate_status().await, original);
21239        let publication_guard = store.pool_meta.write().await;
21240        let recovery = tokio::spawn({
21241            let store = store.clone();
21242            async move { store.recover_pool_meta_transaction().await }
21243        });
21244        tokio::time::timeout(std::time::Duration::from_secs(10), async {
21245            loop {
21246                if store.pool_meta_save_gate.try_lock().is_err() {
21247                    break;
21248                }
21249                tokio::task::yield_now().await;
21250            }
21251        })
21252        .await
21253        .unwrap();
21254        recovery.abort();
21255        assert!(recovery.await.unwrap_err().is_cancelled());
21256        drop(publication_guard);
21257        assert!(!store.pool_meta_writes_ready().await);
21258        assert_eq!(store.pool_meta_write_gate_status().await, original);
21259        let start_guard = store.start_gate.lock().await;
21260        let recovery = tokio::spawn({
21261            let store = store.clone();
21262            async move { store.recover_pool_meta_transaction().await }
21263        });
21264        store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
21265        drop(start_guard);
21266        assert!(!recovery.await.unwrap().unwrap());
21267        assert_eq!(
21268            store
21269                .pool_meta_write_status()
21270                .await
21271                .unwrap_err()
21272                .pool_metadata_failure()
21273                .unwrap()
21274                .kind,
21275            crate::error::PoolMetadataFailure::FenceLost
21276        );
21277        let fenced = store.pool_meta_write_gate_status().await;
21278        assert!(!fenced.writes_ready);
21279        assert!(fenced.write_blocked);
21280        assert_eq!(fenced.reason, Some("fence_lost"));
21281        assert_eq!(fenced.phase, Some("format_heal"));
21282        assert_eq!(fenced.since_unix_secs, original.since_unix_secs);
21283    }
21284
21285    #[test]
21286    fn pool_meta_read_probe_rejects_missing_runtime_metadata_without_latching() {
21287        let write_state = PoolMetaWriteState {
21288            expected_cluster_id: Some(uuid::Uuid::new_v4()),
21289            identity_initialized: Some(true),
21290            ..Default::default()
21291        };
21292        select_pool_meta_replicas_for_read_probe(&write_state, vec![PoolMetaReplica::Missing], "capacity probe")
21293            .expect_err("runtime metadata disappearance must reject the current probe");
21294        write_state
21295            .ensure_write_safe("ordinary object write")
21296            .expect("a missing-metadata probe must not permanently latch the writer");
21297    }
21298
21299    #[tokio::test]
21300    #[serial_test::serial]
21301    async fn pool_meta_read_guard_does_not_latch_after_unreadable_replica() {
21302        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
21303        let mut saved_disks = Vec::new();
21304        for set in &store.pools[1].disk_set {
21305            let mut disks = set.disks.write().await;
21306            let original = std::mem::take(&mut *disks);
21307            let disk_count = original.len();
21308            saved_disks.push((set.clone(), original));
21309            *disks = vec![None; disk_count];
21310        }
21311
21312        let write_state = store.pool_meta_save_gate.lock().await;
21313        store
21314            .acquire_pool_meta_read_guard(&write_state, "capacity probe")
21315            .await
21316            .expect_err("an unreadable metadata replica must reject this probe");
21317        write_state
21318            .ensure_write_safe("ordinary object write")
21319            .expect("a failed read-only probe must remain retryable");
21320
21321        for (set, disks) in saved_disks {
21322            *set.disks.write().await = disks;
21323        }
21324        store
21325            .acquire_pool_meta_read_guard(&write_state, "capacity probe retry")
21326            .await
21327            .expect("a read-only probe must succeed after the replica recovers");
21328    }
21329
21330    #[test]
21331    fn pool_meta_writer_read_failure_is_retryable_when_all_replicas_are_unreadable() {
21332        let replicas = vec![
21333            PoolMetaReplica::Unreadable(Arc::new(Error::other("pool 0 read quorum unavailable"))),
21334            PoolMetaReplica::Unreadable(Arc::new(Error::other("pool 1 read quorum unavailable"))),
21335        ];
21336        let mut write_state = PoolMetaWriteState::default();
21337
21338        let err = select_pool_meta_replicas_observing(&mut write_state, replicas)
21339            .expect_err("all unreadable replicas must fail selection");
21340        assert_eq!(
21341            err.pool_metadata_failure().unwrap().kind,
21342            crate::error::PoolMetadataFailure::ReadUnavailable
21343        );
21344        write_state
21345            .ensure_write_safe("pool metadata save failed")
21346            .expect("a read-only preflight has no unknown writes and must remain retryable");
21347    }
21348
21349    #[test]
21350    fn pool_meta_write_state_blocks_on_any_recovery_required_selection() {
21351        fn assert_selection_blocks(replicas: Vec<PoolMetaReplica>) {
21352            let mut write_state = PoolMetaWriteState::default();
21353            select_pool_meta_replicas_observing(&mut write_state, replicas)
21354                .expect_err("recovery-required replicas must fail selection");
21355            write_state
21356                .ensure_write_safe("pool metadata save failed")
21357                .expect_err("a recovery-required selection must latch the write block");
21358        }
21359
21360        assert_selection_blocks(vec![PoolMetaReplica::Corrupt("truncated".to_string())]);
21361        assert_selection_blocks(vec![PoolMetaReplica::Incompatible("future format".to_string())]);
21362        assert_selection_blocks(vec![
21363            PoolMetaReplica::Corrupt("canonical unavailable".to_string()),
21364            decode_pool_meta_replica(pool_meta_replica_test_data("pool-old")),
21365            decode_pool_meta_replica(pool_meta_replica_test_data("pool-new")),
21366        ]);
21367    }
21368
21369    #[test]
21370    fn pool_meta_replica_selection_rejects_same_version_tuple_extension() {
21371        #[derive(Serialize)]
21372        struct FuturePersistedPoolMeta {
21373            version: u16,
21374            pools: Vec<PersistedPoolStatus>,
21375            generation: u64,
21376        }
21377
21378        let mut data = Vec::new();
21379        data.write_u16::<LittleEndian>(POOL_META_FORMAT)
21380            .expect("pool metadata format should encode");
21381        data.write_u16::<LittleEndian>(POOL_META_VERSION)
21382            .expect("pool metadata version should encode");
21383        FuturePersistedPoolMeta {
21384            version: POOL_META_VERSION,
21385            pools: Vec::new(),
21386            generation: 2,
21387        }
21388        .serialize(&mut Serializer::new(&mut data))
21389        .expect("extended tuple pool metadata should encode");
21390
21391        let err = select_pool_meta_replica(vec![
21392            decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
21393            decode_pool_meta_replica(data),
21394        ])
21395        .expect_err("same-version tuple extensions must block fallback repair writes");
21396
21397        assert!(err.to_string().contains("pool 1 is incompatible"));
21398        assert!(err.to_string().contains("version 2 tuple has unsupported field count"));
21399    }
21400
21401    #[test]
21402    fn pool_meta_replica_selection_falls_back_from_truncated_current_tuple() {
21403        let mut truncated = pool_meta_replica_test_data("pool-truncated");
21404        truncated.pop();
21405
21406        let selection = select_pool_meta_replica(vec![
21407            decode_pool_meta_replica(truncated),
21408            decode_pool_meta_replica(pool_meta_replica_test_data("pool-valid")),
21409        ])
21410        .expect("a truncated tuple should not block a validated backup replica");
21411
21412        assert!(selection.replica_state.needs_repair);
21413        assert!(selection.replica_state.repair_write_safe);
21414        assert_eq!(selection.meta.pools[0].cmd_line, "pool-valid");
21415    }
21416
21417    #[test]
21418    fn ensure_pool_not_left_in_cmdline_after_decommission_allows_active_pool() {
21419        assert!(ensure_pool_not_left_in_cmdline_after_decommission(0, "http://node{1...4}/disk{1...4}", false).is_ok());
21420    }
21421
21422    #[test]
21423    fn ensure_pool_not_left_in_cmdline_after_decommission_rejects_completed_pool() {
21424        let err = ensure_pool_not_left_in_cmdline_after_decommission(1, "http://node{1...4}/disk{1...4}", true)
21425            .expect_err("completed decommissioned pool should fail validation");
21426
21427        assert!(
21428            err.to_string()
21429                .contains("pool(2) = http://node{1...4}/disk{1...4} is decommissioned, please remove from server command line")
21430        );
21431    }
21432
21433    #[test]
21434    fn determine_decommission_final_state_marks_failures_and_cancellations() {
21435        assert_eq!(determine_decommission_final_state(0, false), DecommissionFinalState::Complete);
21436        assert_eq!(determine_decommission_final_state(1, false), DecommissionFinalState::Failed);
21437        assert_eq!(determine_decommission_final_state(0, true), DecommissionFinalState::Failed);
21438    }
21439
21440    #[test]
21441    fn lifecycle_action_removes_data_movement_version_rejects_delete_marker_action() {
21442        assert!(!lifecycle_action_removes_data_movement_version(IlmAction::DeleteAction));
21443    }
21444
21445    #[test]
21446    fn lifecycle_action_removes_data_movement_version_accepts_version_delete_actions() {
21447        assert!(lifecycle_action_removes_data_movement_version(IlmAction::DeleteVersionAction));
21448        assert!(lifecycle_action_removes_data_movement_version(IlmAction::DeleteAllVersionsAction));
21449        assert!(lifecycle_action_removes_data_movement_version(
21450            IlmAction::DelMarkerDeleteAllVersionsAction
21451        ));
21452    }
21453
21454    #[test]
21455    fn lifecycle_action_skips_heal_version_for_every_delete_action() {
21456        assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAction));
21457        assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteVersionAction));
21458        assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredAction));
21459        assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredVersionAction));
21460        assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAllVersionsAction));
21461        assert!(lifecycle_action_skips_heal_version(IlmAction::DelMarkerDeleteAllVersionsAction));
21462        assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionAction));
21463        assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionVersionAction));
21464        assert!(!lifecycle_action_skips_heal_version(IlmAction::NoneAction));
21465    }
21466
21467    #[test]
21468    fn resolve_data_movement_lifecycle_expiry_result_allows_dry_run_skip() {
21469        let skip = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, false, false)
21470            .expect("dry-run lifecycle evaluation should not require expiry enqueue");
21471
21472        assert!(skip);
21473    }
21474
21475    #[test]
21476    fn resolve_data_movement_lifecycle_expiry_result_rejects_apply_failure() {
21477        let err = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, true, false)
21478            .expect_err("failed lifecycle expiry enqueue should not be treated as skipped");
21479
21480        assert!(err.to_string().contains("failed to apply lifecycle expiry action"));
21481    }
21482
21483    #[test]
21484    fn decommission_copy_cleanup_safe_error_accepts_missing_source_errors() {
21485        assert!(is_decommission_copy_cleanup_safe_error(&Error::ObjectNotFound(
21486            "bucket".to_string(),
21487            "object".to_string()
21488        )));
21489        assert!(is_decommission_copy_cleanup_safe_error(&Error::VersionNotFound(
21490            "bucket".to_string(),
21491            "object".to_string(),
21492            "version".to_string()
21493        )));
21494    }
21495
21496    #[test]
21497    fn decommission_free_version_attempt_treats_missing_source_as_consumed() {
21498        let attempt =
21499            classify_decommission_free_version_attempt(Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())));
21500
21501        assert!(matches!(attempt, DecommissionFreeVersionAttempt::Consumed));
21502    }
21503
21504    #[test]
21505    fn decommission_free_version_attempt_preserves_capacity_failure() {
21506        let attempt = classify_decommission_free_version_attempt(Err(Error::DiskFull));
21507
21508        assert!(matches!(attempt, DecommissionFreeVersionAttempt::CapacityFailure(Error::DiskFull)));
21509    }
21510
21511    #[test]
21512    fn decommission_delete_marker_copy_error_rejects_data_movement_overwrite() {
21513        let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
21514
21515        assert!(!is_decommission_copy_cleanup_safe_error(&err));
21516    }
21517
21518    #[test]
21519    fn decommission_remote_tiered_copy_error_rejects_data_movement_overwrite() {
21520        let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
21521
21522        assert!(!is_decommission_copy_cleanup_safe_error(&err));
21523    }
21524
21525    #[test]
21526    fn decommission_target_capacity_error_accepts_direct_capacity_errors() {
21527        assert!(is_decommission_target_capacity_error(&Error::DiskFull));
21528        assert!(is_decommission_target_capacity_error(&Error::StorageFull));
21529    }
21530
21531    /// The decommission loop classifies errors that came back through a
21532    /// data-movement stage wrapper. Before backlog#1827 T2 the wrapper flattened
21533    /// everything into `Error::other(String)`, so these two classifiers had to
21534    /// match on rendered text; now the wrapped error is recoverable by type.
21535    #[test]
21536    fn decommission_classifiers_see_through_a_stage_wrapper() {
21537        let wrap = |inner: Error| {
21538            data_movement::data_movement_stage_error_for_test("decommission_object", "put_object", "bucket-a", "object-a", inner)
21539        };
21540
21541        // Capacity: the target pool filling up must still stop the loop.
21542        assert!(is_decommission_target_capacity_error(&wrap(Error::DiskFull)));
21543        assert!(is_decommission_target_capacity_error(&wrap(Error::StorageFull)));
21544        assert!(!is_decommission_target_capacity_error(&wrap(Error::SlowDown)));
21545
21546        let gate_busy = decommission_capacity_blocked_error(format!(
21547            "{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}7{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}"
21548        ));
21549        assert_eq!(decommission_capacity_target_gate_busy_index(&wrap(gate_busy)), Some(7));
21550
21551        // Cleanup safety: a not-found surfacing from inside a stage is the same
21552        // condition as one surfacing directly, so the source entry stays
21553        // eligible for cleanup.
21554        let not_found = Error::ObjectNotFound("bucket-a".to_string(), "object-a".to_string());
21555        assert!(is_decommission_copy_cleanup_safe_error(&not_found));
21556        assert!(is_decommission_copy_cleanup_safe_error(&wrap(not_found)));
21557        assert!(!is_decommission_copy_cleanup_safe_error(&wrap(Error::SlowDown)));
21558    }
21559
21560    #[test]
21561    fn decommission_target_gate_retry_recognizes_multipart_part_errors() {
21562        let wrap = |inner: Error| {
21563            data_movement::data_movement_part_stage_error_for_test(
21564                "decommission_object",
21565                "put_object_part",
21566                "bucket-a",
21567                "object-a",
21568                1,
21569                inner,
21570            )
21571        };
21572        let gate_busy_message =
21573            format!("{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}7{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}");
21574        let wrapped = wrap(decommission_capacity_blocked_error(&gate_busy_message));
21575        assert!(is_decommission_capacity_target_gate_busy(&wrapped));
21576        assert_eq!(decommission_capacity_target_gate_busy_index(&wrapped), Some(7));
21577        assert_eq!(
21578            wrapped.to_string(),
21579            format!(
21580                "Io error: decommission_object: put_object_part failed for bucket-a/object-a part 1: {}",
21581                decommission_capacity_blocked_error(&gate_busy_message)
21582            )
21583        );
21584
21585        for unrelated in [
21586            Error::SlowDown,
21587            Error::DiskFull,
21588            decommission_capacity_blocked_error("target capacity is exhausted"),
21589            Error::other(gate_busy_message),
21590        ] {
21591            let wrapped = wrap(unrelated);
21592            assert!(!is_decommission_capacity_target_gate_busy(&wrapped));
21593            assert_eq!(decommission_capacity_target_gate_busy_index(&wrapped), None);
21594        }
21595        for missing_target in [
21596            Error::FileNotFound,
21597            Error::ObjectNotFound("bucket-a".to_string(), "object-a".to_string()),
21598            Error::VersionNotFound("bucket-a".to_string(), "object-a".to_string(), "version-a".to_string()),
21599        ] {
21600            assert!(is_decommission_copy_cleanup_safe_error(&missing_target));
21601            assert!(
21602                !is_decommission_copy_cleanup_safe_error(&wrap(missing_target)),
21603                "a missing target part must never authorize source cleanup"
21604            );
21605        }
21606        assert!(is_decommission_target_capacity_error(&wrap(Error::DiskFull)));
21607    }
21608
21609    #[test]
21610    fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
21611        let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
21612        let storage_full = Error::other(format!(
21613            "decommission_object: put_object failed for bucket/object: {}",
21614            Error::StorageFull
21615        ));
21616
21617        assert!(is_decommission_target_capacity_error(&disk_full));
21618        assert!(is_decommission_target_capacity_error(&storage_full));
21619    }
21620
21621    #[test]
21622    fn decommission_capacity_intent_conflict_accepts_context_wrapped_error() {
21623        let err = with_decommission_entry_context(
21624            "migrate_object",
21625            "bucket",
21626            "object",
21627            decommission_capacity_blocked_error("target has an unresolved target capacity intent"),
21628        );
21629
21630        assert!(is_decommission_capacity_intent_conflict(&err));
21631        assert_eq!(
21632            decommission_capacity_retry_kind(&err, DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS - 1),
21633            Some(DecommissionCapacityRetryKind::IntentConflict)
21634        );
21635        assert_eq!(
21636            decommission_capacity_retry_kind(&err, DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS),
21637            None,
21638            "a durable intent conflict must retain its bounded recovery policy"
21639        );
21640    }
21641
21642    #[test]
21643    fn decommission_target_capacity_error_rejects_unrelated_errors() {
21644        assert!(!is_decommission_target_capacity_error(&Error::SlowDown));
21645    }
21646
21647    #[test]
21648    fn should_skip_decommission_delete_marker_characterizes_empty_marker_without_replication() {
21649        let version = rustfs_filemeta::FileInfo {
21650            deleted: true,
21651            ..Default::default()
21652        };
21653
21654        assert!(should_skip_decommission_delete_marker(&version, 1, false));
21655    }
21656
21657    #[test]
21658    fn should_skip_decommission_delete_marker_characterizes_replication_configured() {
21659        let version = rustfs_filemeta::FileInfo {
21660            deleted: true,
21661            ..Default::default()
21662        };
21663
21664        assert!(!should_skip_decommission_delete_marker(&version, 1, true));
21665    }
21666
21667    #[test]
21668    fn should_skip_decommission_delete_marker_rejects_non_deleted_versions() {
21669        let version = rustfs_filemeta::FileInfo::default();
21670
21671        assert!(!should_skip_decommission_delete_marker(&version, 1, false));
21672    }
21673
21674    #[test]
21675    fn should_skip_decommission_delete_marker_rejects_multiple_remaining_versions() {
21676        let version = rustfs_filemeta::FileInfo {
21677            deleted: true,
21678            ..Default::default()
21679        };
21680
21681        assert!(!should_skip_decommission_delete_marker(&version, 2, false));
21682    }
21683
21684    #[test]
21685    fn decommission_delete_marker_opts_preserves_replication_state() {
21686        let mod_time = OffsetDateTime::now_utc();
21687        let version = rustfs_filemeta::FileInfo {
21688            mod_time: Some(mod_time),
21689            replication_state_internal: Some(crate::bucket::replication::replication_state_to_filemeta(&ReplicationState {
21690                replica_status: ReplicationStatusType::Replica,
21691                delete_marker: true,
21692                replicate_decision_str: "existing".to_string(),
21693                ..Default::default()
21694            })),
21695            ..Default::default()
21696        };
21697
21698        let incarnation = uuid::Uuid::new_v4();
21699        let opts = decommission_delete_marker_opts(&version, Some("version-id".to_string()), 7, Some(incarnation));
21700        let replication = opts.delete_replication.expect("replication state should be preserved");
21701
21702        assert!(opts.versioned);
21703        assert!(opts.data_movement);
21704        assert!(opts.delete_marker);
21705        assert!(opts.skip_decommissioned);
21706        assert_eq!(opts.src_pool_idx, 7);
21707        assert_eq!(opts.version_id.as_deref(), Some("version-id"));
21708        assert_eq!(opts.mod_time, Some(mod_time));
21709        assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
21710        assert_eq!(replication.replica_status, ReplicationStatusType::Replica);
21711        assert!(replication.delete_marker);
21712        assert_eq!(replication.replicate_decision_str, "existing");
21713    }
21714
21715    #[test]
21716    fn decommission_delete_marker_opts_preserves_suspended_null_version() {
21717        let version = rustfs_filemeta::FileInfo {
21718            name: "object".to_string(),
21719            deleted: true,
21720            ..Default::default()
21721        };
21722        let opts = decommission_delete_marker_opts(&version, None, 7, None);
21723
21724        assert!(!opts.versioned);
21725        assert!(opts.version_suspended);
21726        assert_eq!(opts.version_id.as_deref(), Some(uuid::Uuid::nil().to_string().as_str()));
21727
21728        let owner = DecommissionCapacityOwner {
21729            source_pool_index: 7,
21730            operation_id: uuid::Uuid::new_v4(),
21731            generation: 1,
21732            owner_nonce: uuid::Uuid::new_v4(),
21733            mutation_id: None,
21734        };
21735        assert_eq!(
21736            decommission_capacity_version_mutation_id(owner, "bucket", &version),
21737            decommission_capacity_mutation_id(
21738                owner,
21739                "bucket",
21740                &version.name,
21741                opts.version_id.as_deref(),
21742                opts.delete_marker,
21743                opts.mod_time,
21744            )
21745        );
21746    }
21747
21748    #[test]
21749    fn test_decommission_object_migration_read_opts_are_raw_data_movement() {
21750        let opts = decommission_object_migration_read_opts(Some("vid-1".to_string()));
21751
21752        assert_eq!(opts.version_id.as_deref(), Some("vid-1"));
21753        assert!(opts.no_lock);
21754        assert!(opts.data_movement);
21755        assert!(opts.raw_data_movement_read);
21756        assert!(opts.skip_rebalancing);
21757        assert!(opts.skip_decommissioned);
21758    }
21759
21760    #[test]
21761    fn decommission_remote_tiered_opts_preserves_versioning_context() {
21762        let mod_time = OffsetDateTime::now_utc();
21763        let version = rustfs_filemeta::FileInfo {
21764            mod_time: Some(mod_time),
21765            metadata: HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())]),
21766            ..Default::default()
21767        };
21768
21769        let incarnation = uuid::Uuid::new_v4();
21770        let opts = decommission_remote_tiered_opts(&version, Some("version-id".to_string()), 9, Some(incarnation));
21771
21772        assert!(opts.versioned);
21773        assert!(opts.data_movement);
21774        assert_eq!(opts.src_pool_idx, 9);
21775        assert_eq!(opts.version_id.as_deref(), Some("version-id"));
21776        assert_eq!(opts.mod_time, Some(mod_time));
21777        assert_eq!(opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value"));
21778        assert!(opts.include_part_checksums);
21779        assert!(opts.http_preconditions.is_some());
21780        assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
21781        assert!(!opts.incl_free_versions);
21782
21783        let mut free_version = version;
21784        free_version.set_tier_free_version();
21785        let free_opts = decommission_remote_tiered_opts(&free_version, Some("free-version-id".to_string()), 9, Some(incarnation));
21786        assert!(free_opts.incl_free_versions);
21787    }
21788
21789    #[test]
21790    fn decommission_terminal_state_transitions_update_start_time() {
21791        let start_time = OffsetDateTime::now_utc();
21792        let build_pool_meta = || PoolMeta {
21793            version: POOL_META_VERSION,
21794            pools: vec![PoolStatus {
21795                id: 0,
21796                cmd_line: "/tmp/pool".to_string(),
21797                last_update: start_time,
21798                decommission: Some(PoolDecommissionInfo {
21799                    start_time: Some(start_time),
21800                    ..Default::default()
21801                }),
21802            }],
21803            dont_save: true,
21804        };
21805
21806        let mut pool_meta = build_pool_meta();
21807        assert!(pool_meta.decommission_failed(0));
21808        assert_eq!(pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), None);
21809
21810        let mut pool_meta = build_pool_meta();
21811        assert!(pool_meta.decommission_complete(0));
21812        assert_eq!(
21813            pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time),
21814            Some(start_time)
21815        );
21816
21817        let mut pool_meta = build_pool_meta();
21818        assert!(pool_meta.decommission_cancel(0));
21819        assert_eq!(pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), None);
21820
21821        let mut pool_meta = build_pool_meta();
21822        assert!(pool_meta.decommission_cancel(0));
21823        assert!(!pool_meta.decommission_complete(0));
21824
21825        let mut pool_meta = build_pool_meta();
21826        assert!(pool_meta.decommission_failed(0));
21827        assert!(!pool_meta.decommission_complete(0));
21828    }
21829
21830    #[test]
21831    fn pool_meta_persists_decommission_resume_queues() {
21832        let start_time = OffsetDateTime::now_utc();
21833        let unresolved_entry = DecommissionUnresolvedEntry {
21834            bucket: "bucket-b".to_string(),
21835            object: "prefix/unresolved.txt".to_string(),
21836            pool_index: 0,
21837            set_index: 1,
21838            source_generation: start_time,
21839            candidate_count: 2,
21840            disk_error_count: 1,
21841            observed_at: start_time,
21842            reason: "metadata_resolution_failed".to_string(),
21843        };
21844        let pool_meta = PoolMeta {
21845            version: POOL_META_VERSION,
21846            pools: vec![PoolStatus {
21847                id: 1,
21848                cmd_line: "/data/pool1/disk{1...4}".to_string(),
21849                last_update: start_time,
21850                decommission: Some(PoolDecommissionInfo {
21851                    start_time: Some(start_time),
21852                    queued: true,
21853                    queued_buckets: vec!["bucket-a".to_string(), "bucket-b/prefix".to_string()],
21854                    decommissioned_buckets: vec!["bucket-done".to_string()],
21855                    bucket: "bucket-b".to_string(),
21856                    prefix: "prefix".to_string(),
21857                    object: "object.txt".to_string(),
21858                    items_decommissioned: 7,
21859                    items_decommission_failed: 1,
21860                    bytes_done: 1024,
21861                    bytes_failed: 128,
21862                    terminal_reload_attempt_at: Some(start_time),
21863                    terminal_reload_failures: vec!["complete_decommission: peer node-a failed".to_string()],
21864                    unresolved_entries: vec![unresolved_entry.clone()],
21865                    ..Default::default()
21866                }),
21867            }],
21868            dont_save: false,
21869        };
21870
21871        let mut buf = Vec::new();
21872        PersistedPoolMeta::from(&pool_meta)
21873            .serialize(&mut Serializer::new(&mut buf))
21874            .expect("pool meta should serialize");
21875
21876        let mut deserializer = Deserializer::new(Cursor::new(&buf));
21877        let restored: PoolMeta = PersistedPoolMeta::deserialize(&mut deserializer)
21878            .expect("pool meta should deserialize")
21879            .try_into()
21880            .expect("pool meta should validate");
21881
21882        let restored_decommission = restored.pools[0]
21883            .decommission
21884            .as_ref()
21885            .expect("decommission info should survive round-trip");
21886        assert_eq!(
21887            restored_decommission.queued_buckets,
21888            vec!["bucket-a".to_string(), "bucket-b/prefix".to_string()]
21889        );
21890        assert_eq!(restored_decommission.decommissioned_buckets, vec!["bucket-done".to_string()]);
21891        assert_eq!(restored_decommission.bucket, "bucket-b");
21892        assert_eq!(restored_decommission.prefix, "prefix");
21893        assert_eq!(restored_decommission.object, "object.txt");
21894        assert!(restored_decommission.stage.is_empty());
21895        assert_eq!(restored_decommission.items_decommissioned, 7);
21896        assert_eq!(restored_decommission.items_decommission_failed, 1);
21897        assert_eq!(restored_decommission.bytes_done, 1024);
21898        assert_eq!(restored_decommission.bytes_failed, 128);
21899        assert_eq!(restored_decommission.terminal_reload_attempt_at, Some(start_time));
21900        assert_eq!(
21901            restored_decommission.terminal_reload_failures,
21902            vec!["complete_decommission: peer node-a failed".to_string()]
21903        );
21904        assert_eq!(restored_decommission.unresolved_entries, vec![unresolved_entry]);
21905        assert!(restored_decommission.queued);
21906        assert_eq!(restored_decommission.items_since_last_progress_save(), 0);
21907    }
21908
21909    #[test]
21910    fn pool_meta_records_decommission_terminal_reload_failure_once() {
21911        let start_time = OffsetDateTime::now_utc();
21912        let mut pool_meta = PoolMeta {
21913            version: POOL_META_VERSION,
21914            pools: vec![PoolStatus {
21915                id: 1,
21916                cmd_line: "/data/pool1/disk{1...4}".to_string(),
21917                last_update: start_time,
21918                decommission: Some(PoolDecommissionInfo {
21919                    complete: true,
21920                    ..Default::default()
21921                }),
21922            }],
21923            dont_save: false,
21924        };
21925
21926        assert!(
21927            pool_meta
21928                .record_decommission_terminal_reload_failure(0, "complete_decommission", "peer node-a failed".to_string())
21929                .expect("terminal reload failure should be recorded")
21930        );
21931        assert!(
21932            !pool_meta
21933                .record_decommission_terminal_reload_failure(0, "complete_decommission", "peer node-a failed".to_string())
21934                .expect("duplicate terminal reload failure should be ignored")
21935        );
21936
21937        let decommission = pool_meta.pools[0].decommission.as_ref().expect("decommission should exist");
21938        assert!(decommission.terminal_reload_attempt_at.is_some());
21939        assert_eq!(
21940            decommission.terminal_reload_failures,
21941            vec!["complete_decommission: peer node-a failed".to_string()]
21942        );
21943    }
21944
21945    #[test]
21946    fn pool_meta_decode_supports_legacy_payload() {
21947        let start_time = OffsetDateTime::now_utc();
21948        let legacy_meta = LegacyPoolMeta {
21949            version: POOL_META_V1_VERSION,
21950            pools: vec![LegacyPoolStatus {
21951                id: 3,
21952                cmd_line: "/legacy/pool".to_string(),
21953                last_update: start_time,
21954                decommission: Some(LegacyPoolDecommissionInfo {
21955                    start_time: Some(start_time),
21956                    items_decommissioned: 9,
21957                    items_decommission_failed: 2,
21958                    bytes_done: 2048,
21959                    bytes_failed: 256,
21960                    ..Default::default()
21961                }),
21962            }],
21963            dont_save: true,
21964        };
21965
21966        let mut legacy_payload = Vec::new();
21967        legacy_meta
21968            .serialize(&mut Serializer::new(&mut legacy_payload))
21969            .expect("legacy payload should serialize");
21970
21971        // New persisted schema has fewer top-level fields and should not decode this legacy struct payload.
21972        let persisted_decode: std::result::Result<PersistedPoolMeta, _> = rmp_serde::from_slice(&legacy_payload);
21973        assert!(persisted_decode.is_err());
21974
21975        let decoded =
21976            PoolMeta::decode_pool_meta_payload(POOL_META_V1_VERSION, &legacy_payload).expect("legacy payload should decode");
21977        assert_eq!(decoded.version, POOL_META_V1_VERSION);
21978        assert!(!decoded.dont_save, "runtime-only flag should reset on load");
21979        assert_eq!(decoded.pools.len(), 1);
21980        assert_eq!(decoded.pools[0].id, 3);
21981        assert_eq!(decoded.pools[0].cmd_line, "/legacy/pool");
21982        assert_eq!(decoded.pools[0].last_update, start_time);
21983
21984        let decommission = decoded.pools[0].decommission.as_ref().expect("decommission should decode");
21985        assert_eq!(decommission.start_time, Some(start_time));
21986        assert_eq!(decommission.items_decommissioned, 9);
21987        assert_eq!(decommission.items_decommission_failed, 2);
21988        assert_eq!(decommission.bytes_done, 2048);
21989        assert_eq!(decommission.bytes_failed, 256);
21990        assert_eq!(decommission.items_since_last_progress_save(), 0);
21991        // These fields were skipped in legacy payload and should be defaulted.
21992        assert!(decommission.queued_buckets.is_empty());
21993        assert!(decommission.decommissioned_buckets.is_empty());
21994        assert!(decommission.bucket.is_empty());
21995        assert!(decommission.prefix.is_empty());
21996        assert!(decommission.object.is_empty());
21997        assert!(decommission.unresolved_entries.is_empty());
21998    }
21999
22000    #[test]
22001    fn pool_meta_decode_rejects_unknown_legacy_fields() {
22002        #[derive(Serialize)]
22003        struct LegacyPoolMetaWithUnknownField {
22004            version: u16,
22005            pools: Vec<LegacyPoolStatus>,
22006            dont_save: bool,
22007            unexpected: bool,
22008        }
22009
22010        let payload = rmp_serde::to_vec_named(&LegacyPoolMetaWithUnknownField {
22011            version: POOL_META_V1_VERSION,
22012            pools: Vec::new(),
22013            dont_save: true,
22014            unexpected: true,
22015        })
22016        .expect("legacy pool metadata with unknown field should serialize");
22017
22018        let err = PoolMeta::decode_pool_meta_payload(POOL_META_V1_VERSION, payload.as_slice())
22019            .expect_err("unknown legacy pool metadata field should fail decode");
22020        let rendered = err.to_string();
22021        assert!(rendered.contains("PoolMeta v1 decode failed for both persisted and legacy formats"));
22022        assert!(rendered.contains("unknown field") || rendered.contains("missing field"));
22023    }
22024
22025    #[test]
22026    fn pool_meta_decode_rejects_unknown_persisted_fields() {
22027        #[derive(Serialize)]
22028        struct PersistedPoolMetaWithUnknownField {
22029            version: u16,
22030            pools: Vec<PersistedPoolStatus>,
22031            unexpected: bool,
22032        }
22033
22034        let payload = rmp_serde::to_vec_named(&PersistedPoolMetaWithUnknownField {
22035            version: POOL_META_VERSION,
22036            pools: Vec::new(),
22037            unexpected: true,
22038        })
22039        .expect("pool metadata with unknown field should serialize");
22040
22041        let err = PoolMeta::decode_pool_meta_payload(POOL_META_VERSION, payload.as_slice())
22042            .expect_err("unknown persisted pool metadata field should fail decode");
22043        let rendered = err.to_string();
22044        assert!(rendered.contains("PoolMeta v2 decode failed"));
22045        assert!(rendered.contains("unknown field") || rendered.contains("missing field"));
22046    }
22047
22048    #[test]
22049    fn pool_meta_decode_rejects_missing_critical_persisted_fields() {
22050        #[derive(Serialize)]
22051        struct PersistedPoolMetaWithoutPools {
22052            version: u16,
22053        }
22054
22055        let payload = rmp_serde::to_vec_named(&PersistedPoolMetaWithoutPools {
22056            version: POOL_META_VERSION,
22057        })
22058        .expect("pool metadata without pools should serialize");
22059
22060        let err = PoolMeta::decode_pool_meta_payload(POOL_META_VERSION, payload.as_slice())
22061            .expect_err("missing persisted pool metadata pools should fail decode");
22062        assert!(err.to_string().contains("PoolMeta v2 decode failed"));
22063    }
22064
22065    #[test]
22066    fn pool_meta_decode_rejects_unknown_decommission_fields() {
22067        #[derive(Serialize)]
22068        struct PersistedPoolStatusWithUnknownDecommission {
22069            #[serde(rename = "id")]
22070            id: usize,
22071            #[serde(rename = "cmdline")]
22072            cmd_line: String,
22073            #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
22074            last_update: OffsetDateTime,
22075            #[serde(rename = "decommissionInfo")]
22076            decommission: Option<PersistedPoolDecommissionInfoWithUnknownField>,
22077        }
22078
22079        #[derive(Serialize)]
22080        struct PersistedPoolDecommissionInfoWithUnknownField {
22081            #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
22082            start_time: Option<OffsetDateTime>,
22083            #[serde(rename = "startSize")]
22084            start_size: usize,
22085            #[serde(rename = "totalSize")]
22086            total_size: usize,
22087            #[serde(rename = "currentSize")]
22088            current_size: usize,
22089            #[serde(rename = "complete")]
22090            complete: bool,
22091            #[serde(rename = "failed")]
22092            failed: bool,
22093            #[serde(rename = "canceled")]
22094            canceled: bool,
22095            #[serde(rename = "queuedBuckets")]
22096            queued_buckets: Vec<String>,
22097            #[serde(rename = "decommissionedBuckets")]
22098            decommissioned_buckets: Vec<String>,
22099            #[serde(rename = "bucket")]
22100            bucket: String,
22101            #[serde(rename = "prefix")]
22102            prefix: String,
22103            #[serde(rename = "object")]
22104            object: String,
22105            #[serde(rename = "objectsDecommissioned")]
22106            items_decommissioned: usize,
22107            #[serde(rename = "objectsDecommissionedFailed")]
22108            items_decommission_failed: usize,
22109            #[serde(rename = "bytesDecommissioned")]
22110            bytes_done: usize,
22111            #[serde(rename = "bytesDecommissionedFailed")]
22112            bytes_failed: usize,
22113            #[serde(rename = "unexpected")]
22114            unexpected: bool,
22115        }
22116
22117        #[derive(Serialize)]
22118        struct PersistedPoolMetaWithUnknownDecommission {
22119            version: u16,
22120            pools: Vec<PersistedPoolStatusWithUnknownDecommission>,
22121        }
22122
22123        let start_time = OffsetDateTime::now_utc();
22124        let payload = rmp_serde::to_vec_named(&PersistedPoolMetaWithUnknownDecommission {
22125            version: POOL_META_VERSION,
22126            pools: vec![PersistedPoolStatusWithUnknownDecommission {
22127                id: 0,
22128                cmd_line: "/data/pool".to_string(),
22129                last_update: start_time,
22130                decommission: Some(PersistedPoolDecommissionInfoWithUnknownField {
22131                    start_time: Some(start_time),
22132                    start_size: 0,
22133                    total_size: 0,
22134                    current_size: 0,
22135                    complete: false,
22136                    failed: false,
22137                    canceled: false,
22138                    queued_buckets: Vec::new(),
22139                    decommissioned_buckets: Vec::new(),
22140                    bucket: String::new(),
22141                    prefix: String::new(),
22142                    object: String::new(),
22143                    items_decommissioned: 0,
22144                    items_decommission_failed: 0,
22145                    bytes_done: 0,
22146                    bytes_failed: 0,
22147                    unexpected: true,
22148                }),
22149            }],
22150        })
22151        .expect("pool metadata with unknown decommission field should serialize");
22152
22153        let err = PoolMeta::decode_pool_meta_payload(POOL_META_VERSION, payload.as_slice())
22154            .expect_err("unknown persisted decommission metadata field should fail decode");
22155        assert!(err.to_string().contains("PoolMeta v2 decode failed"));
22156    }
22157
22158    #[test]
22159    fn pool_meta_decode_rejects_invalid_decommission_terminal_state() {
22160        let start_time = OffsetDateTime::now_utc();
22161        let persisted_meta = PersistedPoolMeta {
22162            version: POOL_META_VERSION,
22163            pools: vec![PersistedPoolStatus {
22164                id: 1,
22165                cmd_line: "/data/pool1/disk{1...4}".to_string(),
22166                last_update: start_time,
22167                decommission: Some(PersistedPoolDecommissionInfo {
22168                    start_time: Some(start_time),
22169                    complete: true,
22170                    failed: true,
22171                    canceled: false,
22172                    ..Default::default()
22173                }),
22174            }],
22175        };
22176
22177        let mut payload = Vec::new();
22178        persisted_meta
22179            .serialize(&mut Serializer::new(&mut payload))
22180            .expect("persisted payload should serialize");
22181
22182        let err = PoolMeta::decode_pool_meta_payload(POOL_META_VERSION, &payload)
22183            .expect_err("invalid terminal state should fail decode");
22184        assert!(err.to_string().contains("invalid decommission terminal state"));
22185    }
22186
22187    #[test]
22188    fn pool_meta_decode_rejects_invalid_legacy_decommission_terminal_state() {
22189        let start_time = OffsetDateTime::now_utc();
22190        let legacy_meta = LegacyPoolMeta {
22191            version: POOL_META_V1_VERSION,
22192            pools: vec![LegacyPoolStatus {
22193                id: 1,
22194                cmd_line: "/legacy/pool".to_string(),
22195                last_update: start_time,
22196                decommission: Some(LegacyPoolDecommissionInfo {
22197                    start_time: Some(start_time),
22198                    complete: true,
22199                    failed: false,
22200                    canceled: true,
22201                    ..Default::default()
22202                }),
22203            }],
22204            dont_save: false,
22205        };
22206
22207        let mut payload = Vec::new();
22208        legacy_meta
22209            .serialize(&mut Serializer::new(&mut payload))
22210            .expect("legacy payload should serialize");
22211
22212        let err = PoolMeta::decode_pool_meta_payload(POOL_META_V1_VERSION, &payload)
22213            .expect_err("invalid legacy terminal state should fail decode");
22214        assert!(err.to_string().contains("invalid decommission terminal state"));
22215    }
22216}
22217
22218// impl Fn(MetaCacheEntry) -> impl Future<Output = Result<(), Error>>
22219
22220pub type ListCallback = Arc<dyn Fn(MetaCacheEntry) -> BoxFuture<'static, ()> + Send + Sync + 'static>;
22221
22222const DECOMMISSION_ENTRY_QUEUE_HARD_CAP: usize = 256;
22223
22224struct QueuedDecommissionEntry {
22225    entry: MetaCacheEntry,
22226    queue_permit: OwnedSemaphorePermit,
22227}
22228
22229enum DecommissionEntryEnqueueResult {
22230    Enqueued,
22231    Canceled,
22232    Closed,
22233}
22234
22235fn decommission_entry_queue_capacity(worker_limit: usize) -> usize {
22236    worker_limit.saturating_mul(2).clamp(1, DECOMMISSION_ENTRY_QUEUE_HARD_CAP)
22237}
22238
22239async fn enqueue_decommission_entry(
22240    rx: &CancellationToken,
22241    outstanding: &Arc<Semaphore>,
22242    tx: &mpsc::Sender<QueuedDecommissionEntry>,
22243    entry: MetaCacheEntry,
22244) -> DecommissionEntryEnqueueResult {
22245    let queue_permit = match tokio::select! {
22246        biased;
22247        _ = rx.cancelled() => return DecommissionEntryEnqueueResult::Canceled,
22248        permit = outstanding.clone().acquire_owned() => permit,
22249    } {
22250        Ok(permit) => permit,
22251        Err(_) => return DecommissionEntryEnqueueResult::Closed,
22252    };
22253
22254    let queued = QueuedDecommissionEntry { entry, queue_permit };
22255    tokio::select! {
22256        biased;
22257        _ = rx.cancelled() => DecommissionEntryEnqueueResult::Canceled,
22258        result = tx.send(queued) => {
22259            if result.is_ok() {
22260                DecommissionEntryEnqueueResult::Enqueued
22261            } else {
22262                DecommissionEntryEnqueueResult::Closed
22263            }
22264        }
22265    }
22266}
22267
22268async fn drain_decommission_entry_queue(rx: &CancellationToken, outstanding: &Arc<Semaphore>, capacity: usize) -> bool {
22269    let Ok(permits) = u32::try_from(capacity) else {
22270        return true;
22271    };
22272
22273    tokio::select! {
22274        _ = rx.cancelled() => true,
22275        result = outstanding.acquire_many(permits) => result.is_err(),
22276    }
22277}
22278
22279async fn record_decommission_entry_error(
22280    entry_error: &Arc<tokio::sync::Mutex<Option<Error>>>,
22281    rx: &CancellationToken,
22282    err: Error,
22283) -> bool {
22284    if rx.is_cancelled() {
22285        return false;
22286    }
22287
22288    let mut first_err = entry_error.lock().await;
22289    if first_err.is_none() && !rx.is_cancelled() {
22290        *first_err = Some(err);
22291        rx.cancel();
22292        return true;
22293    }
22294    false
22295}
22296
22297fn ensure_decommission_unresolved_verification_disk_count(
22298    expected: usize,
22299    actual: usize,
22300    pool_index: usize,
22301    set_index: usize,
22302) -> Result<()> {
22303    if actual == expected {
22304        return Ok(());
22305    }
22306    Err(Error::other(format!(
22307        "decommission unresolved-entry verification for pool {pool_index} set {set_index} requires all {expected} source disks, but only {actual} are online"
22308    )))
22309}
22310
22311impl SetDisks {
22312    async fn decommission_unresolved_entry_absent_on_all_disks(
22313        &self,
22314        pool_index: usize,
22315        entry: &DecommissionUnresolvedEntry,
22316    ) -> Result<bool> {
22317        let (disks, _) = self.get_online_disks_with_healing(false).await;
22318        ensure_decommission_unresolved_verification_disk_count(self.set_drive_count, disks.len(), pool_index, entry.set_index)?;
22319
22320        let object = encode_dir_object(&entry.object);
22321        let reads = join_all(disks.iter().map(|disk| disk.read_xl(&entry.bucket, &object, false))).await;
22322        let mut found = false;
22323        for read in reads {
22324            match read {
22325                Ok(_) => found = true,
22326                Err(DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound) => {}
22327                Err(err) => {
22328                    return Err(Error::other(format!(
22329                        "decommission unresolved-entry verification failed for pool {pool_index} set {} path {}/{}: {err}",
22330                        entry.set_index, entry.bucket, entry.object
22331                    )));
22332                }
22333            }
22334        }
22335        Ok(!found)
22336    }
22337
22338    #[tracing::instrument(skip(self, store, rx, cb_func, entry_error))]
22339    #[allow(clippy::too_many_arguments)]
22340    async fn list_objects_to_decommission(
22341        self: &Arc<Self>,
22342        store: Arc<ECStore>,
22343        rx: CancellationToken,
22344        bucket_info: DecomBucketInfo,
22345        cb_func: ListCallback,
22346        entry_error: Arc<tokio::sync::Mutex<Option<Error>>>,
22347        pool_index: usize,
22348        set_index: usize,
22349        source_generation: OffsetDateTime,
22350        require_all_disks: bool,
22351    ) -> Result<()> {
22352        let (disks, _) = self.get_online_disks_with_healing(false).await;
22353        ensure_decommission_listing_disks_available(!disks.is_empty(), &bucket_info.name)?;
22354        if require_all_disks {
22355            ensure_decommission_unresolved_verification_disk_count(self.set_drive_count, disks.len(), pool_index, set_index)?;
22356        }
22357
22358        let listing_quorum = self.set_drive_count.div_ceil(2);
22359
22360        let resolver = MetadataResolutionParams {
22361            dir_quorum: listing_quorum,
22362            obj_quorum: listing_quorum,
22363            bucket: bucket_info.name.clone(),
22364            ..Default::default()
22365        };
22366
22367        let cb1 = cb_func.clone();
22368        let unresolved_error = entry_error.clone();
22369        let unresolved_rx = rx.clone();
22370        let unresolved_bucket = bucket_info.name.clone();
22371        let unresolved_prefix = bucket_info.prefix.clone();
22372        let unresolved_pool_index = pool_index;
22373        let unresolved_set_index = set_index;
22374        let unresolved_generation = source_generation;
22375        let unresolved_store = store;
22376
22377        list_path_raw(
22378            rx,
22379            ListPathRawOptions {
22380                disks: disks.iter().cloned().map(Some).collect(),
22381                bucket: bucket_info.name.clone(),
22382                path: bucket_info.prefix.clone(),
22383                recursive: true,
22384                min_disks: listing_quorum,
22385                skip_walkdir_total_timeout: true,
22386                walkdir_stall_timeout: Some(DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT),
22387                agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
22388                partial: Some(Box::new(move |entries: MetaCacheEntries, errs: &[Option<DiskError>]| {
22389                    let resolver = resolver.clone();
22390                    let cb_func = cb_func.clone();
22391                    let bucket = unresolved_bucket.clone();
22392                    let prefix = unresolved_prefix.clone();
22393                    let unresolved_error = unresolved_error.clone();
22394                    let unresolved_rx = unresolved_rx.clone();
22395                    let unresolved_store = unresolved_store.clone();
22396                    let pool_index = unresolved_pool_index;
22397                    let set_index = unresolved_set_index;
22398                    let source_generation = unresolved_generation;
22399                    let disk_error_count = errs.iter().flatten().count();
22400                    if unresolved_rx.is_cancelled() {
22401                        return Box::pin(async {});
22402                    }
22403
22404                    match resolve_decommission_partial_listing_entry(
22405                        entries,
22406                        resolver,
22407                        &bucket,
22408                        &prefix,
22409                        disk_error_count,
22410                        pool_index,
22411                        set_index,
22412                        source_generation,
22413                    ) {
22414                        Ok(entry) => {
22415                            warn!("decommission_pool: list_objects_to_decommission get {}", &entry.name);
22416                            Box::pin(async move {
22417                                cb_func(entry).await;
22418                            })
22419                        }
22420                        Err(unresolved_entry) => Box::pin(async move {
22421                            if unresolved_rx.is_cancelled() {
22422                                return;
22423                            }
22424                            let err = decommission_unresolved_listing_error(&unresolved_entry);
22425                            warn!(
22426                                event = EVENT_DECOMMISSION_BUCKET,
22427                                component = LOG_COMPONENT_ECSTORE,
22428                                subsystem = LOG_SUBSYSTEM_POOLS,
22429                                bucket = %bucket,
22430                                prefix = %prefix,
22431                                state = "unresolved_entry",
22432                                error = %err,
22433                                "Decommission listing failed closed on unresolved metadata"
22434                            );
22435                            let err = match unresolved_store
22436                                .persist_decommission_unresolved_entry(pool_index, source_generation, unresolved_entry)
22437                                .await
22438                            {
22439                                Ok(()) => err,
22440                                Err(ledger_err) => Error::other(format!("{err}; {ledger_err}")),
22441                            };
22442                            record_decommission_entry_error(&unresolved_error, &unresolved_rx, err).await;
22443                        }),
22444                    }
22445                })),
22446                ..Default::default()
22447            },
22448        )
22449        .await?;
22450
22451        if let Some(err) = entry_error.lock().await.clone() {
22452            return Err(err);
22453        }
22454
22455        Ok(())
22456    }
22457}
22458
22459fn is_disk_online_state(state: &str) -> bool {
22460    // The disk state strings are produced from rustfs_utils::os::get_drive_stats or DiskError::to_string().
22461    // Conventionally, online is "ok"/"online" (may evolve). Be conservative:
22462    // - Treat empty as unknown -> include it (to avoid dropping capacity).
22463    // - Exclude explicit offline-ish states.
22464    let s = state.trim().to_lowercase();
22465    if s.is_empty() {
22466        return true;
22467    }
22468    if s.contains("offline") {
22469        return false;
22470    }
22471    if s.contains("not found") || s.contains("disk not found") {
22472        return false;
22473    }
22474    true
22475}
22476
22477fn decommission_physical_pool_capacity(
22478    disks: &[rustfs_madmin::Disk],
22479    pool_index: usize,
22480    layout: DecommissionErasureLayout,
22481    logical: PoolSpaceInfo,
22482) -> (usize, usize, usize) {
22483    let width = layout.width();
22484    let mut sets: HashMap<i32, Vec<&rustfs_madmin::Disk>> = HashMap::new();
22485    let mut seen = HashSet::new();
22486    for disk in disks {
22487        let state = disk.state.trim().to_ascii_lowercase();
22488        if disk.pool_index != pool_index as i32
22489            || disk.set_index < 0
22490            || disk.disk_index < 0
22491            || disk.disk_index as usize >= width
22492            || !matches!(state.as_str(), "ok" | "online")
22493        {
22494            continue;
22495        }
22496        let identity = (disk.set_index, disk.disk_index);
22497        if seen.insert(identity) {
22498            sets.entry(disk.set_index).or_default().push(disk);
22499        }
22500    }
22501
22502    let mut physical_total = 0usize;
22503    let mut physical_free = 0usize;
22504    let mut physical_used = 0usize;
22505    let mut observed_set = false;
22506    for set_disks in sets.values() {
22507        if set_disks.is_empty() {
22508            continue;
22509        }
22510        observed_set = true;
22511        let min_total = set_disks
22512            .iter()
22513            .map(|disk| disk.total_space as usize)
22514            .min()
22515            .unwrap_or_default();
22516        let min_free = set_disks
22517            .iter()
22518            .map(|disk| disk.available_space as usize)
22519            .min()
22520            .unwrap_or_default();
22521        let max_used = set_disks
22522            .iter()
22523            .map(|disk| (disk.used_space as usize).max((disk.total_space as usize).saturating_sub(disk.available_space as usize)))
22524            .max()
22525            .unwrap_or_default();
22526        physical_total = physical_total.saturating_add(min_total.saturating_mul(width));
22527        physical_used = physical_used.saturating_add(max_used.saturating_mul(width));
22528        if set_disks.len() >= width {
22529            physical_free = physical_free.saturating_add(min_free.saturating_mul(width));
22530        }
22531    }
22532
22533    if observed_set {
22534        return (physical_total, physical_free, physical_used);
22535    }
22536
22537    let fallback_total = capacity_mul_div_ceil(logical.total, width, layout.data);
22538    let fallback_used = capacity_mul_div_ceil(logical.used, width, layout.data)
22539        .max(fallback_total.saturating_sub(capacity_mul_div_ceil(logical.free, width, layout.data)));
22540    (fallback_total, 0, fallback_used)
22541}
22542
22543#[deprecated(since = "0.1.0", note = "Use fallback_total_capacity_dedup instead")]
22544#[allow(
22545    dead_code,
22546    reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
22547)]
22548fn fallback_total_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
22549    fallback_total_capacity_dedup(disks)
22550}
22551
22552#[deprecated(since = "0.1.0", note = "Use fallback_free_capacity_dedup instead")]
22553#[allow(
22554    dead_code,
22555    reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
22556)]
22557fn fallback_free_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
22558    fallback_free_capacity_dedup(disks)
22559}
22560
22561pub fn get_total_usable_capacity(disks: &[rustfs_madmin::Disk], info: &rustfs_madmin::StorageInfo) -> usize {
22562    // If backend info is missing or inconsistent, do a safe fallback to avoid reporting nonsense.
22563    if info.backend.standard_sc_data.is_empty() {
22564        return fallback_total_capacity_dedup(disks);
22565    }
22566    let mut capacity = 0usize;
22567    let mut matched_any = false;
22568    let mut counted_disks: HashSet<String> = HashSet::new();
22569
22570    for disk in disks.iter() {
22571        if disk.pool_index < 0 {
22572            continue;
22573        }
22574        let pool_idx = disk.pool_index as usize;
22575        if info.backend.standard_sc_data.len() <= pool_idx {
22576            continue;
22577        }
22578
22579        let usable_disks_per_set = info.backend.standard_sc_data[pool_idx];
22580        if usable_disks_per_set == 0 {
22581            continue;
22582        }
22583
22584        if (disk.disk_index as usize) < usable_disks_per_set {
22585            // 🔧 Generate a unique identity using a combination of fields
22586            let disk_key = format!(
22587                "{}|{}|p{}s{}d{}",
22588                disk.endpoint,   // Node address
22589                disk.drive_path, // mount path
22590                disk.pool_index, // Pool index
22591                disk.set_index,  // Collection index
22592                disk.disk_index  // Disk index
22593            );
22594            debug!("get_total_usable_capacity disk_key: {}", disk_key);
22595            // 🔧 Only disks that have not been counted are counted towards capacity
22596            if counted_disks.insert(disk_key) {
22597                matched_any = true;
22598                capacity += disk.total_space as usize;
22599            } else {
22600                // Log duplicate disks: this likely indicates a configuration issue and should always be visible.
22601                warn!(
22602                    "Duplicate disk detected in capacity calculation: {} at {}",
22603                    disk.endpoint, disk.drive_path
22604                );
22605            }
22606        }
22607    }
22608
22609    if matched_any {
22610        capacity
22611    } else {
22612        // Even if standard_sc_data exists, it might not match disk indexes due to upstream bugs.
22613        // Fallback to summing all online disks to prevent under-reporting.
22614        fallback_total_capacity_dedup(disks)
22615    }
22616}
22617
22618pub fn get_total_usable_capacity_free(disks: &[rustfs_madmin::Disk], info: &rustfs_madmin::StorageInfo) -> usize {
22619    if info.backend.standard_sc_data.is_empty() {
22620        return fallback_free_capacity_dedup(disks);
22621    }
22622
22623    let mut capacity = 0usize;
22624    let mut matched_any = false;
22625    let mut counted_disks: HashSet<String> = HashSet::new();
22626
22627    for disk in disks.iter() {
22628        if disk.pool_index < 0 {
22629            continue;
22630        }
22631        let pool_idx = disk.pool_index as usize;
22632        if info.backend.standard_sc_data.len() <= pool_idx {
22633            continue;
22634        }
22635
22636        let usable_disks_per_set = info.backend.standard_sc_data[pool_idx];
22637        if usable_disks_per_set == 0 {
22638            continue;
22639        }
22640
22641        if (disk.disk_index as usize) < usable_disks_per_set {
22642            let disk_key = format!(
22643                "{}|{}|p{}s{}d{}",
22644                disk.endpoint, disk.drive_path, disk.pool_index, disk.set_index, disk.disk_index
22645            );
22646
22647            if counted_disks.insert(disk_key) {
22648                matched_any = true;
22649                capacity += disk.available_space as usize;
22650            }
22651        }
22652    }
22653
22654    if matched_any {
22655        capacity
22656    } else {
22657        fallback_free_capacity_dedup(disks)
22658    }
22659}
22660
22661/// Total fallback capacity calculation with deweight
22662///
22663/// Replace original function: fallback_total_capacity()
22664pub(crate) fn fallback_total_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usize {
22665    let mut counted_disks: HashSet<String> = HashSet::new();
22666    let mut total = 0usize;
22667
22668    for disk in disks.iter() {
22669        // Only online disks are counted
22670        if !is_disk_online_state(&disk.state) {
22671            continue;
22672        }
22673
22674        // Use endpoint + drive_path as a unique identifier
22675        let disk_key = format!("{}|{}", disk.endpoint, disk.drive_path);
22676
22677        // Capacity is counted only when the disk is encountered for the first time
22678        if counted_disks.insert(disk_key) {
22679            total += disk.total_space as usize;
22680        }
22681    }
22682
22683    total
22684}
22685
22686/// Remove the heavy fallback idle capacity calculation
22687///
22688/// Replace original function: fallback_free_capacity()
22689pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usize {
22690    let mut counted_disks: HashSet<String> = HashSet::new();
22691    let mut total = 0usize;
22692
22693    for disk in disks.iter() {
22694        if !is_disk_online_state(&disk.state) {
22695            continue;
22696        }
22697
22698        let disk_key = format!("{}|{}", disk.endpoint, disk.drive_path);
22699
22700        if counted_disks.insert(disk_key) {
22701            total += disk.available_space as usize;
22702        }
22703    }
22704
22705    total
22706}
22707
22708#[cfg(test)]
22709mod pools_tests {
22710    use super::DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
22711    use super::persist_v3_pool_meta_for_test;
22712    use super::record_decommission_entry_error;
22713    use super::resolve_decommission_listing_error;
22714    use super::resolve_decommission_partial_listing_entry;
22715    use super::{
22716        DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, DECOMMISSION_CAPACITY_MODEL_VERSION, DECOMMISSION_CAPACITY_RELEASE_CANCELED,
22717        DECOMMISSION_CAPACITY_RELEASE_COMPLETED, DECOMMISSION_CAPACITY_RELEASE_FAILED, DECOMMISSION_CAPACITY_RESERVATION_TTL,
22718        DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
22719        DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP, DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP, DECOMMISSION_ENTRY_QUEUE_HARD_CAP,
22720        DECOMMISSION_META_PREFIXES, DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
22721        DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT, DecomBucketInfo, DecommissionCanceler, DecommissionCapacityTarget,
22722        DecommissionDurableIlmReceipt, DecommissionEntryEnqueueResult, DecommissionErasureLayout, DecommissionPoolCapacityInfo,
22723        DecommissionStartPoolState, DecommissionTargetConsumption, DecommissionTerminalState, DecommissionUnresolvedEntry,
22724        ListCallback, POOL_META_GENERATION_VERSION, POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_V1_VERSION,
22725        POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolMetaCasToken, PoolMetaPersistenceFence, PoolSpaceInfo, PoolStatus,
22726        QueuedDecommissionEntry, REBAL_META_NAME, acquire_pool_rebalance_activation_locks, active_decommission_source_indices,
22727        apply_decommission_status_space_info, await_decommission_worker, bind_decommission_cancelers,
22728        bind_missing_decommission_cancelers, build_decommission_capacity_reservation,
22729        build_decommission_capacity_reservation_with_model, cancel_decommission_canceler, clamp_decommission_entry_concurrency,
22730        classify_decommission_terminal_state, count_decommission_item, decommission_cancel_signal_result,
22731        decommission_durable_ilm_receipt_path, decommission_durable_ilm_receipt_run_prefix,
22732        decommission_durable_ilm_receipt_run_token, decommission_entry_queue_capacity, decommission_item_size,
22733        decommission_meta_bucket_options, decommission_physical_pool_capacity, decommission_retry_backoff_delay,
22734        decommission_start_pool_state, decommission_unresolved_listing_error, dedup_indices,
22735        default_decommission_bucket_concurrency, default_decommission_entry_concurrency, drain_decommission_entry_queue,
22736        enqueue_decommission_entry, ensure_decommission_cancel_allowed, ensure_decommission_capacity_reservations_available,
22737        ensure_decommission_clear_allowed, ensure_decommission_generation, ensure_decommission_listing_disks_available,
22738        ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool,
22739        ensure_decommission_start_local_leader, ensure_decommission_start_pool_states,
22740        ensure_decommission_start_rebalance_meta_allowed, ensure_decommission_start_target_capacity,
22741        ensure_decommission_terminal_operation_supported, ensure_decommission_unresolved_verification_disk_count,
22742        ensure_local_decommission_pool_leaders, ensure_pool_meta_write_fence, ensure_valid_decommission_pool_index, get_by_index,
22743        guard_decommission_cancelers, has_active_decommission_canceler, is_decommission_active, is_decommission_cancel_requested,
22744        load_decommission_entry_versions, local_decommission_queue_prefix, mark_decommission_bucket_done,
22745        merge_decommission_durable_ilm_receipts, merge_pool_meta_updates_for_save, merge_pool_status_refresh,
22746        missing_decommission_worker_prefix, next_decommission_capacity_generation, observe_decommission_terminal_reload_result,
22747        parse_decommission_durable_ilm_receipt_path, pool_meta_has_active_decommission, publish_pool_meta_updates,
22748        read_pool_meta_replica, reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion,
22749        record_decommission_unresolved_entry, recover_decommission_capacity_reservations,
22750        renew_decommission_capacity_reservation, require_decommission_store, reserve_decommission_start_cancelers,
22751        reserve_decommission_start_target_capacity, resolve_decommission_bucket_state,
22752        resolve_decommission_check_after_list_result, resolve_decommission_entry_cleanup_delete_result,
22753        resolve_decommission_entry_exact_versions, resolve_decommission_entry_reload_result,
22754        resolve_decommission_listing_worker_result, resolve_decommission_optional_bucket_config_result,
22755        resolve_decommission_pool_meta_reload_result, resolve_decommission_preflight_heal_result,
22756        resolve_decommission_progress_save_result, resolve_decommission_terminal_mark_after_error_result,
22757        resolve_decommission_terminal_mark_result, resolve_decommission_update_after_result,
22758        resolve_start_decommission_pool_meta_reload_result, resumable_decommission_queue_indices,
22759        rollback_start_decommission_pool_meta, run_decommission_buckets_bounded, run_decommission_listing_with_retry,
22760        run_decommission_listing_with_retry_and_drain, run_decommission_phases, run_decommission_side_effect,
22761        save_pool_meta_object_cas, select_decommission_capacity_model, should_cleanup_decommission_source_entry,
22762        should_continue_decommission_queue, should_count_decommission_version_complete,
22763        should_fail_decommission_pool_after_exhausted_source_changed, should_preserve_decommission_canceled_state,
22764        should_reject_decommission_cancel_as_terminal, should_retry_decommission_cancel_reload,
22765        should_retry_decommission_listing, should_skip_canceled_decommission_routine, spawn_decommission_index_cancelers,
22766        split_decommission_buckets, take_and_cancel_decommission_canceler, take_decommission_canceler,
22767        track_decommission_current_object, track_decommission_current_object_stage, update_decommission_for_operation,
22768        validate_start_decommission_request, wait_decommission_retry_backoff, wait_decommission_worker_drain,
22769        with_decommission_entry_context,
22770    };
22771    use super::{
22772        DecommissionCapacityAdmission, DecommissionCapacityOwner, DecommissionCapacityReleaseProof,
22773        DecommissionCapacityReservation, DecommissionCapacityTemporaryMutation, decommission_capacity_mutation_id,
22774        ensure_decommission_target_owner_admission, ensure_exact_delete_capacity_namespace_fences,
22775        ensure_external_decommission_target_admission, is_decommission_capacity_blocked_error,
22776        plan_exact_delete_capacity_reconciliations, record_decommission_target_consumption, release_decommission_target_inflight,
22777        reserve_decommission_target_pending, resolve_decommission_target_pending,
22778        set_decommission_capacity_info_overrides_for_test,
22779    };
22780    use crate::bucket::lifecycle::{
22781        DurableIlmRecordCheckpoint,
22782        bucket_lifecycle_ops::{ManualTransitionQueueSnapshot, ManualTransitionRunOptions},
22783        manual_transition_job::{ManualTransitionJobRecord, manual_transition_job_record_object_name},
22784        validate_durable_ilm_record,
22785    };
22786    use crate::bucket::metadata_sys;
22787    use crate::data_movement;
22788    use crate::disk::{STORAGE_FORMAT_FILE, endpoint::Endpoint};
22789    use crate::error::{Error, StorageError};
22790    use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
22791    use crate::object_api::{ObjectInfo, ObjectOptions};
22792    use crate::runtime::instance::InstanceContext;
22793    use crate::services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats};
22794    use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
22795    use crate::storage_api_contracts::object::HTTPPreconditions;
22796    use crate::storage_api_contracts::{object::ObjectIO, range::HTTPRangeSpec};
22797    use crate::store::ECStore;
22798    use byteorder::{ByteOrder, LittleEndian};
22799    use rmp_serde::Serializer;
22800    use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntry, ObjectPartInfo};
22801    use rustfs_filemeta::{MetaCacheEntries, MetadataResolutionParams};
22802    use rustfs_lock::{GlobalLockManager, LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
22803    use rustfs_rio::Index;
22804    use serde::Serialize;
22805    use std::future::Future;
22806    use std::io::Cursor;
22807    use std::sync::{
22808        Arc, Mutex as StdMutex,
22809        atomic::{AtomicBool, AtomicUsize, Ordering},
22810    };
22811    use std::task::{Context, Poll};
22812    use std::time::Duration as StdDuration;
22813    use time::{Duration, OffsetDateTime};
22814    use tokio::io::AsyncReadExt;
22815    use tokio::sync::Semaphore;
22816    use tokio_util::sync::CancellationToken;
22817
22818    #[derive(Debug)]
22819    struct ActivationLockRecorder {
22820        lock_manager: Arc<rustfs_lock::GlobalLockManager>,
22821        owner: &'static str,
22822        resources: StdMutex<Vec<String>>,
22823    }
22824
22825    #[async_trait::async_trait]
22826    impl crate::storage_api_contracts::namespace::NamespaceLocking for ActivationLockRecorder {
22827        type Error = Error;
22828        type NamespaceLock = rustfs_lock::NamespaceLockWrapper;
22829
22830        async fn new_ns_lock(&self, bucket: &str, object: &str) -> crate::error::Result<Self::NamespaceLock> {
22831            self.resources
22832                .lock()
22833                .expect("activation lock recorder should not be poisoned")
22834                .push(object.to_string());
22835            Ok(rustfs_lock::NamespaceLockWrapper::new(
22836                rustfs_lock::NamespaceLock::with_local_manager(
22837                    "activation-lock-test".to_string(),
22838                    Arc::clone(&self.lock_manager),
22839                ),
22840                rustfs_lock::ObjectKey::new(bucket, object),
22841                self.owner.to_string(),
22842            ))
22843        }
22844    }
22845
22846    fn noop_decommission_list_callback() -> ListCallback {
22847        Arc::new(|_| Box::pin(async {}))
22848    }
22849
22850    fn decommission_worker_test_store(pool_meta: PoolMeta, cancelers: Vec<Option<DecommissionCanceler>>) -> Arc<ECStore> {
22851        let ctx = Arc::new(InstanceContext::new());
22852        let endpoint_pools = EndpointServerPools::default();
22853        Arc::new(ECStore {
22854            id: uuid::Uuid::new_v4(),
22855            disk_map: std::collections::HashMap::new(),
22856            pools: Vec::new(),
22857            peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
22858            pool_meta: tokio::sync::RwLock::new(pool_meta),
22859            rebalance_meta: tokio::sync::RwLock::new(None),
22860            decommission_cancelers: tokio::sync::RwLock::new(cancelers),
22861            start_gate: tokio::sync::Mutex::new(()),
22862            pool_meta_save_gate: tokio::sync::Mutex::new(super::PoolMetaWriteState::for_test_bootstrap()),
22863            ctx,
22864            bucket_fence_registry: Arc::default(),
22865        })
22866    }
22867
22868    #[derive(Debug, Default)]
22869    struct PartialPoolMetaWriteStorage {
22870        fail_write: bool,
22871        fail_after_first_write: bool,
22872        pending_write: bool,
22873        write_started: tokio::sync::Notify,
22874        wrote: AtomicBool,
22875        revision: AtomicUsize,
22876        stored: StdMutex<Option<(Vec<u8>, String)>>,
22877        identity: StdMutex<Option<(Vec<u8>, String)>>,
22878    }
22879
22880    #[tokio::test]
22881    async fn pool_meta_recovery_reconciles_prepare_and_partial_commit_without_format_downgrade() {
22882        for previous_version in [POOL_META_V1_VERSION, POOL_META_VERSION, POOL_META_GENERATION_VERSION] {
22883            for commit_first in [false, true] {
22884                let cluster_id = uuid::Uuid::new_v4();
22885                let previous_meta = PoolMeta {
22886                    version: previous_version,
22887                    pools: vec![decommission_test_pool_status(0, None)],
22888                    ..Default::default()
22889                };
22890                let previous_revision = if previous_version == POOL_META_GENERATION_VERSION {
22891                    super::PoolMetaRevision {
22892                        version: previous_version,
22893                        cluster_id: Some(cluster_id),
22894                        epoch: 1,
22895                        generation: 9,
22896                        transaction_id: Some(uuid::Uuid::new_v4()),
22897                    }
22898                } else {
22899                    super::PoolMetaRevision::legacy(previous_version)
22900                };
22901                let previous_data = if previous_revision.is_generation_protocol() {
22902                    super::encode_pool_meta_v3_envelope(&previous_meta, previous_revision, true, None).unwrap()
22903                } else {
22904                    previous_meta.encode_config_data_for_v2_gate(true).unwrap()
22905                };
22906                let previous = super::PoolMetaCommittedCandidate {
22907                    canonical: previous_data,
22908                    meta: previous_meta.clone(),
22909                    revision: previous_revision,
22910                };
22911                let mut next = previous_meta.clone();
22912                next.version = POOL_META_GENERATION_VERSION;
22913                next.pools[0].last_update += Duration::seconds(1);
22914                let revision = super::PoolMetaRevision {
22915                    version: POOL_META_GENERATION_VERSION,
22916                    cluster_id: Some(cluster_id),
22917                    epoch: 1,
22918                    generation: previous_revision.generation + 1,
22919                    transaction_id: Some(uuid::Uuid::new_v4()),
22920                };
22921                let pending = super::encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous)).unwrap();
22922                let durable = super::encode_pool_meta_v3_envelope(&next, revision, true, None).unwrap();
22923                let identity = super::initialized_pool_meta_identity_for_test(cluster_id, 1).unwrap();
22924                let pools = [if commit_first { durable } else { pending.clone() }, pending]
22925                    .into_iter()
22926                    .map(|data| {
22927                        Arc::new(PartialPoolMetaWriteStorage {
22928                            stored: StdMutex::new(Some((data, "initial".to_owned()))),
22929                            identity: StdMutex::new(Some((identity.clone(), "identity".to_owned()))),
22930                            ..Default::default()
22931                        })
22932                    })
22933                    .collect::<Vec<_>>();
22934                let mut state = super::PoolMetaWriteState::for_startup(cluster_id, false);
22935                let selection = super::load_pool_meta_for_transaction_recovery(pools.clone(), &mut state)
22936                    .await
22937                    .unwrap();
22938                assert_eq!(
22939                    selection.meta.pools[0].last_update,
22940                    if commit_first {
22941                        next.pools[0].last_update
22942                    } else {
22943                        previous_meta.pools[0].last_update
22944                    }
22945                );
22946                let outcome = super::repair_pool_meta_transaction(
22947                    pools.clone(),
22948                    &mut state,
22949                    selection,
22950                    &PoolMetaPersistenceFence::Distributed(None),
22951                )
22952                .await
22953                .unwrap();
22954                assert_eq!(outcome.committed.version, POOL_META_GENERATION_VERSION);
22955                assert_eq!(
22956                    outcome.committed.pools[0].last_update,
22957                    if commit_first {
22958                        next.pools[0].last_update
22959                    } else {
22960                        previous_meta.pools[0].last_update
22961                    }
22962                );
22963                outcome.disarm();
22964                let confirmed = super::load_pool_meta_for_transaction_recovery(pools, &mut state)
22965                    .await
22966                    .unwrap();
22967                assert!(!confirmed.replica_state.needs_repair);
22968                state.ensure_write_safe("repaired").unwrap();
22969            }
22970        }
22971    }
22972
22973    #[tokio::test]
22974    async fn pool_meta_recovery_rejects_missing_corrupt_and_incompatible_replicas_without_writes() {
22975        let cluster_id = uuid::Uuid::new_v4();
22976        let identity = super::initialized_pool_meta_identity_for_test(cluster_id, 1).unwrap();
22977        for payload in [None, Some(Vec::new()), Some(vec![1, 0, 255, 255, 0])] {
22978            let pool = Arc::new(PartialPoolMetaWriteStorage {
22979                stored: StdMutex::new(payload.map(|data| (data, "initial".to_owned()))),
22980                identity: StdMutex::new(Some((identity.clone(), "identity".to_owned()))),
22981                ..Default::default()
22982            });
22983            let mut state = super::PoolMetaWriteState::for_startup(cluster_id, false);
22984            super::load_pool_meta_for_transaction_recovery(vec![pool.clone()], &mut state)
22985                .await
22986                .unwrap_err();
22987            assert!(!pool.wrote.load(Ordering::SeqCst));
22988        }
22989    }
22990
22991    #[tokio::test]
22992    async fn pool_meta_first_cas_rejection_is_retryable_but_later_rejection_keeps_prior_write_armed() {
22993        let pool = Arc::new(PartialPoolMetaWriteStorage::default());
22994        let state = super::PoolMetaWriteState::default();
22995        let fence = PoolMetaPersistenceFence::Distributed(None);
22996        let mut arm = state.arm_transaction();
22997        save_pool_meta_object_cas(
22998            pool.clone(),
22999            POOL_META_NAME,
23000            vec![1],
23001            &PoolMetaCasToken::Existing("missing".to_owned()),
23002            &fence,
23003            "prepare_cas",
23004            &mut arm,
23005        )
23006        .await
23007        .unwrap_err();
23008        drop(arm);
23009        state.ensure_write_safe("first rejection").unwrap();
23010        let mut arm = state.arm_transaction();
23011        save_pool_meta_object_cas(
23012            pool.clone(),
23013            POOL_META_NAME,
23014            vec![1],
23015            &PoolMetaCasToken::Missing,
23016            &fence,
23017            "prepare_cas",
23018            &mut arm,
23019        )
23020        .await
23021        .unwrap();
23022        save_pool_meta_object_cas(pool, POOL_META_NAME, vec![2], &PoolMetaCasToken::Missing, &fence, "commit_cas", &mut arm)
23023            .await
23024            .unwrap_err();
23025        drop(arm);
23026        let err = state.ensure_write_safe("later rejection").unwrap_err();
23027        assert_eq!(err.pool_metadata_failure().unwrap().phase, "prepare_cas");
23028        assert!(matches!(
23029            err.pool_metadata_failure().unwrap().source.as_deref(),
23030            Some(Error::PreconditionFailed)
23031        ));
23032    }
23033
23034    #[tokio::test]
23035    async fn pool_meta_recovery_cas_conflict_after_repair_write_keeps_transaction_blocked() {
23036        let base = PoolMeta {
23037            version: POOL_META_VERSION,
23038            pools: vec![decommission_test_pool_status(0, None)],
23039            ..Default::default()
23040        };
23041        let mut current = base.clone();
23042        current.pools[0].last_update += Duration::seconds(1);
23043        let first = Arc::new(PartialPoolMetaWriteStorage {
23044            stored: StdMutex::new(Some((current.encode_config_data_for_v2_gate(true).unwrap(), "first".to_owned()))),
23045            ..Default::default()
23046        });
23047        let second = Arc::new(PartialPoolMetaWriteStorage {
23048            stored: StdMutex::new(Some((base.encode_config_data_for_v2_gate(true).unwrap(), "second".to_owned()))),
23049            ..Default::default()
23050        });
23051        let mut state = super::PoolMetaWriteState::default();
23052        let selection = super::load_pool_meta_for_transaction_recovery(vec![first.clone(), second.clone()], &mut state)
23053            .await
23054            .unwrap();
23055        let raced = base.encode_config_data_for_v2_gate(true).unwrap();
23056        *second.stored.lock().unwrap() = Some((raced.clone(), "raced".to_owned()));
23057        let result = super::repair_pool_meta_transaction(
23058            vec![first.clone(), second.clone()],
23059            &mut state,
23060            selection,
23061            &PoolMetaPersistenceFence::Distributed(None),
23062        )
23063        .await;
23064        assert!(matches!(result, Err(Error::PreconditionFailed)));
23065        assert!(first.wrote.load(Ordering::SeqCst));
23066        assert!(!second.wrote.load(Ordering::SeqCst));
23067        assert_eq!(second.stored.lock().unwrap().as_ref().unwrap().0, raced);
23068        let err = state.ensure_write_safe("recovery retry").unwrap_err();
23069        assert_eq!(err.pool_metadata_failure().unwrap().phase, "recovery_cas");
23070    }
23071
23072    #[tokio::test]
23073    async fn pool_meta_cas_failure_retains_the_original_source_for_caller_and_latch() {
23074        let pool = Arc::new(PartialPoolMetaWriteStorage {
23075            fail_write: true,
23076            ..Default::default()
23077        });
23078        let state = super::PoolMetaWriteState::default();
23079        let mut arm = state.arm_transaction();
23080        let err = save_pool_meta_object_cas(
23081            pool,
23082            POOL_META_NAME,
23083            Vec::new(),
23084            &PoolMetaCasToken::Missing,
23085            &PoolMetaPersistenceFence::Distributed(None),
23086            "commit_cas",
23087            &mut arm,
23088        )
23089        .await
23090        .unwrap_err();
23091        let source = err.pool_metadata_failure().unwrap().source.as_ref().unwrap();
23092        assert!(matches!(source.as_ref(), Error::Timeout));
23093        assert!(Arc::ptr_eq(source, arm.source.as_ref().unwrap()));
23094        drop(arm);
23095        let blocked = state.ensure_write_safe("after CAS failure").unwrap_err();
23096        assert!(Arc::ptr_eq(source, blocked.pool_metadata_failure().unwrap().source.as_ref().unwrap()));
23097    }
23098
23099    #[tokio::test]
23100    async fn pool_meta_cancelled_identity_dispatch_is_armed_before_pool_bin_writes() {
23101        let pool = Arc::new(PartialPoolMetaWriteStorage {
23102            pending_write: true,
23103            ..Default::default()
23104        });
23105        let state = Arc::new(tokio::sync::Mutex::new(super::PoolMetaWriteState::for_startup(
23106            uuid::Uuid::new_v4(),
23107            true,
23108        )));
23109        let task = tokio::spawn({
23110            let pool = pool.clone();
23111            let state = state.clone();
23112            async move {
23113                let mut state = state.lock().await;
23114                super::persist_pool_meta_identity_for_startup(vec![pool], &mut state, false).await
23115            }
23116        });
23117        pool.write_started.notified().await;
23118        task.abort();
23119        assert!(task.await.unwrap_err().is_cancelled());
23120        let state = state.lock().await;
23121        let err = state.ensure_write_safe("identity retry").unwrap_err();
23122        assert_eq!(err.pool_metadata_failure().unwrap().phase, "identity_cas");
23123        assert_eq!(
23124            err.pool_metadata_failure().unwrap().kind,
23125            crate::error::PoolMetadataFailure::TransactionUnknown
23126        );
23127        assert!(pool.stored.lock().unwrap().is_none());
23128    }
23129
23130    #[async_trait::async_trait]
23131    impl ObjectIO for PartialPoolMetaWriteStorage {
23132        type Error = Error;
23133        type RangeSpec = HTTPRangeSpec;
23134        type HeaderMap = http::HeaderMap;
23135        type ObjectOptions = crate::object_api::ObjectOptions;
23136        type ObjectInfo = crate::object_api::ObjectInfo;
23137        type GetObjectReader = crate::object_api::GetObjectReader;
23138        type PutObjectReader = crate::object_api::PutObjReader;
23139
23140        async fn get_object_reader(
23141            &self,
23142            bucket: &str,
23143            object: &str,
23144            _range: Option<Self::RangeSpec>,
23145            _h: Self::HeaderMap,
23146            _opts: &Self::ObjectOptions,
23147        ) -> std::result::Result<Self::GetObjectReader, Error> {
23148            let stored = if object == POOL_META_IDENTITY_NAME {
23149                &self.identity
23150            } else {
23151                &self.stored
23152            };
23153            let Some((data, etag)) = stored
23154                .lock()
23155                .expect("pool metadata test storage should not be poisoned")
23156                .clone()
23157            else {
23158                return Err(Error::FileNotFound);
23159            };
23160            Ok(crate::object_api::GetObjectReader {
23161                stream: Box::new(Cursor::new(data.clone())),
23162                object_info: crate::object_api::ObjectInfo {
23163                    bucket: bucket.to_string(),
23164                    name: object.to_string(),
23165                    size: data.len() as i64,
23166                    etag: Some(etag),
23167                    ..Default::default()
23168                },
23169                buffered_body: None,
23170                body_source: Default::default(),
23171            })
23172        }
23173
23174        async fn put_object(
23175            &self,
23176            _bucket: &str,
23177            object: &str,
23178            data: &mut Self::PutObjectReader,
23179            opts: &Self::ObjectOptions,
23180        ) -> std::result::Result<Self::ObjectInfo, Error> {
23181            self.write_started.notify_one();
23182            if self.pending_write {
23183                std::future::pending().await
23184            }
23185            if object == POOL_META_NAME
23186                && (self.fail_write || (self.fail_after_first_write && self.revision.load(Ordering::SeqCst) > 0))
23187            {
23188                return Err(Error::Timeout);
23189            }
23190            let stored = if object == POOL_META_IDENTITY_NAME {
23191                &self.identity
23192            } else {
23193                &self.stored
23194            };
23195            let current_etag = stored
23196                .lock()
23197                .expect("pool metadata test storage should not be poisoned")
23198                .as_ref()
23199                .map(|(_, etag)| etag.clone());
23200            if opts
23201                .http_preconditions
23202                .as_ref()
23203                .and_then(HTTPPreconditions::if_none_match_value)
23204                == Some("*")
23205                && current_etag.is_some()
23206            {
23207                return Err(Error::PreconditionFailed);
23208            }
23209            if let Some(expected) = opts.http_preconditions.as_ref().and_then(HTTPPreconditions::if_match_value)
23210                && current_etag.as_deref() != Some(expected)
23211            {
23212                return Err(Error::PreconditionFailed);
23213            }
23214            let mut payload = Vec::new();
23215            data.stream.read_to_end(&mut payload).await?;
23216            let etag = format!("pool-meta-test-{}", self.revision.fetch_add(1, Ordering::SeqCst) + 1);
23217            *stored.lock().expect("pool metadata test storage should not be poisoned") = Some((payload, etag.clone()));
23218            self.wrote.store(true, Ordering::SeqCst);
23219            Ok(crate::object_api::ObjectInfo {
23220                etag: Some(etag),
23221                ..Default::default()
23222            })
23223        }
23224    }
23225
23226    mod phase_tests {
23227        use super::super as implementation;
23228        use super::*;
23229
23230        #[derive(Clone, Copy, Debug, Default)]
23231        enum WriteFault {
23232            #[default]
23233            None,
23234            Reject,
23235            Timeout,
23236            TimeoutAfterWrite,
23237            EmptyEtag,
23238            OmitWrite,
23239        }
23240
23241        #[derive(Clone, Debug, Default)]
23242        struct WriteStep {
23243            gate: Option<Arc<tokio::sync::Semaphore>>,
23244            fault: WriteFault,
23245        }
23246
23247        #[derive(Clone, Debug)]
23248        struct WriteCall {
23249            pool: usize,
23250            phase: usize,
23251            if_match: Option<String>,
23252        }
23253
23254        #[derive(Debug, Default)]
23255        struct WriteTrace {
23256            calls: StdMutex<Vec<WriteCall>>,
23257            active: AtomicUsize,
23258            maximum: AtomicUsize,
23259            finished: AtomicUsize,
23260            changed: tokio::sync::Notify,
23261        }
23262
23263        impl WriteTrace {
23264            async fn wait_for(&self, calls: usize, finished: usize) {
23265                tokio::time::timeout(StdDuration::from_secs(3), async {
23266                    loop {
23267                        let changed = self.changed.notified();
23268                        if self.calls.lock().expect("trace lock").len() >= calls
23269                            && self.finished.load(Ordering::SeqCst) >= finished
23270                        {
23271                            return;
23272                        }
23273                        changed.await;
23274                    }
23275                })
23276                .await
23277                .expect("phase must reach the requested barrier");
23278            }
23279
23280            fn commit_count(&self) -> usize {
23281                self.calls
23282                    .lock()
23283                    .expect("trace lock")
23284                    .iter()
23285                    .filter(|call| call.phase == 1)
23286                    .count()
23287            }
23288        }
23289
23290        struct ActiveWrite<'a>(&'a WriteTrace);
23291        impl Drop for ActiveWrite<'_> {
23292            fn drop(&mut self) {
23293                self.0.active.fetch_sub(1, Ordering::SeqCst);
23294                self.0.finished.fetch_add(1, Ordering::SeqCst);
23295                self.0.changed.notify_one();
23296            }
23297        }
23298
23299        #[derive(Debug)]
23300        struct PhaseStorage {
23301            inner: PartialPoolMetaWriteStorage,
23302            pool: usize,
23303            steps: [WriteStep; 2],
23304            trace: Arc<WriteTrace>,
23305        }
23306
23307        #[async_trait::async_trait]
23308        impl ObjectIO for PhaseStorage {
23309            type Error = Error;
23310            type RangeSpec = HTTPRangeSpec;
23311            type HeaderMap = http::HeaderMap;
23312            type ObjectOptions = crate::object_api::ObjectOptions;
23313            type ObjectInfo = crate::object_api::ObjectInfo;
23314            type GetObjectReader = crate::object_api::GetObjectReader;
23315            type PutObjectReader = crate::object_api::PutObjReader;
23316
23317            async fn get_object_reader(
23318                &self,
23319                bucket: &str,
23320                object: &str,
23321                range: Option<Self::RangeSpec>,
23322                headers: Self::HeaderMap,
23323                opts: &Self::ObjectOptions,
23324            ) -> Result<Self::GetObjectReader, Error> {
23325                self.inner.get_object_reader(bucket, object, range, headers, opts).await
23326            }
23327
23328            async fn put_object(
23329                &self,
23330                bucket: &str,
23331                object: &str,
23332                data: &mut Self::PutObjectReader,
23333                opts: &Self::ObjectOptions,
23334            ) -> Result<Self::ObjectInfo, Error> {
23335                if object != POOL_META_NAME {
23336                    return self.inner.put_object(bucket, object, data, opts).await;
23337                }
23338                let mut payload = Vec::new();
23339                data.stream.read_to_end(&mut payload).await?;
23340                let phase = match implementation::decode_pool_meta_replica(payload.clone()) {
23341                    implementation::PoolMetaReplica::Valid { committed, .. } => usize::from(committed),
23342                    _ => panic!("phase fixture must receive valid pool metadata"),
23343                };
23344                assert!(opts.max_parity && opts.no_lock);
23345                assert_eq!(opts.write_completion, crate::object_api::WriteCompletion::TailDrained);
23346                self.trace.calls.lock().expect("trace lock").push(WriteCall {
23347                    pool: self.pool,
23348                    phase,
23349                    if_match: opts
23350                        .http_preconditions
23351                        .as_ref()
23352                        .and_then(HTTPPreconditions::if_match_value)
23353                        .map(str::to_owned),
23354                });
23355                let active = self.trace.active.fetch_add(1, Ordering::SeqCst) + 1;
23356                self.trace.maximum.fetch_max(active, Ordering::SeqCst);
23357                let _active = ActiveWrite(&self.trace);
23358                self.trace.changed.notify_one();
23359                let step = &self.steps[phase];
23360                if let Some(gate) = &step.gate {
23361                    gate.acquire().await.expect("phase gate stays open").forget();
23362                }
23363                match step.fault {
23364                    WriteFault::Reject => return Err(Error::PreconditionFailed),
23365                    WriteFault::Timeout => return Err(Error::Timeout),
23366                    WriteFault::OmitWrite => {
23367                        return Ok(crate::object_api::ObjectInfo {
23368                            etag: Some("uncommitted".to_owned()),
23369                            ..Default::default()
23370                        });
23371                    }
23372                    _ => {}
23373                }
23374                let mut data = crate::object_api::PutObjReader::from_vec(payload);
23375                let mut result = self.inner.put_object(bucket, object, &mut data, opts).await?;
23376                match step.fault {
23377                    WriteFault::TimeoutAfterWrite => return Err(Error::Timeout),
23378                    WriteFault::EmptyEtag => result.etag = Some("  ".to_owned()),
23379                    _ => {}
23380                }
23381                Ok(result)
23382            }
23383        }
23384
23385        struct Fixture {
23386            pools: Vec<Arc<PhaseStorage>>,
23387            trace: Arc<WriteTrace>,
23388            previous: implementation::PoolMetaCommittedCandidate,
23389            desired: PoolMeta,
23390            pending: Vec<u8>,
23391            durable: Vec<u8>,
23392            revision: implementation::PoolMetaRevision,
23393            cluster_id: uuid::Uuid,
23394        }
23395
23396        impl Fixture {
23397            fn new(steps: Vec<[WriteStep; 2]>) -> Self {
23398                let cluster_id = uuid::Uuid::new_v4();
23399                let meta = PoolMeta {
23400                    version: POOL_META_GENERATION_VERSION,
23401                    pools: (0..steps.len())
23402                        .map(|index| {
23403                            decommission_test_pool_status(
23404                                index,
23405                                (index == 0).then(|| PoolDecommissionInfo {
23406                                    start_time: Some(OffsetDateTime::UNIX_EPOCH),
23407                                    start_size: 4096,
23408                                    total_size: 16384,
23409                                    current_size: 8192,
23410                                    items_decommissioned: 7,
23411                                    items_decommission_failed: 1,
23412                                    bytes_done: 4096,
23413                                    bytes_failed: 512,
23414                                    queued_buckets: vec!["remaining-a".to_owned(), "remaining-b".to_owned()],
23415                                    decommissioned_buckets: vec!["completed".to_owned()],
23416                                    bucket: "remaining-a".to_owned(),
23417                                    prefix: "objects/".to_owned(),
23418                                    object: "objects/007".to_owned(),
23419                                    ..Default::default()
23420                                }),
23421                            )
23422                        })
23423                        .collect(),
23424                    ..Default::default()
23425                };
23426                let previous_revision = implementation::PoolMetaRevision {
23427                    version: POOL_META_GENERATION_VERSION,
23428                    cluster_id: Some(cluster_id),
23429                    epoch: 1,
23430                    generation: 7,
23431                    transaction_id: Some(uuid::Uuid::new_v4()),
23432                };
23433                let previous = implementation::PoolMetaCommittedCandidate {
23434                    canonical: implementation::encode_pool_meta_v3_envelope(&meta, previous_revision, true, None)
23435                        .expect("committed predecessor"),
23436                    meta: meta.clone(),
23437                    revision: previous_revision,
23438                };
23439                let mut desired = meta;
23440                desired.pools[0].last_update += Duration::seconds(1);
23441                let progress = desired.pools[0].decommission.as_mut().expect("active decommission");
23442                progress.items_decommissioned += 1;
23443                progress.bytes_done += 512;
23444                progress.current_size += 512;
23445                progress.object = "objects/008".to_owned();
23446                let revision = implementation::PoolMetaRevision {
23447                    generation: 8,
23448                    transaction_id: Some(uuid::Uuid::new_v4()),
23449                    ..previous_revision
23450                };
23451                let pending = implementation::encode_pool_meta_v3_envelope(&desired, revision, false, Some(&previous))
23452                    .expect("pending candidate");
23453                let durable =
23454                    implementation::encode_pool_meta_v3_envelope(&desired, revision, true, None).expect("committed candidate");
23455                let identity = implementation::initialized_pool_meta_identity_for_test(cluster_id, 1).expect("identity");
23456                let trace = Arc::new(WriteTrace::default());
23457                let pools = steps
23458                    .into_iter()
23459                    .enumerate()
23460                    .map(|(index, steps)| {
23461                        Arc::new(PhaseStorage {
23462                            inner: PartialPoolMetaWriteStorage {
23463                                stored: StdMutex::new(Some((previous.canonical.clone(), format!("initial-{index}")))),
23464                                identity: StdMutex::new(Some((identity.clone(), "identity".to_owned()))),
23465                                revision: AtomicUsize::new(index * 10),
23466                                ..Default::default()
23467                            },
23468                            pool: index,
23469                            steps,
23470                            trace: trace.clone(),
23471                        })
23472                    })
23473                    .collect();
23474                Self {
23475                    pools,
23476                    trace,
23477                    previous,
23478                    desired,
23479                    pending,
23480                    durable,
23481                    revision,
23482                    cluster_id,
23483                }
23484            }
23485
23486            fn state(&self) -> implementation::PoolMetaWriteState {
23487                implementation::PoolMetaWriteState::for_startup(self.cluster_id, false)
23488            }
23489
23490            fn tokens(&self) -> Vec<PoolMetaCasToken> {
23491                self.pools
23492                    .iter()
23493                    .map(|pool| {
23494                        PoolMetaCasToken::Existing(
23495                            pool.inner
23496                                .stored
23497                                .lock()
23498                                .expect("fixture storage")
23499                                .as_ref()
23500                                .expect("existing replica")
23501                                .1
23502                                .clone(),
23503                        )
23504                    })
23505                    .collect()
23506            }
23507        }
23508
23509        fn steps(count: usize) -> Vec<[WriteStep; 2]> {
23510            (0..count).map(|_| [WriteStep::default(), WriteStep::default()]).collect()
23511        }
23512
23513        fn signal() -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
23514            Some(Arc::default())
23515        }
23516
23517        async fn prepare(
23518            fixture: &Fixture,
23519            fence: &PoolMetaPersistenceFence<'_>,
23520            arm: &mut implementation::PoolMetaTransactionArm,
23521        ) -> Vec<Result<crate::object_api::ObjectInfo, Error>> {
23522            implementation::save_pool_meta_phase(
23523                &fixture.pools,
23524                &fixture.pending,
23525                &fixture.tokens(),
23526                fence,
23527                "prepare_cas",
23528                arm,
23529                #[cfg(feature = "e2e-test-hooks")]
23530                None,
23531            )
23532            .await
23533        }
23534
23535        #[tokio::test]
23536        async fn pool_meta_v3_phase_concurrency_is_bounded() {
23537            let gate = Arc::new(tokio::sync::Semaphore::new(0));
23538            let mut config = steps(6);
23539            for step in &mut config {
23540                step[0].gate = Some(gate.clone());
23541            }
23542            let fixture = Fixture::new(config);
23543            let state = fixture.state();
23544            let mut arm = state.arm_transaction();
23545            let fence = PoolMetaPersistenceFence::Distributed(signal());
23546            let mut pending = Box::pin(prepare(&fixture, &fence, &mut arm));
23547            tokio::select! { _ = fixture.trace.wait_for(4, 0) => {}, _ = &mut pending => panic!("blocked writes finished") }
23548            assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 4);
23549            gate.add_permits(1);
23550            tokio::select! { _ = fixture.trace.wait_for(5, 1) => {}, _ = &mut pending => panic!("unreleased writes finished") }
23551            assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 5);
23552            gate.add_permits(5);
23553            assert!(pending.await.into_iter().all(|result| result.is_ok()));
23554            assert_eq!(fixture.trace.maximum.load(Ordering::SeqCst), 4);
23555            assert_eq!(fixture.trace.active.load(Ordering::SeqCst), 0);
23556            arm.disarm();
23557        }
23558
23559        #[tokio::test]
23560        async fn pool_meta_v3_commit_waits_for_all_prepare_etags() {
23561            let gate = Arc::new(tokio::sync::Semaphore::new(0));
23562            let mut config = steps(4);
23563            config[3][0].gate = Some(gate.clone());
23564            let fixture = Fixture::new(config);
23565            let mut state = fixture.state();
23566            let mut save = Box::pin(
23567                fixture
23568                    .desired
23569                    .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]),
23570            );
23571            tokio::select! { _ = fixture.trace.wait_for(4, 3) => {}, _ = &mut save => panic!("commit crossed the last prepare") }
23572            assert_eq!(fixture.trace.commit_count(), 0);
23573            gate.add_permits(1);
23574            save.await.expect("all prepares and commits succeed").disarm();
23575            let calls = fixture.trace.calls.lock().expect("trace");
23576            for call in calls.iter().filter(|call| call.phase == 1) {
23577                assert_eq!(call.if_match, Some(format!("pool-meta-test-{}", call.pool * 10 + 1)));
23578            }
23579            assert_eq!(calls.iter().filter(|call| call.phase == 1).count(), 4);
23580        }
23581
23582        #[tokio::test]
23583        async fn pool_meta_v3_empty_prepare_etag_never_starts_commit() {
23584            let mut config = steps(4);
23585            config[3][0].fault = WriteFault::EmptyEtag;
23586            let fixture = Fixture::new(config);
23587            let mut state = fixture.state();
23588            let err = fixture
23589                .desired
23590                .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0])
23591                .await
23592                .expect_err("a missing conditional-write revision must fail");
23593            assert!(err.to_string().contains("without a conditional-write revision"));
23594            assert_eq!(fixture.trace.finished.load(Ordering::SeqCst), 4);
23595            assert_eq!(fixture.trace.commit_count(), 0);
23596            assert!(state.ensure_write_safe("empty etag wrote pending data").is_err());
23597        }
23598
23599        #[tokio::test]
23600        async fn pool_meta_v3_prepare_failure_drains_late_writes() {
23601            let gate = Arc::new(tokio::sync::Semaphore::new(0));
23602            let mut config = steps(4);
23603            config[1][0].fault = WriteFault::Timeout;
23604            config[3][0].gate = Some(gate.clone());
23605            let fixture = Fixture::new(config);
23606            let mut state = fixture.state();
23607            let mut save = Box::pin(
23608                fixture
23609                    .desired
23610                    .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]),
23611            );
23612            tokio::select! { _ = fixture.trace.wait_for(4, 3) => {}, _ = &mut save => panic!("phase returned with a live replica") }
23613            assert_eq!(fixture.trace.commit_count(), 0);
23614            gate.add_permits(1);
23615            assert!(save.await.is_err());
23616            assert_eq!(fixture.trace.finished.load(Ordering::SeqCst), 4);
23617            assert!(fixture.pools[3].inner.wrote.load(Ordering::SeqCst));
23618            assert!(state.ensure_write_safe("ambiguous prepare").is_err());
23619        }
23620
23621        #[tokio::test]
23622        async fn pool_meta_v3_rejections_restore_only_an_unmutated_phase() {
23623            for previous in [None, Some("identity_cas")] {
23624                let mut config = steps(4);
23625                for step in &mut config {
23626                    step[0].fault = WriteFault::Reject;
23627                }
23628                let fixture = Fixture::new(config);
23629                let state = fixture.state();
23630                let mut arm = state.arm_transaction();
23631                arm.phase = previous;
23632                let results = prepare(&fixture, &PoolMetaPersistenceFence::Distributed(signal()), &mut arm).await;
23633                assert!(
23634                    results
23635                        .into_iter()
23636                        .all(|result| matches!(result, Err(Error::PreconditionFailed)))
23637                );
23638                assert_eq!(arm.phase, previous);
23639                drop(arm);
23640                assert_eq!(state.ensure_write_safe("all rejected").is_ok(), previous.is_none());
23641            }
23642            let mut config = steps(4);
23643            for step in &mut config[1..] {
23644                step[0].fault = WriteFault::Reject;
23645            }
23646            let fixture = Fixture::new(config);
23647            let state = fixture.state();
23648            let mut arm = state.arm_transaction();
23649            let results = prepare(&fixture, &PoolMetaPersistenceFence::Distributed(signal()), &mut arm).await;
23650            assert!(results[0].is_ok());
23651            assert_eq!(arm.phase, Some("prepare_cas"));
23652            drop(arm);
23653            assert!(state.ensure_write_safe("one sibling wrote").is_err());
23654        }
23655
23656        #[tokio::test]
23657        async fn pool_meta_v3_parallel_errors_keep_original_source_pointers() {
23658            let mut config = steps(4);
23659            config[0][0].fault = WriteFault::Timeout;
23660            config[1][0].fault = WriteFault::Timeout;
23661            config[3][0].fault = WriteFault::Reject;
23662            let fixture = Fixture::new(config);
23663            let state = fixture.state();
23664            let mut arm = state.arm_transaction();
23665            let results = prepare(&fixture, &PoolMetaPersistenceFence::Distributed(signal()), &mut arm).await;
23666            let err = results[0].as_ref().expect_err("first pool error stays first");
23667            let source = err
23668                .pool_metadata_failure()
23669                .expect("typed error")
23670                .source
23671                .as_ref()
23672                .expect("original source");
23673            assert!(matches!(source.as_ref(), Error::Timeout));
23674            assert!(Arc::ptr_eq(source, arm.source.as_ref().expect("arm source")));
23675            drop(arm);
23676            let blocked = state
23677                .ensure_write_safe("after parallel errors")
23678                .expect_err("recovery required");
23679            assert!(Arc::ptr_eq(
23680                source,
23681                blocked
23682                    .pool_metadata_failure()
23683                    .expect("typed latch")
23684                    .source
23685                    .as_ref()
23686                    .expect("latch source")
23687            ));
23688        }
23689
23690        #[tokio::test]
23691        async fn pool_meta_v3_retry_rejections_keep_the_previous_io_source() {
23692            let mut config = steps(4);
23693            config[0][0].fault = WriteFault::Reject;
23694            config[1][0].fault = WriteFault::Timeout;
23695            let first = Fixture::new(config);
23696            let state = first.state();
23697            let mut arm = state.arm_transaction();
23698            let fence = PoolMetaPersistenceFence::Distributed(signal());
23699            let results = prepare(&first, &fence, &mut arm).await;
23700            assert!(matches!(&results[0], Err(Error::PreconditionFailed)));
23701            let original = arm.source.as_ref().expect("first attempt I/O source").clone();
23702            assert!(matches!(original.as_ref(), Error::Timeout));
23703            let mut config = steps(4);
23704            for step in &mut config {
23705                step[0].fault = WriteFault::Reject;
23706            }
23707            let retry = Fixture::new(config);
23708            assert!(
23709                prepare(&retry, &fence, &mut arm)
23710                    .await
23711                    .into_iter()
23712                    .all(|result| matches!(result, Err(Error::PreconditionFailed)))
23713            );
23714            assert!(Arc::ptr_eq(&original, arm.source.as_ref().expect("source survives phase retry")));
23715            implementation::save_pool_meta_object_cas(
23716                retry.pools[0].clone(),
23717                POOL_META_NAME,
23718                retry.pending.clone(),
23719                &retry.tokens()[0],
23720                &fence,
23721                "prepare_cas",
23722                &mut arm,
23723            )
23724            .await
23725            .expect_err("serial retry rejected");
23726            assert!(Arc::ptr_eq(&original, arm.source.as_ref().expect("source survives serial retry")));
23727            drop(arm);
23728            let blocked = state
23729                .ensure_write_safe("cancelled after retry rejection")
23730                .expect_err("recovery required");
23731            assert!(Arc::ptr_eq(
23732                &original,
23733                blocked
23734                    .pool_metadata_failure()
23735                    .expect("typed latch")
23736                    .source
23737                    .as_ref()
23738                    .expect("source")
23739            ));
23740        }
23741
23742        #[tokio::test]
23743        async fn pool_meta_v3_cancelled_drain_does_not_replace_io_error_with_cas_rejection() {
23744            let gate = Arc::new(tokio::sync::Semaphore::new(0));
23745            let mut config = steps(4);
23746            config[0][0].gate = Some(gate.clone());
23747            config[2][0].gate = Some(gate);
23748            config[1][0].fault = WriteFault::Timeout;
23749            config[3][0].fault = WriteFault::Reject;
23750            let fixture = Fixture::new(config);
23751            let state = fixture.state();
23752            let mut arm = state.arm_transaction();
23753            let fence = PoolMetaPersistenceFence::Distributed(signal());
23754            let mut phase = Box::pin(prepare(&fixture, &fence, &mut arm));
23755            tokio::select! { _ = fixture.trace.wait_for(4, 2) => {}, _ = &mut phase => panic!("blocked phase finished") }
23756            assert!(futures::poll!(phase.as_mut()).is_pending(), "drain remains parked on the two slow pools");
23757            drop(phase);
23758            let source = arm.source.as_ref().expect("observed I/O source").clone();
23759            assert!(matches!(source.as_ref(), Error::Timeout));
23760            drop(arm);
23761            let blocked = state
23762                .ensure_write_safe("cancelled after mixed results")
23763                .expect_err("recovery required");
23764            assert!(Arc::ptr_eq(
23765                &source,
23766                blocked
23767                    .pool_metadata_failure()
23768                    .expect("typed latch")
23769                    .source
23770                    .as_ref()
23771                    .expect("source")
23772            ));
23773            assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
23774        }
23775
23776        #[tokio::test]
23777        async fn pool_meta_v3_cancelled_prepare_commit_and_publication_keep_gate_blocked() {
23778            for phase in [0, 1] {
23779                let gate = Arc::new(tokio::sync::Semaphore::new(0));
23780                let mut config = steps(4);
23781                for step in &mut config {
23782                    step[phase].gate = Some(gate.clone());
23783                }
23784                let fixture = Fixture::new(config);
23785                let mut state = fixture.state();
23786                let mut save = Box::pin(
23787                    fixture
23788                        .desired
23789                        .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]),
23790                );
23791                tokio::select! { _ = fixture.trace.wait_for(4 * (phase + 1), 4 * phase) => {}, _ = &mut save => panic!("blocked phase finished") }
23792                drop(save);
23793                assert!(state.ensure_write_safe("cancelled phase").is_err());
23794                assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
23795                assert_eq!(fixture.trace.active.load(Ordering::SeqCst), 0);
23796            }
23797            let fixture = Fixture::new(steps(4));
23798            let mut state = fixture.state();
23799            let outcome = fixture
23800                .desired
23801                .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0])
23802                .await
23803                .expect("durable save before publication");
23804            drop(outcome);
23805            assert!(state.ensure_write_safe("publication was cancelled").is_err());
23806            assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
23807        }
23808
23809        #[tokio::test]
23810        async fn pool_meta_v3_fence_loss_blocks_queued_dispatch_and_marks_late_writes() {
23811            let lock = NamespaceLock::with_clients_and_quorum(
23812                "parallel-fence".to_owned(),
23813                vec![Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())))],
23814                1,
23815            );
23816            let request = LockRequest::new(
23817                ObjectKey::new(super::super::RUSTFS_META_BUCKET, POOL_META_NAME),
23818                LockType::Exclusive,
23819                "phase-test",
23820            )
23821            .with_acquire_timeout(StdDuration::from_secs(1))
23822            .with_ttl(StdDuration::from_secs(1))
23823            .with_refresh_interval(StdDuration::from_secs(1));
23824            let guard = lock
23825                .acquire_guard(&request)
23826                .await
23827                .expect("lease request")
23828                .expect("lease acquired");
23829            let gate = Arc::new(tokio::sync::Semaphore::new(0));
23830            let mut config = steps(6);
23831            for step in &mut config {
23832                step[0].gate = Some(gate.clone());
23833            }
23834            let fixture = Fixture::new(config);
23835            let state = fixture.state();
23836            let mut arm = state.arm_transaction();
23837            let fence = PoolMetaPersistenceFence::Distributed(guard.lock_lost_signal());
23838            let mut pending = Box::pin(prepare(&fixture, &fence, &mut arm));
23839            tokio::select! { _ = fixture.trace.wait_for(4, 0) => {}, _ = &mut pending => panic!("gate ignored") }
23840            tokio::time::timeout(StdDuration::from_secs(3), guard.lock_lost_notified())
23841                .await
23842                .expect("lease expires");
23843            gate.add_permits(6);
23844            assert!(pending.await.into_iter().all(|result| result.is_err()));
23845            assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 4, "queued pools must not reach storage");
23846            assert!(fixture.pools[..4].iter().all(|pool| pool.inner.wrote.load(Ordering::SeqCst)));
23847            assert_eq!(arm.phase, Some("prepare_cas"));
23848            drop(arm);
23849            assert!(state.ensure_write_safe("writes landed after lease loss").is_err());
23850        }
23851
23852        #[tokio::test]
23853        async fn pool_meta_v3_all_commit_errors_still_require_canonical_reread() {
23854            for fault in [WriteFault::TimeoutAfterWrite, WriteFault::OmitWrite] {
23855                let mut config = steps(4);
23856                for step in &mut config {
23857                    step[1].fault = fault;
23858                }
23859                let fixture = Fixture::new(config);
23860                let mut state = fixture.state();
23861                let result = fixture
23862                    .desired
23863                    .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0])
23864                    .await;
23865                assert_eq!(fixture.trace.commit_count(), 4);
23866                if matches!(fault, WriteFault::TimeoutAfterWrite) {
23867                    let outcome = result.expect("reread proves commits whose replies timed out");
23868                    assert_eq!(outcome.committed.pools[0].last_update, fixture.desired.pools[0].last_update);
23869                    outcome.disarm();
23870                    state
23871                        .ensure_write_safe("confirmed ambiguous commits")
23872                        .expect("confirmed transaction is safe");
23873                } else {
23874                    assert!(result.is_err(), "successful replies cannot replace canonical proof");
23875                    assert!(state.ensure_write_safe("no committed canonical generation").is_err());
23876                }
23877            }
23878        }
23879
23880        #[tokio::test]
23881        async fn pool_meta_v3_none_and_legacy_paths_remain_serial() {
23882            for (legacy, needs_repair) in [(false, false), (true, false), (false, true)] {
23883                let gate = Arc::new(tokio::sync::Semaphore::new(0));
23884                let mut config = steps(4);
23885                for step in &mut config {
23886                    step[0].gate = Some(gate.clone());
23887                    step[1].gate = Some(gate.clone());
23888                }
23889                let mut fixture = Fixture::new(config);
23890                if legacy {
23891                    let mut meta = fixture.previous.meta.clone();
23892                    meta.version = POOL_META_VERSION;
23893                    let data = meta.encode_config_data_for_v2_gate(true).expect("legacy metadata");
23894                    for pool in &fixture.pools {
23895                        pool.inner.stored.lock().expect("fixture").as_mut().expect("replica").0 = data.clone();
23896                    }
23897                    fixture.desired.version = POOL_META_VERSION;
23898                }
23899                if needs_repair {
23900                    fixture.pools[3]
23901                        .inner
23902                        .stored
23903                        .lock()
23904                        .expect("fixture")
23905                        .as_mut()
23906                        .expect("replica")
23907                        .0 = fixture.pending.clone();
23908                }
23909                let mut state = fixture.state();
23910                let mut save = Box::pin(fixture.desired.save_no_lock_armed(
23911                    fixture.pools.clone(),
23912                    &mut state,
23913                    if legacy || needs_repair { signal() } else { None },
23914                    &[0],
23915                ));
23916                tokio::select! { _ = fixture.trace.wait_for(1, 0) => {}, _ = &mut save => panic!("serial write gate ignored") }
23917                assert!(futures::poll!(save.as_mut()).is_pending());
23918                assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 1);
23919                gate.add_permits(16);
23920                save.await.expect("serial save").disarm();
23921                assert_eq!(fixture.trace.maximum.load(Ordering::SeqCst), 1);
23922            }
23923        }
23924
23925        #[cfg(feature = "e2e-test-hooks")]
23926        #[tokio::test]
23927        async fn pool_meta_phase_barrier_leaves_noop_arm_for_the_next_changed_transaction() {
23928            for case in [
23929                implementation::PoolMetaPhaseBarrierCase::PrepareSubset,
23930                implementation::PoolMetaPhaseBarrierCase::PreparedAll,
23931                implementation::PoolMetaPhaseBarrierCase::CommitOne,
23932                implementation::PoolMetaPhaseBarrierCase::BeforePublish,
23933            ] {
23934                let fixture = Fixture::new(steps(4));
23935                let directory = tempfile::TempDir::new().expect("barrier directory");
23936                tokio::fs::write(
23937                    directory.path().join("arm.json"),
23938                    serde_json::to_vec(&serde_json::json!({"nonce": uuid::Uuid::new_v4(), "case": case})).expect("arm"),
23939                )
23940                .await
23941                .expect("write arm");
23942                let noop = implementation::encode_pool_meta_v3_envelope(&fixture.previous.meta, fixture.revision, true, None)
23943                    .expect("new revision with unchanged pools");
23944                assert_ne!(noop, fixture.previous.canonical);
23945                let barrier = implementation::PoolMetaPhaseBarrier::bind_in_directory(
23946                    directory.path().to_path_buf(),
23947                    &fixture.previous,
23948                    &fixture.previous.meta,
23949                    fixture.revision,
23950                    &noop,
23951                )
23952                .await
23953                .expect("unchanged pools must not consume the arm");
23954                assert!(barrier.is_none());
23955                assert!(!directory.path().join("claimed").exists());
23956                assert!(!directory.path().join("events.jsonl").exists());
23957
23958                let previous = implementation::PoolMetaCommittedCandidate {
23959                    meta: fixture.previous.meta.clone(),
23960                    revision: fixture.revision,
23961                    canonical: noop,
23962                };
23963                let mut candidate = previous.meta.clone();
23964                let progress = candidate.pools[0].decommission.as_mut().expect("progress");
23965                progress.items_decommissioned += 1;
23966                progress.bytes_done += 512;
23967                let revision = implementation::PoolMetaRevision {
23968                    generation: previous.revision.generation + 1,
23969                    transaction_id: Some(uuid::Uuid::new_v4()),
23970                    ..previous.revision
23971                };
23972                let durable = implementation::encode_pool_meta_v3_envelope(&candidate, revision, true, None)
23973                    .expect("changed progress without a timestamp change");
23974                let barrier = implementation::PoolMetaPhaseBarrier::bind_in_directory(
23975                    directory.path().to_path_buf(),
23976                    &previous,
23977                    &candidate,
23978                    revision,
23979                    &durable,
23980                )
23981                .await
23982                .expect("next changed transaction must bind")
23983                .expect("bound barrier");
23984                assert_eq!(barrier.transaction_id, revision.transaction_id.expect("transaction"));
23985                assert!(directory.path().join("claimed").is_file());
23986                let events = tokio::fs::read_to_string(directory.path().join("events.jsonl"))
23987                    .await
23988                    .expect("armed event");
23989                assert_eq!(events.lines().count(), 1);
23990                let event: serde_json::Value = serde_json::from_str(events.trim()).expect("armed event JSON");
23991                assert_eq!(event["kind"], "armed");
23992                assert_eq!(event["previous"]["generation"], previous.revision.generation);
23993                assert_eq!(event["candidate"]["generation"], revision.generation);
23994                assert_ne!(event["previous"]["persisted_pools_sha256"], event["candidate"]["persisted_pools_sha256"]);
23995                assert!(
23996                    implementation::PoolMetaPhaseBarrier::bind_in_directory(
23997                        directory.path().to_path_buf(),
23998                        &previous,
23999                        &candidate,
24000                        revision,
24001                        &durable,
24002                    )
24003                    .await
24004                    .expect("already claimed arm")
24005                    .is_none()
24006                );
24007            }
24008        }
24009
24010        #[cfg(feature = "e2e-test-hooks")]
24011        #[tokio::test]
24012        async fn pool_meta_phase_barrier_uses_persisted_responsibility_not_runtime_progress() {
24013            let fixture = Fixture::new(steps(4));
24014            let directory = tempfile::TempDir::new().expect("barrier directory");
24015            tokio::fs::write(
24016                directory.path().join("arm.json"),
24017                serde_json::to_vec(&serde_json::json!({"nonce": uuid::Uuid::new_v4(), "case": "prepare_subset"})).expect("arm"),
24018            )
24019            .await
24020            .expect("write arm");
24021            let mut candidate = fixture.previous.meta.clone();
24022            candidate.pools[0].decommission.as_mut().expect("progress").stage = "entry_started".to_owned();
24023            for changed in [false, true] {
24024                if changed {
24025                    candidate.pools[0]
24026                        .decommission
24027                        .as_mut()
24028                        .expect("progress")
24029                        .queued_buckets
24030                        .push("new-responsibility".to_owned());
24031                }
24032                let durable =
24033                    implementation::encode_pool_meta_v3_envelope(&candidate, fixture.revision, true, None).expect("candidate");
24034                let barrier = implementation::PoolMetaPhaseBarrier::bind_in_directory(
24035                    directory.path().to_path_buf(),
24036                    &fixture.previous,
24037                    &candidate,
24038                    fixture.revision,
24039                    &durable,
24040                )
24041                .await
24042                .expect("binding by persisted state");
24043                assert_eq!(barrier.is_some(), changed);
24044                assert_eq!(directory.path().join("claimed").exists(), changed);
24045                assert_eq!(directory.path().join("events.jsonl").exists(), changed);
24046            }
24047        }
24048
24049        fn persisted_pools(meta: &PoolMeta) -> serde_json::Value {
24050            serde_json::to_value(
24051                meta.pools
24052                    .iter()
24053                    .map(implementation::PersistedPoolStatus::from)
24054                    .collect::<Vec<_>>(),
24055            )
24056            .expect("persisted pool state")
24057        }
24058
24059        #[tokio::test]
24060        async fn pool_meta_v3_all_prepare_subsets_preserve_previous_snapshot() {
24061            for previous_version in [POOL_META_VERSION, POOL_META_GENERATION_VERSION] {
24062                for mask in 0usize..16 {
24063                    let mut fixture = Fixture::new(steps(4));
24064                    if previous_version != POOL_META_GENERATION_VERSION {
24065                        fixture.previous.meta.version = previous_version;
24066                        fixture.previous.revision = implementation::PoolMetaRevision::legacy(previous_version);
24067                        fixture.revision.generation = 1;
24068                        fixture.previous.canonical = fixture
24069                            .previous
24070                            .meta
24071                            .encode_config_data_for_v2_gate(true)
24072                            .expect("legacy predecessor");
24073                        fixture.pending = implementation::encode_pool_meta_v3_envelope(
24074                            &fixture.desired,
24075                            fixture.revision,
24076                            false,
24077                            Some(&fixture.previous),
24078                        )
24079                        .expect("legacy pending envelope");
24080                    }
24081                    let payloads = (0..4)
24082                        .map(|pool| {
24083                            if mask & (1 << pool) != 0 {
24084                                fixture.pending.clone()
24085                            } else {
24086                                fixture.previous.canonical.clone()
24087                            }
24088                        })
24089                        .collect::<Vec<_>>();
24090                    let selected = implementation::select_pool_meta_replica(
24091                        payloads
24092                            .iter()
24093                            .cloned()
24094                            .map(implementation::decode_pool_meta_replica)
24095                            .collect(),
24096                    )
24097                    .expect("every pending subset selects its predecessor");
24098                    assert_eq!(
24099                        persisted_pools(&selected.meta),
24100                        persisted_pools(&fixture.previous.meta),
24101                        "mask={mask:04b}"
24102                    );
24103                    assert_eq!(selected.revision, fixture.previous.revision);
24104                    if previous_version == POOL_META_GENERATION_VERSION {
24105                        for (pool, data) in fixture.pools.iter().zip(payloads) {
24106                            pool.inner.stored.lock().expect("fixture").as_mut().expect("replica").0 = data;
24107                        }
24108                        let mut state = fixture.state();
24109                        let selected = implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state)
24110                            .await
24111                            .expect("production recovery selection");
24112                        let outcome = implementation::repair_pool_meta_transaction(
24113                            fixture.pools.clone(),
24114                            &mut state,
24115                            selected,
24116                            &PoolMetaPersistenceFence::Distributed(None),
24117                        )
24118                        .await
24119                        .expect("production recovery repair");
24120                        assert_eq!(persisted_pools(&outcome.committed), persisted_pools(&fixture.previous.meta));
24121                        outcome.disarm();
24122                        let confirmed =
24123                            implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state)
24124                                .await
24125                                .expect("repaired reread");
24126                        assert_eq!(persisted_pools(&confirmed.meta), persisted_pools(&fixture.previous.meta));
24127                        assert!(!confirmed.replica_state.needs_repair);
24128                    }
24129                }
24130            }
24131        }
24132
24133        #[tokio::test]
24134        async fn pool_meta_v3_every_sole_commit_position_recovers_new_snapshot() {
24135            for committed_pool in 0..4 {
24136                let fixture = Fixture::new(steps(4));
24137                for (index, pool) in fixture.pools.iter().enumerate() {
24138                    pool.inner.stored.lock().expect("fixture").as_mut().expect("replica").0 = if index == committed_pool {
24139                        fixture.durable.clone()
24140                    } else {
24141                        fixture.pending.clone()
24142                    };
24143                }
24144                let mut state = fixture.state();
24145                let selected = implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state)
24146                    .await
24147                    .expect("sole commit must select new generation");
24148                assert_eq!(selected.revision, fixture.revision);
24149                assert_eq!(persisted_pools(&selected.meta), persisted_pools(&fixture.desired));
24150                let outcome = implementation::repair_pool_meta_transaction(
24151                    fixture.pools.clone(),
24152                    &mut state,
24153                    selected,
24154                    &PoolMetaPersistenceFence::Distributed(None),
24155                )
24156                .await
24157                .expect("repair partial commit");
24158                assert_eq!(persisted_pools(&outcome.committed), persisted_pools(&fixture.desired));
24159                outcome.disarm();
24160                let confirmed = implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state)
24161                    .await
24162                    .expect("repaired sole commit reread");
24163                assert_eq!(persisted_pools(&confirmed.meta), persisted_pools(&fixture.desired));
24164                assert!(!confirmed.replica_state.needs_repair);
24165                let fork = implementation::encode_pool_meta_v3_envelope(
24166                    &fixture.desired,
24167                    implementation::PoolMetaRevision {
24168                        transaction_id: Some(uuid::Uuid::new_v4()),
24169                        ..fixture.revision
24170                    },
24171                    true,
24172                    None,
24173                )
24174                .expect("same generation fork");
24175                assert!(
24176                    implementation::select_pool_meta_replica(vec![
24177                        implementation::decode_pool_meta_replica(fixture.durable.clone()),
24178                        implementation::decode_pool_meta_replica(fork)
24179                    ])
24180                    .is_err()
24181                );
24182            }
24183        }
24184    }
24185
24186    #[tokio::test]
24187    async fn test_pool_meta_cas_deterministically_rejects_stale_writer() {
24188        let storage = Arc::new(PartialPoolMetaWriteStorage {
24189            fail_write: false,
24190            fail_after_first_write: false,
24191            pending_write: false,
24192            write_started: tokio::sync::Notify::new(),
24193            wrote: AtomicBool::new(false),
24194            revision: AtomicUsize::new(0),
24195            stored: StdMutex::new(None),
24196            identity: StdMutex::new(None),
24197        });
24198        let stale_token = read_pool_meta_replica(storage.clone(), true).await.cas;
24199        assert!(matches!(&stale_token, PoolMetaCasToken::Missing));
24200        let winner = PoolMeta {
24201            version: POOL_META_VERSION,
24202            pools: vec![decommission_test_pool_status(0, None)],
24203            ..Default::default()
24204        }
24205        .encode_config_data_for_test()
24206        .expect("winning pool metadata should encode");
24207        let stale = PoolMeta {
24208            version: POOL_META_VERSION,
24209            pools: vec![decommission_test_pool_status(
24210                0,
24211                Some(PoolDecommissionInfo {
24212                    complete: true,
24213                    ..Default::default()
24214                }),
24215            )],
24216            ..Default::default()
24217        }
24218        .encode_config_data_for_test()
24219        .expect("stale pool metadata should encode");
24220        let fence = PoolMetaPersistenceFence::Distributed(None);
24221
24222        let mut arm = super::PoolMetaWriteState::default().arm_transaction();
24223        save_pool_meta_object_cas(
24224            storage.clone(),
24225            POOL_META_NAME,
24226            winner.clone(),
24227            &stale_token,
24228            &fence,
24229            "prepare_cas",
24230            &mut arm,
24231        )
24232        .await
24233        .expect("the first writer should create pool metadata");
24234        let err =
24235            save_pool_meta_object_cas(storage.clone(), POOL_META_NAME, stale, &stale_token, &fence, "prepare_cas", &mut arm)
24236                .await
24237                .expect_err("the second writer must not reuse the stale missing-object revision");
24238
24239        assert_eq!(err, Error::PreconditionFailed);
24240        let stored = storage
24241            .stored
24242            .lock()
24243            .expect("pool metadata test storage should not be poisoned")
24244            .as_ref()
24245            .map(|(data, _)| data.clone())
24246            .expect("the winning pool metadata should remain stored");
24247        assert_eq!(stored, winner);
24248    }
24249
24250    #[tokio::test]
24251    async fn test_pool_meta_v3_single_replica_commit_failure_recovers_on_restart() {
24252        let committed_replica = Arc::new(PartialPoolMetaWriteStorage {
24253            fail_write: false,
24254            fail_after_first_write: false,
24255            pending_write: false,
24256            write_started: tokio::sync::Notify::new(),
24257            wrote: AtomicBool::new(false),
24258            revision: AtomicUsize::new(0),
24259            stored: StdMutex::new(None),
24260            identity: StdMutex::new(None),
24261        });
24262        let pending_replica = Arc::new(PartialPoolMetaWriteStorage {
24263            fail_write: false,
24264            fail_after_first_write: true,
24265            pending_write: false,
24266            write_started: tokio::sync::Notify::new(),
24267            wrote: AtomicBool::new(false),
24268            revision: AtomicUsize::new(0),
24269            stored: StdMutex::new(None),
24270            identity: StdMutex::new(None),
24271        });
24272        let cluster_id = uuid::Uuid::new_v4();
24273        let fresh_bootstrap_nonce = uuid::Uuid::new_v4();
24274        let identity = super::encode_pool_meta_identity(super::PersistedPoolMetaIdentity {
24275            version: super::POOL_META_IDENTITY_VERSION,
24276            cluster_id,
24277            epoch: super::POOL_META_INITIAL_EPOCH,
24278            initialized: false,
24279            fresh_bootstrap_nonce: Some(fresh_bootstrap_nonce),
24280        })
24281        .expect("pending bootstrap identity should encode");
24282        *committed_replica
24283            .identity
24284            .lock()
24285            .expect("identity storage should not be poisoned") = Some((identity.clone(), "identity-0".to_string()));
24286        *pending_replica
24287            .identity
24288            .lock()
24289            .expect("identity storage should not be poisoned") = Some((identity, "identity-0".to_string()));
24290        let mut write_state = super::PoolMetaWriteState::for_startup(cluster_id, true);
24291        let snapshot = PoolMeta {
24292            version: super::POOL_META_GENERATION_VERSION,
24293            pools: vec![decommission_test_pool_status(
24294                0,
24295                Some(PoolDecommissionInfo {
24296                    queued: true,
24297                    ..Default::default()
24298                }),
24299            )],
24300            ..Default::default()
24301        };
24302
24303        snapshot
24304            .save_no_lock_observing(vec![committed_replica.clone(), pending_replica.clone()], &mut write_state)
24305            .await
24306            .expect("one committed replica should durably complete the V3 transaction");
24307
24308        let mut restarted = PoolMeta::default();
24309        let replica_state = restarted
24310            .load_no_lock_from_replicas(vec![committed_replica, pending_replica.clone()])
24311            .await
24312            .expect("restart should select the committed generation over a pending replica");
24313        assert_eq!(restarted.version, super::POOL_META_GENERATION_VERSION);
24314        assert!(restarted.pools[0].decommission.as_ref().is_some_and(|info| info.queued));
24315        assert!(replica_state.needs_repair);
24316
24317        let mut pending_only = PoolMeta::default();
24318        pending_only
24319            .load_no_lock_from_replicas(vec![pending_replica])
24320            .await
24321            .expect("a prepare-only replica should expose the embedded predecessor");
24322        assert!(pending_only.pools.is_empty());
24323    }
24324
24325    #[tokio::test]
24326    async fn test_partial_pool_meta_save_failure_blocks_following_side_effect() {
24327        let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
24328        let committed = Arc::new(PartialPoolMetaWriteStorage {
24329            fail_write: false,
24330            fail_after_first_write: false,
24331            pending_write: false,
24332            write_started: tokio::sync::Notify::new(),
24333            wrote: AtomicBool::new(false),
24334            revision: AtomicUsize::new(0),
24335            stored: StdMutex::new(None),
24336            identity: StdMutex::new(None),
24337        });
24338        let failed = Arc::new(PartialPoolMetaWriteStorage {
24339            fail_write: true,
24340            fail_after_first_write: false,
24341            pending_write: false,
24342            write_started: tokio::sync::Notify::new(),
24343            wrote: AtomicBool::new(false),
24344            revision: AtomicUsize::new(0),
24345            stored: StdMutex::new(None),
24346            identity: StdMutex::new(None),
24347        });
24348        let snapshot = PoolMeta {
24349            version: super::POOL_META_VERSION,
24350            pools: vec![decommission_test_pool_status(0, None)],
24351            ..Default::default()
24352        };
24353
24354        {
24355            let mut save_guard = store.pool_meta_save_gate.lock().await;
24356            snapshot
24357                .save_no_lock_observing(vec![committed.clone(), failed], &mut save_guard)
24358                .await
24359                .expect_err("the second replica write should fail after the first commits");
24360        }
24361        assert!(committed.wrote.load(Ordering::SeqCst));
24362
24363        let ran = Arc::new(AtomicBool::new(false));
24364        let ran_by_operation = ran.clone();
24365        let movement_gate = store.ctx.data_movement_operation_gate();
24366        let result: std::result::Result<(), Error> = store
24367            .run_guarded_decommission_side_effect(&CancellationToken::new(), &movement_gate, move || async move {
24368                ran_by_operation.store(true, Ordering::SeqCst);
24369                Ok(())
24370            })
24371            .await;
24372
24373        assert!(result.is_err(), "a partial pool metadata save must latch the sticky safety gate");
24374        assert!(!ran.load(Ordering::SeqCst), "the side effect must not run after a partial save");
24375    }
24376
24377    #[tokio::test]
24378    async fn test_cancelled_pool_meta_save_blocks_following_side_effect() {
24379        let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
24380        let committed = Arc::new(PartialPoolMetaWriteStorage {
24381            fail_write: false,
24382            fail_after_first_write: false,
24383            pending_write: false,
24384            write_started: tokio::sync::Notify::new(),
24385            wrote: AtomicBool::new(false),
24386            revision: AtomicUsize::new(0),
24387            stored: StdMutex::new(None),
24388            identity: StdMutex::new(None),
24389        });
24390        let pending = Arc::new(PartialPoolMetaWriteStorage {
24391            fail_write: false,
24392            fail_after_first_write: false,
24393            pending_write: true,
24394            write_started: tokio::sync::Notify::new(),
24395            wrote: AtomicBool::new(false),
24396            revision: AtomicUsize::new(0),
24397            stored: StdMutex::new(None),
24398            identity: StdMutex::new(None),
24399        });
24400        let snapshot = PoolMeta {
24401            version: super::POOL_META_VERSION,
24402            pools: vec![decommission_test_pool_status(0, None)],
24403            ..Default::default()
24404        };
24405        let save_store = store.clone();
24406        let save_committed = committed.clone();
24407        let save_pending = pending.clone();
24408        let save_task = tokio::spawn(async move {
24409            let mut save_guard = save_store.pool_meta_save_gate.lock().await;
24410            snapshot
24411                .save_no_lock_observing(vec![save_committed, save_pending], &mut save_guard)
24412                .await
24413        });
24414
24415        tokio::time::timeout(StdDuration::from_secs(1), pending.write_started.notified())
24416            .await
24417            .expect("the second replica write should start");
24418        assert!(committed.wrote.load(Ordering::SeqCst));
24419        save_task.abort();
24420        assert!(
24421            save_task.await.expect_err("the save task should be cancelled").is_cancelled(),
24422            "the pending replica write should be aborted"
24423        );
24424
24425        {
24426            let save_guard = store.pool_meta_save_gate.lock().await;
24427            save_guard
24428                .ensure_write_safe("cancelled pool metadata save")
24429                .expect_err("a cancelled replica update must latch the sticky safety gate");
24430        }
24431
24432        let ran = Arc::new(AtomicBool::new(false));
24433        let ran_by_operation = ran.clone();
24434        let movement_gate = store.ctx.data_movement_operation_gate();
24435        let result: std::result::Result<(), Error> = store
24436            .run_guarded_decommission_side_effect(&CancellationToken::new(), &movement_gate, move || async move {
24437                ran_by_operation.store(true, Ordering::SeqCst);
24438                Ok(())
24439            })
24440            .await;
24441
24442        assert!(result.is_err(), "a cancelled pool metadata save must block following side effects");
24443        assert!(!ran.load(Ordering::SeqCst), "the side effect must not run after a cancelled save");
24444    }
24445
24446    #[tokio::test]
24447    async fn test_cancelled_pool_meta_publish_keeps_write_gate_blocked() {
24448        let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
24449        let committed = Arc::new(PartialPoolMetaWriteStorage {
24450            fail_write: false,
24451            fail_after_first_write: false,
24452            pending_write: false,
24453            write_started: tokio::sync::Notify::new(),
24454            wrote: AtomicBool::new(false),
24455            revision: AtomicUsize::new(0),
24456            stored: StdMutex::new(None),
24457            identity: StdMutex::new(None),
24458        });
24459        let snapshot = PoolMeta {
24460            version: super::POOL_META_VERSION,
24461            pools: vec![decommission_test_pool_status(0, None)],
24462            ..Default::default()
24463        };
24464        let publish_started = Arc::new(tokio::sync::Notify::new());
24465        let publish_release = Arc::new(tokio::sync::Notify::new());
24466        let save_store = store.clone();
24467        let save_committed = committed.clone();
24468        let task_publish_started = publish_started.clone();
24469        let task_publish_release = publish_release.clone();
24470        let save_task = tokio::spawn(async move {
24471            let mut save_guard = save_store.pool_meta_save_gate.lock().await;
24472            let outcome = snapshot
24473                .save_no_lock_armed(vec![save_committed], &mut save_guard, None, &[0])
24474                .await?;
24475            task_publish_started.notify_one();
24476            task_publish_release.notified().await;
24477            outcome.disarm();
24478            Ok::<(), Error>(())
24479        });
24480
24481        tokio::time::timeout(StdDuration::from_secs(1), publish_started.notified())
24482            .await
24483            .expect("the replica save should complete before publication");
24484        assert!(committed.wrote.load(Ordering::SeqCst));
24485        save_task.abort();
24486        assert!(
24487            save_task
24488                .await
24489                .expect_err("the publication task should be cancelled")
24490                .is_cancelled(),
24491            "the task should be aborted while publication is pending"
24492        );
24493
24494        let save_guard = store.pool_meta_save_gate.lock().await;
24495        save_guard
24496            .ensure_write_safe("cancelled pool metadata publication")
24497            .expect_err("cancellation after replica save but before publication must keep writes blocked");
24498    }
24499
24500    #[tokio::test]
24501    async fn test_lost_pool_meta_fence_rejects_replica_write() {
24502        let client = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
24503        let lock = NamespaceLock::with_clients_and_quorum("pool-meta-fence-loss".to_string(), vec![client], 1);
24504        let request = LockRequest::new(
24505            ObjectKey::new(super::RUSTFS_META_BUCKET, super::POOL_META_NAME),
24506            LockType::Exclusive,
24507            "stale-writer",
24508        )
24509        .with_acquire_timeout(StdDuration::from_secs(1))
24510        .with_ttl(StdDuration::from_millis(50))
24511        .with_refresh_interval(StdDuration::from_millis(50));
24512        let guard = lock
24513            .acquire_guard(&request)
24514            .await
24515            .expect("pool metadata fence acquisition should not error")
24516            .expect("the stale writer should acquire the pool metadata fence");
24517        tokio::time::timeout(StdDuration::from_secs(2), guard.lock_lost_notified())
24518            .await
24519            .expect("the stale writer lease should expire");
24520
24521        let storage = Arc::new(PartialPoolMetaWriteStorage {
24522            fail_write: false,
24523            fail_after_first_write: false,
24524            pending_write: false,
24525            write_started: tokio::sync::Notify::new(),
24526            wrote: AtomicBool::new(false),
24527            revision: AtomicUsize::new(0),
24528            stored: StdMutex::new(None),
24529            identity: StdMutex::new(None),
24530        });
24531        let snapshot = PoolMeta {
24532            version: super::POOL_META_VERSION,
24533            pools: vec![decommission_test_pool_status(0, None)],
24534            ..Default::default()
24535        };
24536        let mut write_state = super::PoolMetaWriteState::default();
24537        let err = snapshot
24538            .save_no_lock_armed(vec![storage.clone()], &mut write_state, guard.lock_lost_signal(), &[0])
24539            .await
24540            .expect_err("a writer must not persist after losing the distributed pool metadata fence");
24541
24542        assert!(err.to_string().contains("distributed fence was lost"));
24543        assert!(!storage.wrote.load(Ordering::SeqCst), "the stale writer must not reach replica storage");
24544        write_state
24545            .ensure_write_safe("lost pool metadata fence")
24546            .expect("fence loss before dispatch has no unknown durable write and must remain retryable");
24547    }
24548
24549    #[tokio::test]
24550    async fn test_pool_meta_fence_loss_after_publish_keeps_write_gate_blocked() {
24551        let client = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
24552        let lock = NamespaceLock::with_clients_and_quorum("pool-meta-publish-fence-loss".to_string(), vec![client], 1);
24553        let request = LockRequest::new(
24554            ObjectKey::new(super::RUSTFS_META_BUCKET, super::POOL_META_NAME),
24555            LockType::Exclusive,
24556            "publishing-writer",
24557        )
24558        .with_acquire_timeout(StdDuration::from_secs(1))
24559        .with_ttl(StdDuration::from_secs(1))
24560        .with_refresh_interval(StdDuration::from_secs(1));
24561        let guard = lock
24562            .acquire_guard(&request)
24563            .await
24564            .expect("pool metadata fence acquisition should not error")
24565            .expect("the publishing writer should acquire the pool metadata fence");
24566        let storage = Arc::new(PartialPoolMetaWriteStorage {
24567            fail_write: false,
24568            fail_after_first_write: false,
24569            pending_write: false,
24570            write_started: tokio::sync::Notify::new(),
24571            wrote: AtomicBool::new(false),
24572            revision: AtomicUsize::new(0),
24573            stored: StdMutex::new(None),
24574            identity: StdMutex::new(None),
24575        });
24576        let mut saved = PoolMeta {
24577            version: super::POOL_META_VERSION,
24578            pools: vec![decommission_test_pool_status(0, None)],
24579            ..Default::default()
24580        };
24581        saved.pools[0].last_update = OffsetDateTime::UNIX_EPOCH + Duration::seconds(1);
24582        let mut current = saved.clone();
24583        current.pools[0].last_update = OffsetDateTime::UNIX_EPOCH;
24584        let mut write_state = super::PoolMetaWriteState::for_test_bootstrap();
24585        let outcome = saved
24586            .save_no_lock_armed(vec![storage], &mut write_state, guard.lock_lost_signal(), &[0])
24587            .await
24588            .expect("the replica save should finish while the fence is valid");
24589
24590        ensure_pool_meta_write_fence(&guard, "test pool metadata publish")
24591            .expect("the fence should remain valid before publication");
24592        publish_pool_meta_updates(&mut current, &saved, &[0]);
24593        tokio::time::timeout(StdDuration::from_secs(2), guard.lock_lost_notified())
24594            .await
24595            .expect("the fence should expire after publication");
24596        ensure_pool_meta_write_fence(&guard, "test pool metadata publish")
24597            .expect_err("the post-publication fence check must observe the loss");
24598
24599        assert_eq!(current.pools[0].last_update, saved.pools[0].last_update);
24600        drop(outcome);
24601        write_state
24602            .ensure_write_safe("lost pool metadata publish fence")
24603            .expect_err("the sticky write gate must remain armed after post-publication fence loss");
24604    }
24605
24606    #[tokio::test]
24607    async fn test_clear_decommission_transaction_survives_caller_abort() {
24608        let pool_meta = PoolMeta {
24609            version: super::POOL_META_VERSION,
24610            pools: vec![decommission_test_pool_status(
24611                0,
24612                Some(PoolDecommissionInfo {
24613                    failed: true,
24614                    ..Default::default()
24615                }),
24616            )],
24617            ..Default::default()
24618        };
24619        let store = decommission_worker_test_store(pool_meta, vec![None]);
24620        let save_started = Arc::new(tokio::sync::Notify::new());
24621        let save_release = Arc::new(tokio::sync::Notify::new());
24622        let save_completed = Arc::new(AtomicBool::new(false));
24623        let caller_store = store.clone();
24624        let caller_save_started = save_started.clone();
24625        let caller_save_release = save_release.clone();
24626        let caller_save_completed = save_completed.clone();
24627        let caller_task = tokio::spawn(async move {
24628            caller_store
24629                .clear_decommission_with_save(0, move || async move {
24630                    caller_save_started.notify_one();
24631                    caller_save_release.notified().await;
24632                    caller_save_completed.store(true, Ordering::SeqCst);
24633                    Ok(())
24634                })
24635                .await
24636        });
24637
24638        tokio::time::timeout(StdDuration::from_secs(1), save_started.notified())
24639            .await
24640            .expect("the clear transaction should reach persistence");
24641        caller_task.abort();
24642        assert!(
24643            caller_task
24644                .await
24645                .expect_err("the RPC waiter should be cancelled")
24646                .is_cancelled(),
24647            "the clear caller should be aborted while persistence is pending"
24648        );
24649        save_release.notify_one();
24650
24651        let _start_guard = tokio::time::timeout(StdDuration::from_secs(1), store.start_gate.lock())
24652            .await
24653            .expect("the detached clear transaction should finish");
24654        assert!(
24655            save_completed.load(Ordering::SeqCst),
24656            "the detached transaction must finish persistence after the caller is aborted"
24657        );
24658        let pool_meta = store.pool_meta.read().await;
24659        assert!(
24660            pool_meta.pools[0]
24661                .decommission
24662                .as_ref()
24663                .is_some_and(|info| !info.has_decommission_state())
24664        );
24665    }
24666
24667    fn decommission_test_pool_endpoint(idx: usize, is_local: bool) -> PoolEndpoints {
24668        let port = 9000usize + idx;
24669        let mut endpoint =
24670            Endpoint::try_from(format!("http://127.0.0.1:{port}/disk").as_str()).expect("test endpoint should parse");
24671        endpoint.is_local = is_local;
24672        endpoint.pool_idx = i32::try_from(idx).expect("test pool index should fit i32");
24673
24674        PoolEndpoints {
24675            legacy: false,
24676            set_count: 1,
24677            drives_per_set: 1,
24678            endpoints: Endpoints::from(vec![endpoint]),
24679            cmd_line: format!("pool-{idx}"),
24680            platform: String::new(),
24681        }
24682    }
24683
24684    fn decommission_test_pool_status(idx: usize, decommission: Option<PoolDecommissionInfo>) -> PoolStatus {
24685        PoolStatus {
24686            id: idx,
24687            cmd_line: format!("pool-{idx}"),
24688            last_update: OffsetDateTime::now_utc(),
24689            decommission,
24690        }
24691    }
24692
24693    fn decommission_test_active_model_meta(model_versions: &[u16]) -> PoolMeta {
24694        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
24695        PoolMeta {
24696            version: POOL_META_VERSION,
24697            pools: model_versions
24698                .iter()
24699                .copied()
24700                .enumerate()
24701                .map(|(idx, model_version)| {
24702                    let reservation = build_decommission_capacity_reservation_with_model(
24703                        DecommissionPoolCapacityInfo::for_test(idx, layout, 0, 10, 10),
24704                        layout,
24705                        uuid::Uuid::new_v4(),
24706                        u64::try_from(idx).unwrap_or_default() + 1,
24707                        OffsetDateTime::UNIX_EPOCH,
24708                        model_version,
24709                    )
24710                    .expect("test capacity model should be supported");
24711                    decommission_test_pool_status(
24712                        idx,
24713                        Some(PoolDecommissionInfo {
24714                            start_time: Some(OffsetDateTime::UNIX_EPOCH),
24715                            capacity_reservation: Some(reservation),
24716                            ..Default::default()
24717                        }),
24718                    )
24719                })
24720                .collect(),
24721            ..Default::default()
24722        }
24723    }
24724
24725    fn decommission_test_cleanup_meta(
24726        model_version: u16,
24727        temporary_mutations: Vec<DecommissionCapacityTemporaryMutation>,
24728        inflight_physical_bytes: usize,
24729        pending: Option<(uuid::Uuid, usize)>,
24730    ) -> PoolMeta {
24731        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
24732        let mut reservation = build_decommission_capacity_reservation_with_model(
24733            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
24734            layout,
24735            uuid::Uuid::new_v4(),
24736            1,
24737            OffsetDateTime::UNIX_EPOCH,
24738            model_version,
24739        )
24740        .expect("test cleanup reservation should be valid");
24741        let (pending_mutation_id, pending_physical_bytes) = pending.unzip();
24742        let pending_physical_bytes = pending_physical_bytes.unwrap_or_default();
24743        reservation.observed_target_physical_bytes = inflight_physical_bytes;
24744        reservation.inflight_target_physical_bytes = inflight_physical_bytes;
24745        reservation.pending_target_physical_bytes = pending_physical_bytes;
24746        reservation.targets.push(DecommissionCapacityTarget {
24747            pool_index: 1,
24748            layout,
24749            physical_total_at_reservation: 200,
24750            physical_free_at_reservation: 200,
24751            reserved_physical_bytes: reservation.peak_physical_bytes,
24752            consumed_physical_bytes: 0,
24753            observed_physical_bytes: inflight_physical_bytes,
24754            inflight_physical_bytes,
24755            pending_physical_bytes,
24756            pending_mutation_id,
24757            temporary_mutations,
24758        });
24759        PoolMeta {
24760            version: POOL_META_VERSION,
24761            pools: vec![
24762                decommission_test_pool_status(
24763                    0,
24764                    Some(PoolDecommissionInfo {
24765                        start_time: Some(OffsetDateTime::UNIX_EPOCH),
24766                        capacity_reservation: Some(reservation),
24767                        ..Default::default()
24768                    }),
24769                ),
24770                decommission_test_pool_status(1, None),
24771            ],
24772            ..Default::default()
24773        }
24774    }
24775
24776    #[tokio::test]
24777    async fn test_activation_fence_uses_one_lock_order_and_serializes_callers() {
24778        let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
24779        let first = Arc::new(ActivationLockRecorder {
24780            lock_manager: Arc::clone(&manager),
24781            owner: "first",
24782            resources: StdMutex::new(Vec::new()),
24783        });
24784        let second = Arc::new(ActivationLockRecorder {
24785            lock_manager: manager,
24786            owner: "second",
24787            resources: StdMutex::new(Vec::new()),
24788        });
24789
24790        let first_guards = acquire_pool_rebalance_activation_locks(first.clone(), None)
24791            .await
24792            .expect("first activation should acquire both locks");
24793        assert_eq!(
24794            *first
24795                .resources
24796                .lock()
24797                .expect("activation lock recorder should not be poisoned"),
24798            vec![REBAL_META_NAME.to_string(), POOL_META_NAME.to_string()]
24799        );
24800
24801        let mut second_acquire = Box::pin(acquire_pool_rebalance_activation_locks(second.clone(), None));
24802        let mut context = Context::from_waker(futures::task::noop_waker_ref());
24803        assert!(matches!(second_acquire.as_mut().poll(&mut context), Poll::Pending));
24804
24805        drop(first_guards);
24806        second_acquire
24807            .await
24808            .expect("second activation should acquire both locks after the first releases them");
24809        assert_eq!(
24810            *second
24811                .resources
24812                .lock()
24813                .expect("activation lock recorder should not be poisoned"),
24814            vec![REBAL_META_NAME.to_string(), POOL_META_NAME.to_string()]
24815        );
24816    }
24817
24818    #[tokio::test]
24819    async fn test_activation_cancellation_releases_rebalance_fence_while_pool_fence_is_contended() {
24820        use crate::storage_api_contracts::namespace::NamespaceLocking as _;
24821        let pool = Arc::new(ActivationLockRecorder {
24822            lock_manager: Arc::new(rustfs_lock::GlobalLockManager::new()),
24823            owner: "activation-cancellation",
24824            resources: StdMutex::new(Vec::new()),
24825        });
24826        let pool_lock = pool
24827            .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, POOL_META_NAME)
24828            .await
24829            .expect("pool lock should be created");
24830        let pool_reader = pool_lock
24831            .get_read_lock(std::time::Duration::from_secs(5))
24832            .await
24833            .expect("ordinary mutation should hold the pool read fence");
24834        pool.resources.lock().expect("recorder should not be poisoned").clear();
24835        let mut activation = Box::pin(acquire_pool_rebalance_activation_locks(Arc::clone(&pool), None));
24836        assert!(matches!(futures::poll!(&mut activation), Poll::Pending));
24837        assert_eq!(
24838            *pool.resources.lock().expect("recorder should not be poisoned"),
24839            vec![REBAL_META_NAME.to_string(), POOL_META_NAME.to_string()],
24840            "activation must hold the run fence before waiting for the pool fence",
24841        );
24842        drop(activation);
24843        let rebalance_lock = pool
24844            .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME)
24845            .await
24846            .expect("run lock should be created");
24847        let run_writer = rebalance_lock
24848            .get_write_lock(std::time::Duration::from_secs(5))
24849            .await
24850            .expect("cancelling activation must release its already-acquired run fence");
24851        assert!(
24852            !pool_reader.is_released(),
24853            "cancelling activation must not release another caller's pool fence"
24854        );
24855        assert!(!run_writer.is_lock_lost());
24856    }
24857
24858    #[test]
24859    fn decommission_receipt_run_token_changes_with_persisted_start_time() {
24860        let first = OffsetDateTime::from_unix_timestamp(1_000).expect("first run timestamp should be valid");
24861        let second = OffsetDateTime::from_unix_timestamp(2_000).expect("second run timestamp should be valid");
24862        let first_token = decommission_durable_ilm_receipt_run_token("pool-0", first);
24863        let second_token = decommission_durable_ilm_receipt_run_token("pool-0", second);
24864
24865        assert_ne!(first_token, second_token);
24866        assert_eq!(first_token, decommission_durable_ilm_receipt_run_token("pool-0", first));
24867        let operation_id = "a".repeat(64);
24868        let old_receipt = decommission_durable_ilm_receipt_path(
24869            &first_token,
24870            &format!("ilm/tier-delete-journal/{operation_id}.json"),
24871            "operation_id",
24872            &operation_id,
24873        );
24874        assert!(!old_receipt.starts_with(&decommission_durable_ilm_receipt_run_prefix(&second_token)));
24875    }
24876
24877    #[test]
24878    fn decommission_recovery_control_receipt_path_round_trips() {
24879        let run_token = "b".repeat(64);
24880        let control_id = "a".repeat(64);
24881        let source_path = format!(
24882            "ilm/recovery-controls/transition_transaction/{}/{}/{}.json",
24883            &control_id[..2],
24884            &control_id[2..4],
24885            control_id
24886        );
24887        let path = decommission_durable_ilm_receipt_path(&run_token, &source_path, "control_id", &control_id);
24888
24889        let locator = parse_decommission_durable_ilm_receipt_path(&path).expect("recovery control receipt path should parse");
24890
24891        assert_eq!(locator.run_token, run_token);
24892        assert_eq!(locator.source_path, source_path);
24893        assert_eq!(locator.id_kind, "control_id");
24894        assert_eq!(locator.id, control_id);
24895    }
24896
24897    #[test]
24898    fn decommission_receipt_merge_preserves_terminal_proof() {
24899        let operation_id = "a".repeat(64);
24900        let source_path = format!("ilm/tier-delete-journal/{operation_id}.json");
24901        let checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal {
24902            content_sha256: "b".repeat(64),
24903            identity_sha256: "c".repeat(64),
24904            committed: false,
24905            dispatch_identity_sha256: None,
24906            state: None,
24907        };
24908        let terminal_checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal {
24909            content_sha256: "d".repeat(64),
24910            identity_sha256: "c".repeat(64),
24911            committed: true,
24912            dispatch_identity_sha256: None,
24913            state: None,
24914        };
24915        let incoming = DecommissionDurableIlmReceipt {
24916            source_path,
24917            namespace: "tier-delete-journal".to_string(),
24918            id_kind: "operation_id".to_string(),
24919            id: operation_id,
24920            checkpoint: checkpoint.clone(),
24921            terminal_checkpoint: None,
24922            fleet_topology_generation: None,
24923        };
24924        let existing = DecommissionDurableIlmReceipt {
24925            terminal_checkpoint: Some(terminal_checkpoint.clone()),
24926            ..incoming.clone()
24927        };
24928
24929        let merged = merge_decommission_durable_ilm_receipts(&existing, &incoming)
24930            .expect("retry receipt must merge with a terminal receipt");
24931
24932        assert_eq!(merged.checkpoint, checkpoint);
24933        assert_eq!(merged.terminal_checkpoint, Some(terminal_checkpoint.clone()));
24934
24935        let topology_bound = DecommissionDurableIlmReceipt {
24936            fleet_topology_generation: Some("e".repeat(64)),
24937            ..incoming
24938        };
24939        let mixed_error = merge_decommission_durable_ilm_receipts(&existing, &topology_bound)
24940            .expect_err("a topology-bound v6 receipt must not mask an unbound receipt")
24941            .to_string();
24942        assert!(mixed_error.contains("fleet topology conflict"));
24943
24944        let topology_existing = DecommissionDurableIlmReceipt {
24945            terminal_checkpoint: Some(terminal_checkpoint),
24946            ..topology_bound.clone()
24947        };
24948        let topology_merged = merge_decommission_durable_ilm_receipts(&topology_existing, &topology_bound)
24949            .expect("receipts bound to the same fleet topology should merge");
24950        assert_eq!(topology_merged.fleet_topology_generation, Some("e".repeat(64)));
24951    }
24952
24953    #[test]
24954    fn decommission_manual_job_receipt_compacts_large_progress() {
24955        let prefix = "p".repeat(12 * 1024);
24956        let options = ManualTransitionRunOptions {
24957            prefix,
24958            ..Default::default()
24959        };
24960        let mut job = ManualTransitionJobRecord::new(uuid::Uuid::new_v4(), "bounded-receipt-bucket", &options, "owner");
24961        let token_bytes = serde_json::to_vec(&serde_json::json!({
24962            "marker": "m".repeat(12 * 1024),
24963            "version_marker": "opaque-version"
24964        }))
24965        .expect("large continuation token should encode");
24966        let mut report = job.report.clone();
24967        report.scanned = 1;
24968        report.continuation_token = Some(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&token_bytes));
24969        job.update_running_progress(report, ManualTransitionQueueSnapshot::default());
24970        let path = manual_transition_job_record_object_name(job.job_id).expect("manual job path should build");
24971        let job_bytes = job.encode().expect("large manual job should remain within its record limit");
24972        assert!(job_bytes.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE);
24973        let record = validate_durable_ilm_record(&path, &job_bytes).expect("large manual job should validate");
24974        let expected_checkpoint = record.checkpoint.clone();
24975        let mut receipt = DecommissionDurableIlmReceipt::new(&path, &record, None);
24976        receipt.terminal_checkpoint = Some(record.checkpoint);
24977
24978        let encoded = receipt.encode().expect("bounded progress proof should fit the receipt limit");
24979        let decoded = DecommissionDurableIlmReceipt::decode(&encoded).expect("bounded receipt should round trip");
24980
24981        assert!(encoded.len() <= DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE);
24982        assert_eq!(decoded.source_path, path);
24983        assert_eq!(decoded.checkpoint, expected_checkpoint);
24984        assert_eq!(decoded.terminal_checkpoint, Some(expected_checkpoint));
24985    }
24986
24987    #[test]
24988    fn test_apply_decommission_status_space_info_adds_idle_pool_usage() {
24989        let status = apply_decommission_status_space_info(
24990            decommission_test_pool_status(0, None),
24991            PoolSpaceInfo {
24992                free: 25,
24993                total: 100,
24994                used: 75,
24995            },
24996        );
24997
24998        let decommission = status.decommission.expect("idle pool status should include usage info");
24999        assert_eq!(decommission.total_size, 100);
25000        assert_eq!(decommission.current_size, 25);
25001        assert!(decommission.start_time.is_none());
25002        assert!(!decommission.complete);
25003        assert!(!decommission.failed);
25004        assert!(!decommission.canceled);
25005    }
25006
25007    #[test]
25008    fn test_apply_decommission_status_space_info_refreshes_active_decommission_sizes() {
25009        let status = apply_decommission_status_space_info(
25010            decommission_test_pool_status(
25011                0,
25012                Some(PoolDecommissionInfo {
25013                    total_size: 1,
25014                    current_size: 1,
25015                    ..Default::default()
25016                }),
25017            ),
25018            PoolSpaceInfo {
25019                free: 25,
25020                total: 100,
25021                used: 75,
25022            },
25023        );
25024
25025        let decommission = status.decommission.expect("active decommission info should remain present");
25026        assert_eq!(decommission.total_size, 100);
25027        assert_eq!(decommission.current_size, 25);
25028    }
25029
25030    #[test]
25031    fn test_merge_pool_status_refresh_uses_persisted_terminal_decommission() {
25032        let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
25033        let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
25034        let mut current = PoolMeta {
25035            pools: vec![PoolStatus {
25036                id: 0,
25037                cmd_line: "pool-0".to_string(),
25038                last_update: older,
25039                decommission: Some(PoolDecommissionInfo {
25040                    start_time: Some(older),
25041                    ..Default::default()
25042                }),
25043            }],
25044            ..Default::default()
25045        };
25046        let persisted = PoolMeta {
25047            pools: vec![PoolStatus {
25048                id: 0,
25049                cmd_line: "pool-0".to_string(),
25050                last_update: newer,
25051                decommission: Some(PoolDecommissionInfo {
25052                    complete: true,
25053                    ..Default::default()
25054                }),
25055            }],
25056            ..Default::default()
25057        };
25058
25059        assert!(merge_pool_status_refresh(&mut current, persisted, &[false]));
25060
25061        let info = current.pools[0]
25062            .decommission
25063            .as_ref()
25064            .expect("decommission info should be present");
25065        assert!(info.complete);
25066        assert!(!info.failed);
25067        assert!(!info.canceled);
25068    }
25069
25070    #[test]
25071    fn test_merge_pool_status_refresh_keeps_newer_local_active_progress() {
25072        let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
25073        let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
25074        let mut current = PoolMeta {
25075            pools: vec![PoolStatus {
25076                id: 0,
25077                cmd_line: "pool-0".to_string(),
25078                last_update: newer,
25079                decommission: Some(PoolDecommissionInfo {
25080                    start_time: Some(older),
25081                    items_decommissioned: 10,
25082                    bytes_done: 1_024,
25083                    ..Default::default()
25084                }),
25085            }],
25086            ..Default::default()
25087        };
25088        let persisted = PoolMeta {
25089            pools: vec![PoolStatus {
25090                id: 0,
25091                cmd_line: "pool-0".to_string(),
25092                last_update: older,
25093                decommission: Some(PoolDecommissionInfo {
25094                    start_time: Some(older),
25095                    items_decommissioned: 1,
25096                    bytes_done: 128,
25097                    ..Default::default()
25098                }),
25099            }],
25100            ..Default::default()
25101        };
25102
25103        assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
25104
25105        let info = current.pools[0]
25106            .decommission
25107            .as_ref()
25108            .expect("local decommission info should remain present");
25109        assert_eq!(info.items_decommissioned, 10);
25110        assert_eq!(info.bytes_done, 1_024);
25111    }
25112
25113    #[test]
25114    fn test_merge_pool_status_refresh_preserves_observed_v2_floor() {
25115        let timestamp = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
25116        let mut current = PoolMeta {
25117            version: POOL_META_V1_VERSION,
25118            pools: vec![PoolStatus {
25119                id: 0,
25120                cmd_line: "pool-0".to_string(),
25121                last_update: timestamp,
25122                decommission: Some(PoolDecommissionInfo::default()),
25123            }],
25124            dont_save: false,
25125        };
25126        let persisted = PoolMeta {
25127            version: POOL_META_VERSION,
25128            pools: current.pools.clone(),
25129            dont_save: false,
25130        };
25131
25132        assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
25133        assert_eq!(current.version, POOL_META_VERSION);
25134        let encoded = current
25135            .encode_config_data_for_v2_gate(false)
25136            .expect("a peer-observed v2 floor must remain sticky");
25137        assert_eq!(LittleEndian::read_u16(&encoded[2..4]), POOL_META_VERSION);
25138    }
25139
25140    #[test]
25141    fn test_merge_pool_status_refresh_keeps_newer_local_active_over_older_terminal() {
25142        let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
25143        let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
25144        let mut current = PoolMeta {
25145            pools: vec![PoolStatus {
25146                id: 0,
25147                cmd_line: "pool-0".to_string(),
25148                last_update: newer,
25149                decommission: Some(PoolDecommissionInfo {
25150                    start_time: Some(older),
25151                    items_decommissioned: 10,
25152                    bytes_done: 1_024,
25153                    ..Default::default()
25154                }),
25155            }],
25156            ..Default::default()
25157        };
25158        let persisted = PoolMeta {
25159            pools: vec![PoolStatus {
25160                id: 0,
25161                cmd_line: "pool-0".to_string(),
25162                last_update: older,
25163                decommission: Some(PoolDecommissionInfo {
25164                    failed: true,
25165                    ..Default::default()
25166                }),
25167            }],
25168            ..Default::default()
25169        };
25170
25171        assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
25172
25173        let info = current.pools[0]
25174            .decommission
25175            .as_ref()
25176            .expect("local active decommission info should remain present");
25177        assert!(!info.failed);
25178        assert_eq!(info.items_decommissioned, 10);
25179        assert_eq!(info.bytes_done, 1_024);
25180    }
25181
25182    #[test]
25183    fn test_merge_pool_status_refresh_fails_closed_on_missing_persisted_pools() {
25184        let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
25185        let mut current = PoolMeta {
25186            pools: vec![decommission_test_pool_status(
25187                0,
25188                Some(PoolDecommissionInfo {
25189                    complete: true,
25190                    ..Default::default()
25191                }),
25192            )],
25193            ..Default::default()
25194        };
25195        current.pools[0].last_update = newer;
25196
25197        assert!(
25198            !merge_pool_status_refresh(&mut current, PoolMeta::default(), &[false]),
25199            "an empty persisted snapshot must fail closed instead of replacing local state"
25200        );
25201
25202        let info = current.pools[0]
25203            .decommission
25204            .as_ref()
25205            .expect("local decommission info should survive a missing snapshot");
25206        assert!(info.complete);
25207        assert_eq!(current.pools[0].last_update, newer);
25208    }
25209
25210    #[test]
25211    fn test_merge_pool_status_refresh_ignores_mislabeled_pool_entries() {
25212        let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
25213        let mut current = PoolMeta {
25214            pools: vec![decommission_test_pool_status(0, None)],
25215            ..Default::default()
25216        };
25217        let mut persisted = PoolMeta {
25218            pools: vec![decommission_test_pool_status(0, Some(PoolDecommissionInfo::default()))],
25219            ..Default::default()
25220        };
25221        persisted.pools[0].id = 7;
25222        persisted.pools[0].last_update = older;
25223
25224        assert!(
25225            !merge_pool_status_refresh(&mut current, persisted, &[false]),
25226            "a pool entry whose id does not match its index must be ignored"
25227        );
25228        assert!(current.pools[0].decommission.is_none());
25229    }
25230
25231    #[test]
25232    fn test_pool_meta_save_merge_preserves_newer_untouched_pool() {
25233        let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
25234        let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
25235        let mut current = PoolMeta {
25236            pools: vec![
25237                PoolStatus {
25238                    id: 0,
25239                    cmd_line: "pool-0".to_string(),
25240                    last_update: older,
25241                    decommission: Some(PoolDecommissionInfo {
25242                        failed: true,
25243                        ..Default::default()
25244                    }),
25245                },
25246                PoolStatus {
25247                    id: 1,
25248                    cmd_line: "pool-1".to_string(),
25249                    last_update: older,
25250                    decommission: None,
25251                },
25252            ],
25253            ..Default::default()
25254        };
25255        let mut persisted = current.clone();
25256        persisted.pools[1].last_update = newer;
25257        persisted.pools[1].decommission = Some(PoolDecommissionInfo {
25258            failed: true,
25259            ..Default::default()
25260        });
25261
25262        assert!(current.clear_decommission(0).expect("terminal decommission should clear"));
25263        merge_pool_meta_updates_for_save(&mut persisted, &current, &[0], "clear decommission")
25264            .expect("the target pool update should merge into the latest snapshot");
25265
25266        assert!(
25267            persisted.pools[0]
25268                .decommission
25269                .as_ref()
25270                .is_some_and(|info| !info.has_decommission_state())
25271        );
25272        assert_eq!(persisted.pools[1].last_update, newer);
25273        assert!(persisted.pools[1].decommission.as_ref().is_some_and(|info| info.failed));
25274    }
25275
25276    #[test]
25277    fn test_pool_meta_save_merge_rejects_stale_terminal_resurrection() {
25278        let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
25279        let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
25280        let mut persisted = PoolMeta {
25281            pools: vec![PoolStatus {
25282                id: 0,
25283                cmd_line: "pool-0".to_string(),
25284                last_update: newer,
25285                decommission: Some(PoolDecommissionInfo {
25286                    canceled: true,
25287                    ..Default::default()
25288                }),
25289            }],
25290            ..Default::default()
25291        };
25292        let stale = PoolMeta {
25293            pools: vec![PoolStatus {
25294                id: 0,
25295                cmd_line: "pool-0".to_string(),
25296                last_update: older,
25297                decommission: Some(PoolDecommissionInfo {
25298                    start_time: Some(older),
25299                    ..Default::default()
25300                }),
25301            }],
25302            ..Default::default()
25303        };
25304
25305        let err = merge_pool_meta_updates_for_save(&mut persisted, &stale, &[0], "stale writer")
25306            .expect_err("a stale active snapshot must not resurrect a canceled pool");
25307
25308        assert!(err.to_string().contains("stale pool metadata update rejected"));
25309        assert!(persisted.pools[0].decommission.as_ref().is_some_and(|info| info.canceled));
25310    }
25311
25312    #[test]
25313    fn test_pool_meta_save_merge_rejects_clear_over_completed_or_unresolved_state() {
25314        let timestamp = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
25315        for persisted_info in [
25316            PoolDecommissionInfo {
25317                complete: true,
25318                ..Default::default()
25319            },
25320            PoolDecommissionInfo {
25321                failed: true,
25322                unresolved_entries: vec![DecommissionUnresolvedEntry {
25323                    bucket: "bucket-a".to_string(),
25324                    object: "object-a".to_string(),
25325                    pool_index: 0,
25326                    set_index: 0,
25327                    source_generation: timestamp,
25328                    candidate_count: 1,
25329                    disk_error_count: 1,
25330                    observed_at: timestamp,
25331                    reason: "test unresolved entry".to_string(),
25332                }],
25333                ..Default::default()
25334            },
25335        ] {
25336            let mut persisted = PoolMeta {
25337                pools: vec![PoolStatus {
25338                    id: 0,
25339                    cmd_line: "pool-0".to_string(),
25340                    last_update: timestamp,
25341                    decommission: Some(persisted_info),
25342                }],
25343                ..Default::default()
25344            };
25345            let cleared = PoolMeta {
25346                pools: vec![PoolStatus {
25347                    id: 0,
25348                    cmd_line: "pool-0".to_string(),
25349                    last_update: OffsetDateTime::now_utc(),
25350                    decommission: Some(PoolDecommissionInfo::default()),
25351                }],
25352                ..Default::default()
25353            };
25354
25355            let err = merge_pool_meta_updates_for_save(&mut persisted, &cleared, &[0], "clear decommission")
25356                .expect_err("a stale clear must not erase completed or unresolved state");
25357
25358            assert!(
25359                err.to_string()
25360                    .contains("completed or unresolved decommission state cannot be cleared")
25361            );
25362        }
25363    }
25364
25365    #[test]
25366    fn test_pool_meta_publish_preserves_untouched_runtime_progress() {
25367        let mut current = PoolMeta {
25368            pools: vec![
25369                decommission_test_pool_status(0, None),
25370                decommission_test_pool_status(
25371                    1,
25372                    Some(PoolDecommissionInfo {
25373                        items_decommissioned: 10,
25374                        bytes_done: 1_024,
25375                        ..Default::default()
25376                    }),
25377                ),
25378            ],
25379            ..Default::default()
25380        };
25381        let mut saved = current.clone();
25382        saved.pools[0].decommission = Some(PoolDecommissionInfo {
25383            complete: true,
25384            ..Default::default()
25385        });
25386        let saved_pool_1 = saved.pools[1].decommission.as_mut().expect("pool 1 progress should exist");
25387        saved_pool_1.items_decommissioned = 1;
25388        saved_pool_1.bytes_done = 128;
25389
25390        publish_pool_meta_updates(&mut current, &saved, &[0]);
25391
25392        assert!(current.pools[0].decommission.as_ref().is_some_and(|info| info.complete));
25393        let pool_1 = current.pools[1]
25394            .decommission
25395            .as_ref()
25396            .expect("pool 1 progress should remain present");
25397        assert_eq!(pool_1.items_decommissioned, 10);
25398        assert_eq!(pool_1.bytes_done, 1_024);
25399    }
25400
25401    #[test]
25402    fn test_dedup_indices_removes_duplicates_preserving_order() {
25403        assert_eq!(dedup_indices(&[0, 2, 1, 2, 3, 0]), vec![0, 2, 1, 3]);
25404    }
25405
25406    #[test]
25407    fn test_dedup_indices_handles_empty_input() {
25408        let empty: Vec<usize> = Vec::new();
25409        assert!(dedup_indices(&empty).is_empty());
25410    }
25411
25412    #[test]
25413    fn test_default_decommission_bucket_concurrency_is_conservative() {
25414        assert_eq!(default_decommission_bucket_concurrency(0), 1);
25415        assert_eq!(default_decommission_bucket_concurrency(1), 1);
25416        assert_eq!(default_decommission_bucket_concurrency(2), 2);
25417        assert_eq!(default_decommission_bucket_concurrency(8), 4);
25418    }
25419
25420    #[test]
25421    fn test_default_decommission_entry_concurrency_is_conservative() {
25422        assert_eq!(default_decommission_entry_concurrency(0), 1);
25423        assert_eq!(default_decommission_entry_concurrency(1), 1);
25424        assert_eq!(default_decommission_entry_concurrency(4), 4);
25425        assert_eq!(default_decommission_entry_concurrency(16), DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP);
25426    }
25427
25428    #[test]
25429    fn test_decommission_entry_concurrency_clamps_operator_configuration() {
25430        assert_eq!(clamp_decommission_entry_concurrency(0), 1);
25431        assert_eq!(clamp_decommission_entry_concurrency(1), 1);
25432        assert_eq!(
25433            clamp_decommission_entry_concurrency(DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP),
25434            DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP
25435        );
25436        assert_eq!(clamp_decommission_entry_concurrency(usize::MAX), DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP);
25437    }
25438
25439    #[test]
25440    fn test_split_decommission_buckets_keeps_meta_buckets_last() {
25441        let (regular, meta) = split_decommission_buckets(vec![
25442            DecomBucketInfo {
25443                name: "bucket-a".to_string(),
25444                ..Default::default()
25445            },
25446            DecomBucketInfo {
25447                name: crate::disk::RUSTFS_META_BUCKET.to_string(),
25448                prefix: crate::config::com::CONFIG_PREFIX.to_string(),
25449            },
25450            DecomBucketInfo {
25451                name: "bucket-b".to_string(),
25452                ..Default::default()
25453            },
25454            DecomBucketInfo {
25455                name: crate::disk::RUSTFS_META_BUCKET.to_string(),
25456                prefix: crate::disk::BUCKET_META_PREFIX.to_string(),
25457            },
25458            DecomBucketInfo {
25459                name: crate::disk::RUSTFS_META_BUCKET.to_string(),
25460                prefix: crate::bucket::lifecycle::ILM_META_PREFIX.to_string(),
25461            },
25462        ]);
25463
25464        assert_eq!(
25465            regular.iter().map(|bucket| bucket.name.as_str()).collect::<Vec<_>>(),
25466            vec!["bucket-a", "bucket-b",]
25467        );
25468        assert_eq!(
25469            meta.iter().map(|bucket| bucket.prefix.as_str()).collect::<Vec<_>>(),
25470            vec![
25471                crate::config::com::CONFIG_PREFIX,
25472                crate::disk::BUCKET_META_PREFIX,
25473                crate::bucket::lifecycle::ILM_META_PREFIX,
25474            ]
25475        );
25476    }
25477
25478    #[test]
25479    fn test_resume_reconciles_missing_decommission_meta_prefixes() {
25480        let mut meta = PoolMeta {
25481            pools: vec![decommission_test_pool_status(
25482                0,
25483                Some(PoolDecommissionInfo {
25484                    queued_buckets: vec![
25485                        format!("{}/{}", crate::disk::RUSTFS_META_BUCKET, crate::config::com::CONFIG_PREFIX),
25486                        format!("{}/{}", crate::disk::RUSTFS_META_BUCKET, crate::disk::BUCKET_META_PREFIX),
25487                    ],
25488                    ..Default::default()
25489                }),
25490            )],
25491            ..Default::default()
25492        };
25493
25494        assert!(reconcile_decommission_meta_buckets(&mut meta, 0));
25495        assert_eq!(
25496            meta.pending_buckets(0)
25497                .iter()
25498                .filter(|bucket| bucket.name == crate::disk::RUSTFS_META_BUCKET)
25499                .map(|bucket| bucket.prefix.as_str())
25500                .collect::<Vec<_>>(),
25501            DECOMMISSION_META_PREFIXES
25502        );
25503        assert!(!reconcile_decommission_meta_buckets(&mut meta, 0));
25504    }
25505
25506    #[tokio::test]
25507    async fn test_decommission_metadata_phase_precedes_regular_failure() {
25508        let events = Arc::new(StdMutex::new(Vec::new()));
25509        let err = run_decommission_phases(
25510            CancellationToken::new(),
25511            vec![
25512                DecomBucketInfo {
25513                    name: "regular-fails".to_string(),
25514                    ..Default::default()
25515                },
25516                DecomBucketInfo {
25517                    name: "regular-not-started".to_string(),
25518                    ..Default::default()
25519                },
25520            ],
25521            vec![
25522                DecomBucketInfo {
25523                    name: crate::disk::RUSTFS_META_BUCKET.to_string(),
25524                    prefix: crate::config::com::CONFIG_PREFIX.to_string(),
25525                },
25526                DecomBucketInfo {
25527                    name: crate::disk::RUSTFS_META_BUCKET.to_string(),
25528                    prefix: crate::disk::BUCKET_META_PREFIX.to_string(),
25529                },
25530            ],
25531            1,
25532            {
25533                let events = Arc::clone(&events);
25534                move |bucket, _rx| {
25535                    let events = Arc::clone(&events);
25536                    Box::pin(async move {
25537                        let event = if bucket.name == crate::disk::RUSTFS_META_BUCKET {
25538                            format!("meta:{}", bucket.prefix)
25539                        } else {
25540                            format!("regular:{}", bucket.name)
25541                        };
25542                        events.lock().expect("phase event lock should not be poisoned").push(event);
25543                        if bucket.name == "regular-fails" {
25544                            Err(Error::SlowDown)
25545                        } else {
25546                            Ok(())
25547                        }
25548                    })
25549                }
25550            },
25551        )
25552        .await
25553        .expect_err("regular failure should remain fatal after metadata completes");
25554
25555        assert!(matches!(err, Error::SlowDown));
25556        assert_eq!(
25557            *events.lock().expect("phase event lock should not be poisoned"),
25558            vec![
25559                format!("meta:{}", crate::config::com::CONFIG_PREFIX),
25560                format!("meta:{}", crate::disk::BUCKET_META_PREFIX),
25561                "regular:regular-fails".to_string(),
25562            ]
25563        );
25564    }
25565
25566    #[tokio::test]
25567    async fn test_run_decommission_buckets_bounded_respects_limit() {
25568        let rx = CancellationToken::new();
25569        let running = Arc::new(AtomicUsize::new(0));
25570        let max_running = Arc::new(AtomicUsize::new(0));
25571        let started = Arc::new(AtomicUsize::new(0));
25572        let buckets = (0..8)
25573            .map(|idx| DecomBucketInfo {
25574                name: format!("bucket-{idx}"),
25575                ..Default::default()
25576            })
25577            .collect::<Vec<_>>();
25578
25579        run_decommission_buckets_bounded(rx, buckets, 2, {
25580            let running = Arc::clone(&running);
25581            let max_running = Arc::clone(&max_running);
25582            let started = Arc::clone(&started);
25583            move |_bucket, _rx| {
25584                let running = Arc::clone(&running);
25585                let max_running = Arc::clone(&max_running);
25586                let started = Arc::clone(&started);
25587                Box::pin(async move {
25588                    started.fetch_add(1, Ordering::SeqCst);
25589                    let current = running.fetch_add(1, Ordering::SeqCst) + 1;
25590                    max_running.fetch_max(current, Ordering::SeqCst);
25591                    tokio::time::sleep(StdDuration::from_millis(10)).await;
25592                    running.fetch_sub(1, Ordering::SeqCst);
25593                    Ok(())
25594                })
25595            }
25596        })
25597        .await
25598        .expect("bounded bucket scheduler should complete");
25599
25600        assert_eq!(started.load(Ordering::SeqCst), 8);
25601        assert_eq!(max_running.load(Ordering::SeqCst), 2);
25602        assert_eq!(running.load(Ordering::SeqCst), 0);
25603    }
25604
25605    #[tokio::test]
25606    async fn test_run_decommission_buckets_bounded_cancels_and_stops_launching_after_failure() {
25607        let rx = CancellationToken::new();
25608        let started = Arc::new(AtomicUsize::new(0));
25609        let observed_cancel = Arc::new(AtomicBool::new(false));
25610        let buckets = (0..5)
25611            .map(|idx| DecomBucketInfo {
25612                name: format!("bucket-{idx}"),
25613                ..Default::default()
25614            })
25615            .collect::<Vec<_>>();
25616
25617        let err = tokio::time::timeout(
25618            StdDuration::from_secs(2),
25619            run_decommission_buckets_bounded(rx.clone(), buckets, 2, {
25620                let started = Arc::clone(&started);
25621                let observed_cancel = Arc::clone(&observed_cancel);
25622                move |bucket, rx| {
25623                    let started = Arc::clone(&started);
25624                    let observed_cancel = Arc::clone(&observed_cancel);
25625                    Box::pin(async move {
25626                        started.fetch_add(1, Ordering::SeqCst);
25627                        if bucket.name == "bucket-0" {
25628                            while started.load(Ordering::SeqCst) < 2 {
25629                                tokio::task::yield_now().await;
25630                            }
25631                            return Err(Error::SlowDown);
25632                        }
25633
25634                        rx.cancelled().await;
25635                        observed_cancel.store(true, Ordering::SeqCst);
25636                        Ok(())
25637                    })
25638                }
25639            }),
25640        )
25641        .await
25642        .expect("bucket scheduler should not hang after a bucket failure")
25643        .expect_err("first bucket failure should be returned");
25644
25645        assert!(matches!(err, Error::SlowDown));
25646        assert!(rx.is_cancelled());
25647        assert!(observed_cancel.load(Ordering::SeqCst));
25648        assert_eq!(started.load(Ordering::SeqCst), 2);
25649    }
25650
25651    #[tokio::test]
25652    async fn test_run_decommission_buckets_bounded_external_cancel_stops_pending_buckets() {
25653        let rx = CancellationToken::new();
25654        let started = Arc::new(AtomicUsize::new(0));
25655        let buckets = (0..4)
25656            .map(|idx| DecomBucketInfo {
25657                name: format!("bucket-{idx}"),
25658                ..Default::default()
25659            })
25660            .collect::<Vec<_>>();
25661
25662        let err = run_decommission_buckets_bounded(rx.clone(), buckets, 1, {
25663            let started = Arc::clone(&started);
25664            move |_bucket, rx| {
25665                let started = Arc::clone(&started);
25666                Box::pin(async move {
25667                    started.fetch_add(1, Ordering::SeqCst);
25668                    rx.cancel();
25669                    Ok(())
25670                })
25671            }
25672        })
25673        .await
25674        .expect_err("external cancellation with pending buckets should stop the scheduler");
25675
25676        assert!(matches!(err, Error::OperationCanceled));
25677        assert!(rx.is_cancelled());
25678        assert_eq!(started.load(Ordering::SeqCst), 1);
25679    }
25680
25681    #[tokio::test]
25682    async fn test_wait_decommission_worker_drain_waits_for_entry_permit() {
25683        let workers = Arc::new(Semaphore::new(1));
25684        let permit = workers
25685            .clone()
25686            .acquire_owned()
25687            .await
25688            .expect("test worker permit should acquire");
25689
25690        let drain = tokio::spawn({
25691            let workers = workers.clone();
25692            async move { wait_decommission_worker_drain(&workers, 1).await }
25693        });
25694
25695        tokio::task::yield_now().await;
25696        assert!(!drain.is_finished(), "drain should wait while a worker permit is held");
25697
25698        drop(permit);
25699        let result = tokio::time::timeout(StdDuration::from_secs(1), drain)
25700            .await
25701            .expect("drain should finish after permit release")
25702            .expect("drain task should not panic");
25703        assert!(result.is_ok());
25704    }
25705
25706    #[test]
25707    fn test_decommission_entry_queue_capacity_is_bounded() {
25708        assert_eq!(decommission_entry_queue_capacity(0), 1);
25709        assert_eq!(decommission_entry_queue_capacity(1), 2);
25710        assert_eq!(
25711            decommission_entry_queue_capacity(DECOMMISSION_ENTRY_QUEUE_HARD_CAP),
25712            DECOMMISSION_ENTRY_QUEUE_HARD_CAP
25713        );
25714        assert_eq!(decommission_entry_queue_capacity(usize::MAX), DECOMMISSION_ENTRY_QUEUE_HARD_CAP);
25715    }
25716
25717    #[tokio::test]
25718    async fn test_drain_decommission_entry_queue_waits_for_all_outstanding_entries() {
25719        let outstanding = Arc::new(Semaphore::new(1));
25720        let held = outstanding
25721            .clone()
25722            .acquire_owned()
25723            .await
25724            .expect("test outstanding permit should acquire");
25725        let rx = CancellationToken::new();
25726        let drain = tokio::spawn({
25727            let outstanding = outstanding.clone();
25728            let rx = rx.clone();
25729            async move { drain_decommission_entry_queue(&rx, &outstanding, 1).await }
25730        });
25731
25732        tokio::task::yield_now().await;
25733        assert!(!drain.is_finished(), "queue drain must wait for active entry work");
25734        drop(held);
25735
25736        let drained = tokio::time::timeout(StdDuration::from_secs(1), drain)
25737            .await
25738            .expect("queue drain should finish after entry completion")
25739            .expect("queue drain task should not panic");
25740        assert!(!drained);
25741    }
25742
25743    #[tokio::test]
25744    async fn test_enqueue_decommission_entry_observes_cancellation_when_queue_is_full() {
25745        let outstanding = Arc::new(Semaphore::new(2));
25746        let (tx, mut queue) = tokio::sync::mpsc::channel(1);
25747        let held = outstanding
25748            .clone()
25749            .acquire_owned()
25750            .await
25751            .expect("first queue permit should acquire");
25752        tx.send(QueuedDecommissionEntry {
25753            entry: MetaCacheEntry::default(),
25754            queue_permit: held,
25755        })
25756        .await
25757        .expect("first entry should fill the queue");
25758
25759        let rx = CancellationToken::new();
25760        let enqueue = tokio::spawn({
25761            let rx = rx.clone();
25762            let outstanding = outstanding.clone();
25763            let tx = tx.clone();
25764            async move { enqueue_decommission_entry(&rx, &outstanding, &tx, MetaCacheEntry::default()).await }
25765        });
25766
25767        tokio::task::yield_now().await;
25768        rx.cancel();
25769        let result = tokio::time::timeout(StdDuration::from_secs(1), enqueue)
25770            .await
25771            .expect("full queue enqueue should observe cancellation")
25772            .expect("enqueue task should not panic");
25773        assert!(matches!(result, DecommissionEntryEnqueueResult::Canceled));
25774        drop(queue.recv().await);
25775    }
25776
25777    #[tokio::test]
25778    async fn test_decommission_side_effect_gate_quiesces_before_transition() {
25779        let operation_gate = Arc::new(tokio::sync::RwLock::new(()));
25780        let rx = CancellationToken::new();
25781        let started = Arc::new(tokio::sync::Notify::new());
25782        let release = Arc::new(tokio::sync::Notify::new());
25783        let operation = tokio::spawn({
25784            let operation_gate = operation_gate.clone();
25785            let rx = rx.clone();
25786            let started = started.clone();
25787            let release = release.clone();
25788            async move {
25789                run_decommission_side_effect(&rx, &operation_gate, || async {
25790                    started.notify_one();
25791                    release.notified().await;
25792                    Ok::<_, Error>(())
25793                })
25794                .await
25795            }
25796        });
25797
25798        started.notified().await;
25799        rx.cancel();
25800        let transition = tokio::spawn({
25801            let operation_gate = operation_gate.clone();
25802            async move {
25803                let _guard = operation_gate.write().await;
25804            }
25805        });
25806        tokio::task::yield_now().await;
25807        assert!(!transition.is_finished(), "transition must wait for the in-flight side effect");
25808
25809        release.notify_one();
25810        let operation_result = operation.await.expect("operation task should not panic");
25811        assert!(matches!(operation_result, Err(Error::OperationCanceled)));
25812        transition.await.expect("transition task should not panic");
25813
25814        let called = Arc::new(AtomicBool::new(false));
25815        let result = run_decommission_side_effect(&rx, &operation_gate, {
25816            let called = called.clone();
25817            move || async move {
25818                called.store(true, Ordering::SeqCst);
25819                Ok::<_, Error>(())
25820            }
25821        })
25822        .await;
25823        assert!(matches!(result, Err(Error::OperationCanceled)));
25824        assert!(!called.load(Ordering::SeqCst));
25825    }
25826
25827    #[tokio::test]
25828    async fn test_decommission_side_effect_stops_after_pool_meta_write_block() {
25829        let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
25830        store
25831            .pool_meta_save_gate
25832            .lock()
25833            .await
25834            .observe_replicas(super::PoolMetaReplicaState {
25835                needs_repair: true,
25836                repair_write_safe: false,
25837            });
25838        let called = Arc::new(AtomicBool::new(false));
25839        let operation_gate = store.ctx.data_movement_operation_gate();
25840
25841        let result = store
25842            .run_guarded_decommission_side_effect(&CancellationToken::new(), &operation_gate, {
25843                let called = called.clone();
25844                move || async move {
25845                    called.store(true, Ordering::SeqCst);
25846                    Ok::<_, Error>(())
25847                }
25848            })
25849            .await;
25850
25851        assert!(
25852            result
25853                .expect_err("sticky pool metadata state must block new movement")
25854                .to_string()
25855                .contains("restart after all replicas are readable and consistent")
25856        );
25857        assert!(!called.load(Ordering::SeqCst));
25858    }
25859
25860    #[tokio::test]
25861    async fn test_decommission_reservation_stops_before_canceler_slot_when_pool_meta_is_blocked() {
25862        let generation = OffsetDateTime::UNIX_EPOCH;
25863        let pool_meta = PoolMeta {
25864            version: super::POOL_META_VERSION,
25865            pools: vec![decommission_test_pool_status(
25866                0,
25867                Some(PoolDecommissionInfo {
25868                    start_time: Some(generation),
25869                    ..Default::default()
25870                }),
25871            )],
25872            ..Default::default()
25873        };
25874        let store = decommission_worker_test_store(pool_meta, vec![None]);
25875        store
25876            .pool_meta_save_gate
25877            .lock()
25878            .await
25879            .observe_replicas(super::PoolMetaReplicaState {
25880                needs_repair: true,
25881                repair_write_safe: false,
25882            });
25883
25884        let err = store
25885            .reserve_decommission_routines(&CancellationToken::new(), &[0])
25886            .await
25887            .err()
25888            .expect("sticky pool metadata state must block worker reservation");
25889
25890        assert!(
25891            err.to_string()
25892                .contains("restart after all replicas are readable and consistent")
25893        );
25894        assert!(store.decommission_cancelers.read().await[0].is_none());
25895    }
25896
25897    #[tokio::test]
25898    async fn test_v1_unresolved_ledger_rejection_keeps_live_state_and_write_gate_safe() {
25899        let generation = OffsetDateTime::UNIX_EPOCH;
25900        let status = decommission_test_pool_status(
25901            0,
25902            Some(PoolDecommissionInfo {
25903                start_time: Some(generation),
25904                ..Default::default()
25905            }),
25906        );
25907        let last_update = status.last_update;
25908        let store = decommission_worker_test_store(
25909            PoolMeta {
25910                version: POOL_META_V1_VERSION,
25911                pools: vec![status],
25912                ..Default::default()
25913            },
25914            vec![None],
25915        );
25916        let entry = DecommissionUnresolvedEntry {
25917            bucket: "bucket-a".to_string(),
25918            object: "directory/".to_string(),
25919            pool_index: 0,
25920            set_index: 0,
25921            source_generation: generation,
25922            candidate_count: 1,
25923            disk_error_count: 0,
25924            observed_at: generation,
25925            reason: "metadata_resolution_failed".to_string(),
25926        };
25927
25928        let err = store
25929            .persist_decommission_unresolved_entry(0, generation, entry)
25930            .await
25931            .expect_err("V1 must reject the ledger before changing live state");
25932
25933        assert!(matches!(err, Error::InvalidArgument(..)));
25934        let pool_meta = store.pool_meta.read().await;
25935        let status = &pool_meta.pools[0];
25936        assert_eq!(status.last_update, last_update);
25937        assert!(
25938            status
25939                .decommission
25940                .as_ref()
25941                .expect("active decommission metadata should remain present")
25942                .unresolved_entries
25943                .is_empty()
25944        );
25945        drop(pool_meta);
25946        store
25947            .pool_meta_save_gate
25948            .lock()
25949            .await
25950            .ensure_write_safe("V1 unresolved-entry preflight")
25951            .expect("a deterministic capability rejection must not latch recovery");
25952    }
25953
25954    #[tokio::test]
25955    async fn test_v1_runtime_recovery_rejects_worker_but_keeps_cancel_persistable() {
25956        let generation = OffsetDateTime::UNIX_EPOCH;
25957        let store = decommission_worker_test_store(
25958            PoolMeta {
25959                version: POOL_META_V1_VERSION,
25960                pools: vec![decommission_test_pool_status(
25961                    0,
25962                    Some(PoolDecommissionInfo {
25963                        start_time: Some(generation),
25964                        ..Default::default()
25965                    }),
25966                )],
25967                ..Default::default()
25968            },
25969            vec![None],
25970        );
25971
25972        let err = store
25973            .reserve_decommission_routines(&CancellationToken::new(), &[0])
25974            .await
25975            .err()
25976            .expect("V1 recovery must not install a worker that cannot persist an unresolved ledger");
25977        assert!(matches!(err, Error::InvalidArgument(..)));
25978        assert!(store.decommission_cancelers.read().await[0].is_none());
25979
25980        let save_called = Arc::new(AtomicBool::new(false));
25981        store
25982            .decommission_cancel_with_owner_and_save(0, None, {
25983                let save_called = save_called.clone();
25984                move |snapshot, _| async move {
25985                    snapshot.encode_config_data_for_v2_gate(false)?;
25986                    save_called.store(true, Ordering::SeqCst);
25987                    Ok(())
25988                }
25989            })
25990            .await
25991            .expect("a rejected V1 recovery must remain cancelable without restart");
25992
25993        assert!(save_called.load(Ordering::SeqCst));
25994        let pool_meta = store.pool_meta.read().await;
25995        let info = pool_meta.pools[0]
25996            .decommission
25997            .as_ref()
25998            .expect("cancel metadata should remain present");
25999        assert!(info.canceled);
26000        assert!(!info.failed);
26001        assert!(!info.complete);
26002    }
26003
26004    #[tokio::test]
26005    async fn test_decommission_transition_waits_without_registered_canceler() {
26006        let store = decommission_worker_test_store(PoolMeta::default(), vec![None]);
26007        let operation_gate = store.ctx.data_movement_operation_gate();
26008        let operation_guard = operation_gate.read().await;
26009        let transition = tokio::spawn({
26010            let store = store.clone();
26011            async move { store.cancel_decommission_routines_and_wait(&[0]).await }
26012        });
26013
26014        tokio::task::yield_now().await;
26015        assert!(
26016            !transition.is_finished(),
26017            "a transition must wait for an in-flight side effect even after its canceler slot is gone"
26018        );
26019
26020        drop(operation_guard);
26021        tokio::time::timeout(StdDuration::from_secs(1), transition)
26022            .await
26023            .expect("transition should finish after the side effect")
26024            .expect("transition task should not panic");
26025    }
26026
26027    #[tokio::test]
26028    async fn test_join_error_quiesces_side_effects_before_failure_transition() {
26029        let canceler = DecommissionCanceler::new(CancellationToken::new());
26030        let store = decommission_worker_test_store(PoolMeta::default(), vec![Some(canceler.clone())]);
26031        let operation_gate = store.ctx.data_movement_operation_gate();
26032        let operation_guard = operation_gate.read().await;
26033        let transition = tokio::spawn({
26034            let store = store.clone();
26035            let canceler = canceler.clone();
26036            async move { store.quiesce_decommission_worker_after_join_error(&canceler).await }
26037        });
26038
26039        tokio::time::timeout(StdDuration::from_secs(1), canceler.token().cancelled())
26040            .await
26041            .expect("join error handling should cancel the detached worker");
26042        assert!(
26043            !transition.is_finished(),
26044            "failure transition must wait for the detached worker's in-flight side effect"
26045        );
26046
26047        drop(operation_guard);
26048        tokio::time::timeout(StdDuration::from_secs(1), transition)
26049            .await
26050            .expect("failure transition should finish after the side effect")
26051            .expect("failure transition task should not panic");
26052    }
26053
26054    #[tokio::test(start_paused = true)]
26055    async fn test_run_decommission_listing_with_retry_drains_before_each_retry() {
26056        let attempts = Arc::new(AtomicUsize::new(0));
26057        let drains = Arc::new(AtomicUsize::new(0));
26058        let err = run_decommission_listing_with_retry_and_drain(
26059            CancellationToken::new(),
26060            "bucket-a".to_string(),
26061            noop_decommission_list_callback(),
26062            1,
26063            2,
26064            2,
26065            {
26066                let attempts = attempts.clone();
26067                move |_| {
26068                    let attempts = attempts.clone();
26069                    async move {
26070                        attempts.fetch_add(1, Ordering::SeqCst);
26071                        Err(Error::SlowDown)
26072                    }
26073                }
26074            },
26075            {
26076                let drains = drains.clone();
26077                move || {
26078                    let drains = drains.clone();
26079                    async move {
26080                        drains.fetch_add(1, Ordering::SeqCst);
26081                        false
26082                    }
26083                }
26084            },
26085        )
26086        .await
26087        .expect_err("permanent listing failure must be returned");
26088
26089        assert!(err.to_string().contains("attempt 2/2"));
26090        assert_eq!(attempts.load(Ordering::SeqCst), 2);
26091        assert_eq!(drains.load(Ordering::SeqCst), 2);
26092    }
26093
26094    #[test]
26095    fn test_get_by_index_returns_value_when_in_range() {
26096        let values = vec!["a", "b", "c"];
26097        let value = get_by_index(values.as_slice(), 1, "fetch decommission status").expect("in-range index should return value");
26098        assert_eq!(*value, "b");
26099    }
26100
26101    #[test]
26102    fn test_get_by_index_returns_error_when_out_of_range() {
26103        let values = vec![1_u8];
26104        let err =
26105            get_by_index(values.as_slice(), 2, "load decommission background pool").expect_err("out-of-range index should fail");
26106        assert!(
26107            err.to_string()
26108                .contains("failed to load decommission background pool: invalid decommission pool index 2 for 1 pools")
26109        );
26110    }
26111
26112    #[test]
26113    fn test_pool_meta_is_suspended_returns_false_for_out_of_range() {
26114        let meta = PoolMeta::default();
26115        assert!(!meta.is_suspended(1));
26116    }
26117
26118    #[test]
26119    fn test_rollback_start_decommission_pool_meta_clears_active_state() {
26120        let previous = PoolMeta {
26121            pools: vec![PoolStatus {
26122                id: 0,
26123                cmd_line: "pool-0".to_string(),
26124                last_update: OffsetDateTime::UNIX_EPOCH,
26125                decommission: None,
26126            }],
26127            ..Default::default()
26128        };
26129        let mut active = previous.clone();
26130        let active_update = OffsetDateTime::UNIX_EPOCH + Duration::seconds(1);
26131        active.pools[0].last_update = active_update;
26132        active.pools[0].decommission = Some(PoolDecommissionInfo {
26133            start_time: Some(OffsetDateTime::UNIX_EPOCH),
26134            ..Default::default()
26135        });
26136        let mut peer = active.clone();
26137
26138        assert!(active.is_suspended(0));
26139        assert_eq!(
26140            decommission_start_pool_state(active.pools.first()),
26141            DecommissionStartPoolState::Decommissioning
26142        );
26143
26144        rollback_start_decommission_pool_meta(&mut active, &previous, &[0]);
26145
26146        assert!(!active.is_suspended(0));
26147        assert_eq!(decommission_start_pool_state(active.pools.first()), DecommissionStartPoolState::Active);
26148        assert!(active.pools[0].last_update > active_update);
26149        assert!(merge_pool_status_refresh(&mut peer, active, &[false]));
26150        assert!(peer.pools[0].decommission.is_none());
26151    }
26152
26153    #[test]
26154    fn test_pool_meta_queue_buckets_ignores_out_of_range_index() {
26155        let mut meta = PoolMeta {
26156            pools: vec![PoolStatus {
26157                id: 0,
26158                cmd_line: "pool-0".to_string(),
26159                last_update: OffsetDateTime::UNIX_EPOCH,
26160                decommission: Some(PoolDecommissionInfo::default()),
26161            }],
26162            ..Default::default()
26163        };
26164
26165        meta.queue_buckets(
26166            9,
26167            vec![DecomBucketInfo {
26168                name: "bucket-a".to_string(),
26169                prefix: String::new(),
26170            }],
26171        );
26172
26173        let queued = meta.pools[0]
26174            .decommission
26175            .as_ref()
26176            .expect("pool should have decommission info")
26177            .queued_buckets
26178            .clone();
26179        assert!(queued.is_empty());
26180    }
26181
26182    #[test]
26183    fn test_pool_meta_is_bucket_decommissioned_returns_false_for_out_of_range() {
26184        let meta = PoolMeta::default();
26185        assert!(!meta.is_bucket_decommissioned(7, "bucket-a".to_string()));
26186    }
26187
26188    #[test]
26189    fn test_resolve_decommission_bucket_state_rejects_out_of_range_index() {
26190        let meta = PoolMeta {
26191            pools: vec![PoolStatus {
26192                id: 0,
26193                cmd_line: "pool-0".to_string(),
26194                last_update: OffsetDateTime::UNIX_EPOCH,
26195                decommission: Some(PoolDecommissionInfo::default()),
26196            }],
26197            ..Default::default()
26198        };
26199
26200        let bucket = DecomBucketInfo {
26201            name: "bucket-a".to_string(),
26202            prefix: String::new(),
26203        };
26204        let err =
26205            resolve_decommission_bucket_state(&meta, 3, &bucket).expect_err("out-of-range index should return invalid argument");
26206        assert!(err.to_string().contains("invalid decommission pool index 3 for 1 pools"));
26207    }
26208
26209    #[test]
26210    fn test_resolve_decommission_bucket_state_rejects_missing_decommission_meta() {
26211        let meta = PoolMeta {
26212            pools: vec![PoolStatus {
26213                id: 0,
26214                cmd_line: "pool-0".to_string(),
26215                last_update: OffsetDateTime::UNIX_EPOCH,
26216                decommission: None,
26217            }],
26218            ..Default::default()
26219        };
26220
26221        let bucket = DecomBucketInfo {
26222            name: "bucket-a".to_string(),
26223            prefix: String::new(),
26224        };
26225        let err = resolve_decommission_bucket_state(&meta, 0, &bucket)
26226            .expect_err("missing decommission metadata should return explicit error");
26227        assert!(
26228            err.to_string()
26229                .contains("failed to resolve decommission bucket state: decommission metadata not initialized")
26230        );
26231    }
26232
26233    #[test]
26234    fn test_resolve_decommission_bucket_state_returns_true_for_done_bucket() {
26235        let meta = PoolMeta {
26236            pools: vec![PoolStatus {
26237                id: 0,
26238                cmd_line: "pool-0".to_string(),
26239                last_update: OffsetDateTime::UNIX_EPOCH,
26240                decommission: Some(PoolDecommissionInfo {
26241                    decommissioned_buckets: vec!["bucket-a".to_string()],
26242                    ..Default::default()
26243                }),
26244            }],
26245            ..Default::default()
26246        };
26247
26248        let bucket = DecomBucketInfo {
26249            name: "bucket-a".to_string(),
26250            prefix: String::new(),
26251        };
26252        let done = resolve_decommission_bucket_state(&meta, 0, &bucket).expect("valid state should resolve");
26253        assert!(done);
26254    }
26255
26256    #[test]
26257    fn test_decommission_capacity_publication_preserves_progress_and_checkpoint_deadline() {
26258        let mut current = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_MODEL_VERSION]);
26259        let start = OffsetDateTime::UNIX_EPOCH;
26260        current.pools[0].last_update = start;
26261        current.count_item(0, 17, false);
26262        let mut saved = current.clone();
26263        saved.pools[0]
26264            .decommission
26265            .as_mut()
26266            .expect("persisted source")
26267            .items_decommissioned = 0;
26268        saved.pools[0].decommission.as_mut().expect("persisted source").bytes_done = 0;
26269        for seconds in [10, 20, 29] {
26270            saved.pools[0].last_update = start + Duration::seconds(seconds);
26271            super::publish_decommission_capacity_update(&mut current, &saved, 0).expect("same-owner capacity publication");
26272        }
26273        let info = current.pools[0].decommission.as_ref().expect("live source");
26274        assert_eq!((info.items_decommissioned, info.bytes_done, info.progress_save_item_baseline), (1, 17, 0));
26275        assert_eq!(info.progress_save_last_at, Some(start));
26276        let checkpoint = current
26277            .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, start + Duration::seconds(30), None)
26278            .expect("checkpoint should remain due despite capacity writes")
26279            .expect("capacity updates must not postpone the 30-second progress checkpoint");
26280        current.count_item(0, 19, false);
26281        assert!(current.commit_decommission_progress_checkpoint(0, checkpoint));
26282        let info = current.pools[0].decommission.as_ref().expect("live source");
26283        assert_eq!(
26284            info.items_since_last_progress_save(),
26285            1,
26286            "only the committed snapshot advances the watermark"
26287        );
26288        assert_eq!(info.progress_save_last_at, Some(checkpoint.checkpoint_at));
26289        assert!(
26290            current
26291                .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, start + Duration::seconds(31), None)
26292                .expect("next checkpoint timing")
26293                .is_none()
26294        );
26295    }
26296
26297    #[test]
26298    fn test_decommission_capacity_publication_rejects_changed_identity_without_mutation() {
26299        let baseline = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_MODEL_VERSION]);
26300        for changed_field in [
26301            "layout",
26302            "start",
26303            "queued",
26304            "terminal",
26305            "operation",
26306            "generation",
26307            "owner",
26308            "model",
26309            "cleared",
26310        ] {
26311            let mut current = baseline.clone();
26312            let mut saved = baseline.clone();
26313            saved.pools[0].last_update += Duration::seconds(1);
26314            let pool = &mut current.pools[0];
26315            let info = pool.decommission.as_mut().expect("current source");
26316            match changed_field {
26317                "layout" => pool.cmd_line.push_str("-replaced"),
26318                "start" => info.start_time = Some(OffsetDateTime::UNIX_EPOCH + Duration::seconds(1)),
26319                "queued" => info.queued = true,
26320                "terminal" => info.canceled = true,
26321                "operation" => info.capacity_reservation.as_mut().expect("owner").operation_id = uuid::Uuid::new_v4(),
26322                "generation" => info.capacity_reservation.as_mut().expect("owner").generation += 1,
26323                "owner" => info.capacity_reservation.as_mut().expect("owner").owner_nonce = uuid::Uuid::new_v4(),
26324                "model" => info.capacity_reservation.as_mut().expect("owner").model_version += 1,
26325                "cleared" => pool.decommission = None,
26326                _ => unreachable!("fixed test matrix"),
26327            }
26328            let before = format!("{current:?}");
26329            let err = super::publish_decommission_capacity_update(&mut current, &saved, 0)
26330                .expect_err("changed operation identity must reject publication");
26331            assert!(matches!(err, Error::StalePoolMetadataUpdate { .. }), "{changed_field}: {err}");
26332            assert_eq!(format!("{current:?}"), before, "{changed_field}: rejected publication mutated live state");
26333        }
26334    }
26335
26336    #[test]
26337    fn test_decommission_capacity_revision_does_not_regress_when_clock_moves_backwards() {
26338        let mut meta = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_MODEL_VERSION, Vec::new(), 0, None);
26339        let revision = meta.pools[0].last_update;
26340        let mutation = uuid::Uuid::new_v4();
26341        reserve_decommission_target_pending(&mut meta, 0, 1, 1, mutation, OffsetDateTime::UNIX_EPOCH)
26342            .expect("clock rollback must not prevent a valid owned reservation");
26343        assert_eq!(meta.pools[0].last_update, revision);
26344        resolve_decommission_target_pending(&mut meta, 0, 1, 1, mutation).expect("resolve the exact intent");
26345        record_decommission_target_consumption(
26346            &mut meta,
26347            0,
26348            1,
26349            DecommissionTargetConsumption {
26350                committed_data_bytes: 1,
26351                target_physical_bytes: 1,
26352                observed_physical_bytes: 1,
26353            },
26354            mutation,
26355            OffsetDateTime::UNIX_EPOCH,
26356        )
26357        .expect("commit consumption during clock rollback");
26358        assert_eq!(meta.pools[0].last_update, revision);
26359    }
26360
26361    #[test]
26362    fn test_mark_decommission_bucket_done_is_idempotent_and_ignores_unknown_buckets() {
26363        let bucket = DecomBucketInfo {
26364            name: "bucket-a".to_string(),
26365            prefix: String::new(),
26366        };
26367        let unknown = DecomBucketInfo {
26368            name: "bucket-b".to_string(),
26369            prefix: String::new(),
26370        };
26371        let mut meta = PoolMeta {
26372            pools: vec![decommission_test_pool_status(
26373                0,
26374                Some(PoolDecommissionInfo {
26375                    queued_buckets: vec![bucket.to_string()],
26376                    ..Default::default()
26377                }),
26378            )],
26379            ..Default::default()
26380        };
26381        assert!(!mark_decommission_bucket_done(&mut meta, 0, &unknown).expect("an unknown bucket is not completed"));
26382        assert!(
26383            meta.pools[0]
26384                .decommission
26385                .as_ref()
26386                .expect("source progress")
26387                .decommissioned_buckets
26388                .is_empty()
26389        );
26390        assert!(mark_decommission_bucket_done(&mut meta, 0, &bucket).expect("complete the queued bucket once"));
26391        assert!(!mark_decommission_bucket_done(&mut meta, 0, &bucket).expect("repeat completion is a no-op"));
26392        let info = meta.pools[0].decommission.as_ref().expect("source progress");
26393        assert!(info.queued_buckets.is_empty());
26394        assert_eq!(info.decommissioned_buckets, vec![bucket.to_string()]);
26395    }
26396
26397    #[test]
26398    fn test_mark_decommission_bucket_done_rejects_missing_decommission_meta() {
26399        let mut meta = PoolMeta {
26400            pools: vec![PoolStatus {
26401                id: 0,
26402                cmd_line: "pool-0".to_string(),
26403                last_update: OffsetDateTime::UNIX_EPOCH,
26404                decommission: None,
26405            }],
26406            ..Default::default()
26407        };
26408
26409        let bucket = DecomBucketInfo {
26410            name: "bucket-a".to_string(),
26411            prefix: String::new(),
26412        };
26413        let err = mark_decommission_bucket_done(&mut meta, 0, &bucket)
26414            .expect_err("missing decommission metadata should return explicit error");
26415        assert!(
26416            err.to_string()
26417                .contains("failed to mark decommission bucket done: decommission metadata not initialized")
26418        );
26419    }
26420
26421    #[test]
26422    fn test_mark_decommission_bucket_done_rejects_out_of_range_index() {
26423        let mut meta = PoolMeta {
26424            pools: vec![PoolStatus {
26425                id: 0,
26426                cmd_line: "pool-0".to_string(),
26427                last_update: OffsetDateTime::UNIX_EPOCH,
26428                decommission: Some(PoolDecommissionInfo::default()),
26429            }],
26430            ..Default::default()
26431        };
26432
26433        let bucket = DecomBucketInfo {
26434            name: "bucket-a".to_string(),
26435            prefix: String::new(),
26436        };
26437        let err =
26438            mark_decommission_bucket_done(&mut meta, 1, &bucket).expect_err("out-of-range index should return invalid argument");
26439        assert!(err.to_string().contains("invalid decommission pool index 1 for 1 pools"));
26440    }
26441
26442    #[test]
26443    fn test_mark_decommission_bucket_done_pops_bucket_when_present() {
26444        let mut meta = PoolMeta {
26445            pools: vec![PoolStatus {
26446                id: 0,
26447                cmd_line: "pool-0".to_string(),
26448                last_update: OffsetDateTime::UNIX_EPOCH,
26449                decommission: Some(PoolDecommissionInfo {
26450                    queued_buckets: vec!["bucket-a".to_string()],
26451                    ..Default::default()
26452                }),
26453            }],
26454            ..Default::default()
26455        };
26456
26457        let bucket = DecomBucketInfo {
26458            name: "bucket-a".to_string(),
26459            prefix: String::new(),
26460        };
26461        let popped = mark_decommission_bucket_done(&mut meta, 0, &bucket).expect("valid state should mark bucket done");
26462        assert!(popped);
26463    }
26464
26465    #[test]
26466    fn test_count_decommission_item_rejects_missing_decommission_meta() {
26467        let mut meta = PoolMeta {
26468            pools: vec![PoolStatus {
26469                id: 0,
26470                cmd_line: "pool-0".to_string(),
26471                last_update: OffsetDateTime::UNIX_EPOCH,
26472                decommission: None,
26473            }],
26474            ..Default::default()
26475        };
26476
26477        let err = count_decommission_item(&mut meta, 0, 64, true)
26478            .expect_err("missing decommission metadata should return explicit error");
26479        assert!(
26480            err.to_string()
26481                .contains("failed to count decommission item: decommission metadata not initialized")
26482        );
26483    }
26484
26485    #[test]
26486    fn test_count_decommission_item_updates_done_and_failed_counters() {
26487        let mut meta = PoolMeta {
26488            pools: vec![PoolStatus {
26489                id: 0,
26490                cmd_line: "pool-0".to_string(),
26491                last_update: OffsetDateTime::UNIX_EPOCH,
26492                decommission: Some(PoolDecommissionInfo::default()),
26493            }],
26494            ..Default::default()
26495        };
26496
26497        count_decommission_item(&mut meta, 0, 32, false).expect("success counter should be updated");
26498        count_decommission_item(&mut meta, 0, 16, true).expect("failed counter should be updated");
26499
26500        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
26501        assert_eq!(info.items_decommissioned, 1);
26502        assert_eq!(info.bytes_done, 32);
26503        assert_eq!(info.items_decommission_failed, 1);
26504        assert_eq!(info.bytes_failed, 16);
26505    }
26506
26507    #[test]
26508    fn test_track_decommission_current_object_rejects_missing_decommission_meta() {
26509        let mut meta = PoolMeta {
26510            pools: vec![PoolStatus {
26511                id: 0,
26512                cmd_line: "pool-0".to_string(),
26513                last_update: OffsetDateTime::UNIX_EPOCH,
26514                decommission: None,
26515            }],
26516            ..Default::default()
26517        };
26518
26519        let err = track_decommission_current_object(&mut meta, 0, "bucket-a", "object-a")
26520            .expect_err("missing decommission metadata should return explicit error");
26521        assert!(
26522            err.to_string()
26523                .contains("failed to track decommission current object: decommission metadata not initialized")
26524        );
26525    }
26526
26527    #[test]
26528    fn test_track_decommission_current_object_updates_bucket_and_object() {
26529        let mut meta = PoolMeta {
26530            pools: vec![PoolStatus {
26531                id: 0,
26532                cmd_line: "pool-0".to_string(),
26533                last_update: OffsetDateTime::UNIX_EPOCH,
26534                decommission: Some(PoolDecommissionInfo::default()),
26535            }],
26536            ..Default::default()
26537        };
26538
26539        track_decommission_current_object(&mut meta, 0, "bucket-a", "object-a").expect("valid state should track bucket/object");
26540
26541        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
26542        assert_eq!(info.bucket, "bucket-a");
26543        assert_eq!(info.object, "object-a");
26544        assert!(info.stage.is_empty());
26545    }
26546
26547    #[test]
26548    fn test_track_decommission_current_object_stage_updates_stage() {
26549        let mut meta = PoolMeta {
26550            pools: vec![PoolStatus {
26551                id: 0,
26552                cmd_line: "pool-0".to_string(),
26553                last_update: OffsetDateTime::UNIX_EPOCH,
26554                decommission: Some(PoolDecommissionInfo::default()),
26555            }],
26556            ..Default::default()
26557        };
26558
26559        track_decommission_current_object_stage(&mut meta, 0, "bucket-a", "object-a", "cleanup_preflight")
26560            .expect("valid state should track bucket/object stage");
26561
26562        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
26563        assert_eq!(info.bucket, "bucket-a");
26564        assert_eq!(info.object, "object-a");
26565        assert_eq!(info.stage, "cleanup_preflight");
26566    }
26567
26568    #[test]
26569    fn test_resolve_decommission_update_after_result_passthrough_ok() {
26570        let ok = resolve_decommission_update_after_result(Ok(true)).expect("ok value should pass through");
26571        assert!(ok);
26572    }
26573
26574    #[test]
26575    fn test_resolve_decommission_update_after_result_wraps_error_context() {
26576        let err = resolve_decommission_update_after_result(ensure_valid_decommission_pool_index(0, 0).map(|_| false))
26577            .expect_err("invalid argument should be wrapped with context");
26578        assert!(err.to_string().contains("decommission metadata update failed"));
26579        assert!(err.to_string().contains("invalid decommission pool index 0 for 0 pools"));
26580    }
26581
26582    #[test]
26583    fn test_resolve_decommission_progress_save_result_returns_none_on_success() {
26584        assert!(resolve_decommission_progress_save_result(Ok(())).is_none());
26585    }
26586
26587    #[test]
26588    fn test_resolve_decommission_progress_save_result_returns_error_for_best_effort_failure() {
26589        let err = resolve_decommission_progress_save_result(Err(Error::SlowDown))
26590            .expect("progress save failure should be returned for logging");
26591
26592        assert!(err.to_string().contains("decommission progress save failed"));
26593        assert!(err.to_string().contains(Error::SlowDown.to_string().as_str()));
26594    }
26595
26596    #[test]
26597    fn test_resolve_decommission_preflight_heal_result_passthrough_ok() {
26598        assert!(resolve_decommission_preflight_heal_result::<()>("bucket-a", Ok(())).is_ok());
26599    }
26600
26601    #[test]
26602    fn test_resolve_decommission_preflight_heal_result_wraps_error_context() {
26603        let err = resolve_decommission_preflight_heal_result::<()>("bucket-a", Err(Error::SlowDown))
26604            .expect_err("heal failure should carry preflight context");
26605        assert!(
26606            err.to_string()
26607                .contains("decommission preflight heal failed for bucket bucket-a")
26608        );
26609    }
26610
26611    #[test]
26612    fn test_resolve_decommission_optional_bucket_config_result_passthrough() {
26613        let result = resolve_decommission_optional_bucket_config_result("bucket-a", "replication", Ok(42_u8))
26614            .expect("bucket config should pass through");
26615        assert_eq!(result, Some(42));
26616    }
26617
26618    #[test]
26619    fn test_resolve_decommission_optional_bucket_config_result_returns_none_for_missing_config() {
26620        let result =
26621            resolve_decommission_optional_bucket_config_result::<()>("bucket-a", "versioning", Err(Error::ConfigNotFound))
26622                .expect("missing bucket config should map to None");
26623        assert!(result.is_none());
26624    }
26625
26626    #[test]
26627    fn test_resolve_decommission_optional_bucket_config_result_wraps_other_errors() {
26628        let err = resolve_decommission_optional_bucket_config_result::<()>("bucket-a", "replication", Err(Error::SlowDown))
26629            .expect_err("unexpected bucket config errors should be wrapped with context");
26630        assert!(
26631            err.to_string()
26632                .contains("decommission replication config load failed for bucket bucket-a")
26633        );
26634    }
26635
26636    #[test]
26637    fn test_resolve_decommission_entry_cleanup_delete_result_passthrough_ok() {
26638        assert!(resolve_decommission_entry_cleanup_delete_result(Ok(()), "bucket-a", "obj.txt").is_ok());
26639    }
26640
26641    #[test]
26642    fn test_resolve_decommission_entry_cleanup_delete_result_ignores_not_found() {
26643        assert!(resolve_decommission_entry_cleanup_delete_result::<()>(Err(Error::FileNotFound), "bucket-a", "obj.txt").is_ok());
26644    }
26645
26646    #[test]
26647    fn test_resolve_decommission_entry_cleanup_delete_result_wraps_error_context() {
26648        let err = resolve_decommission_entry_cleanup_delete_result::<()>(Err(Error::SlowDown), "bucket-a", "obj.txt")
26649            .expect_err("cleanup delete failure should be wrapped with explicit context");
26650        assert!(
26651            err.to_string()
26652                .contains("decommission cleanup_delete_object failed for bucket-a/obj.txt")
26653        );
26654    }
26655
26656    #[test]
26657    fn test_resolve_decommission_entry_reload_result_passthrough_ok() {
26658        assert!(resolve_decommission_entry_reload_result(Ok(()), "bucket-a", "obj.txt").is_ok());
26659    }
26660
26661    #[test]
26662    fn test_resolve_decommission_entry_reload_result_wraps_error_context() {
26663        let err = resolve_decommission_entry_reload_result(Err(Error::SlowDown), "bucket-a", "obj.txt")
26664            .expect_err("reload failure should be wrapped with explicit context");
26665        assert!(
26666            err.to_string()
26667                .contains("decommission reload_pool_meta failed for bucket-a/obj.txt")
26668        );
26669    }
26670
26671    #[test]
26672    fn test_resolve_decommission_terminal_mark_result_passthrough_ok() {
26673        assert!(resolve_decommission_terminal_mark_result(Ok(()), "completed", "pool-a").is_ok());
26674    }
26675
26676    #[test]
26677    fn test_resolve_decommission_terminal_mark_result_wraps_error_context() {
26678        let err = resolve_decommission_terminal_mark_result(Err(Error::SlowDown), "failed", "pool-a")
26679            .expect_err("terminal mark failure should include stage and pool context");
26680        let message = err.to_string();
26681        assert!(message.contains("decommission terminal mark failed failed for pool pool-a"));
26682    }
26683
26684    #[test]
26685    fn test_resolve_decommission_terminal_mark_after_error_result_passthrough_ok() {
26686        assert!(resolve_decommission_terminal_mark_after_error_result(Ok(()), 3, &Error::SlowDown).is_ok());
26687    }
26688
26689    #[test]
26690    fn test_resolve_decommission_terminal_mark_after_error_result_wraps_error_context() {
26691        let err = resolve_decommission_terminal_mark_after_error_result(Err(Error::OperationCanceled), 3, &Error::SlowDown)
26692            .expect_err("terminal mark after-error failure should include both errors");
26693        let message = err.to_string();
26694        assert!(message.contains("decommission terminal mark failed after background error on pool 3"));
26695        assert!(message.contains("mark error"));
26696    }
26697
26698    #[test]
26699    fn test_observe_decommission_terminal_reload_result_returns_none_on_success() {
26700        assert!(observe_decommission_terminal_reload_result(Ok(()), "complete_decommission for pool 3").is_none());
26701    }
26702
26703    #[test]
26704    fn test_observe_decommission_terminal_reload_result_keeps_failure_for_logging() {
26705        let err = observe_decommission_terminal_reload_result(Err(Error::SlowDown), "decommission_failed for pool 3")
26706            .expect("reload failure should be observable");
26707        let message = err.to_string();
26708        assert!(message.contains("decommission terminal pool meta reload failed during decommission_failed for pool 3"));
26709        assert!(message.contains(Error::SlowDown.to_string().as_str()));
26710    }
26711
26712    #[test]
26713    fn test_decommission_item_size_converts_positive_values() {
26714        assert_eq!(decommission_item_size(42_i64), 42);
26715    }
26716
26717    #[test]
26718    fn test_decommission_item_size_clamps_negative_values_to_zero() {
26719        assert_eq!(decommission_item_size(-1_i64), 0);
26720    }
26721
26722    #[test]
26723    fn test_new_multipart_abort_flag_defaults_to_abort_enabled() {
26724        let flag = data_movement::new_multipart_abort_flag();
26725        assert!(data_movement::should_abort_multipart_upload(&flag));
26726    }
26727
26728    #[test]
26729    fn test_mark_multipart_upload_completed_disables_abort_cleanup() {
26730        let flag = data_movement::new_multipart_abort_flag();
26731        data_movement::mark_multipart_upload_completed(&flag);
26732        assert!(!data_movement::should_abort_multipart_upload(&flag));
26733    }
26734
26735    #[test]
26736    fn test_decode_part_index_returns_some_for_valid_payload() {
26737        let mut index = Index::new();
26738        index.add(0, 0).expect("first index entry should be accepted");
26739        index
26740            .add(2_097_152, 2_097_152)
26741            .expect("second index entry should advance totals");
26742
26743        let encoded = index.into_vec();
26744        let decoded = data_movement::decode_part_index(Some(&encoded)).expect("valid index payload should decode");
26745
26746        assert_eq!(decoded.total_uncompressed, 2_097_152);
26747        assert_eq!(decoded.total_compressed, 2_097_152);
26748    }
26749
26750    #[test]
26751    fn test_with_decommission_entry_context_formats_stage_bucket_and_object() {
26752        let err = with_decommission_entry_context("update_after", "bucket-a", "obj.txt", Error::SlowDown);
26753        let message = err.to_string();
26754        assert!(message.contains("decommission entry update_after failed"));
26755        assert!(message.contains("bucket bucket-a"));
26756        assert!(message.contains("object obj.txt"));
26757    }
26758
26759    #[test]
26760    fn test_load_decommission_entry_versions_wraps_parse_errors_with_context() {
26761        let entry = MetaCacheEntry {
26762            name: "obj.txt".to_string(),
26763            metadata: vec![1, 2, 3],
26764            cached: None,
26765            reusable: false,
26766        };
26767
26768        let err = load_decommission_entry_versions(&entry, "bucket-a", "check_after_decommission.file_info_versions")
26769            .expect_err("invalid metadata should fail");
26770        let message = err.to_string();
26771        assert!(message.contains("decommission entry check_after_decommission.file_info_versions failed"));
26772        assert!(message.contains("bucket bucket-a"));
26773        assert!(message.contains("object obj.txt"));
26774    }
26775
26776    #[test]
26777    fn test_resolve_decommission_entry_exact_versions_preserves_full_parts() {
26778        let entry = MetaCacheEntry {
26779            name: "obj.txt".to_string(),
26780            metadata: Vec::new(),
26781            cached: None,
26782            reusable: false,
26783        };
26784        let fivs = FileInfoVersions {
26785            volume: "bucket-a".to_string(),
26786            name: "obj.txt".to_string(),
26787            versions: vec![FileInfo {
26788                name: "obj.txt".to_string(),
26789                parts: vec![ObjectPartInfo {
26790                    number: 1,
26791                    etag: "part-etag".to_string(),
26792                    size: 128,
26793                    actual_size: 128,
26794                    ..Default::default()
26795                }],
26796                ..Default::default()
26797            }],
26798            ..Default::default()
26799        };
26800
26801        let resolved = resolve_decommission_entry_exact_versions(Ok(Some(fivs)), &entry, "bucket-a", "file_info_versions")
26802            .expect("exact versions should be preserved");
26803
26804        assert_eq!(resolved.versions[0].parts.len(), 1);
26805        assert_eq!(resolved.versions[0].parts[0].etag, "part-etag");
26806    }
26807
26808    #[test]
26809    fn test_resolve_decommission_entry_exact_versions_uses_empty_when_source_missing() {
26810        let entry = MetaCacheEntry {
26811            name: "obj.txt".to_string(),
26812            metadata: Vec::new(),
26813            cached: None,
26814            reusable: false,
26815        };
26816
26817        let resolved = resolve_decommission_entry_exact_versions(Ok(None), &entry, "bucket-a", "file_info_versions")
26818            .expect("missing source metadata should be treated as empty");
26819
26820        assert_eq!(resolved.volume, "bucket-a");
26821        assert_eq!(resolved.name, "obj.txt");
26822        assert!(resolved.versions.is_empty());
26823    }
26824
26825    #[test]
26826    fn test_resolve_decommission_check_after_list_result_prefers_entry_error() {
26827        let err = resolve_decommission_check_after_list_result(Err(Error::OperationCanceled), Some(Error::SlowDown))
26828            .expect_err("entry error should win over cancellation");
26829        assert!(matches!(err, Error::SlowDown));
26830    }
26831
26832    #[test]
26833    fn test_resolve_decommission_partial_listing_entry_rejects_unresolved_metadata() {
26834        let generation = OffsetDateTime::now_utc();
26835        let unresolved_entry = resolve_decommission_partial_listing_entry(
26836            MetaCacheEntries(vec![None]),
26837            MetadataResolutionParams {
26838                dir_quorum: 2,
26839                obj_quorum: 2,
26840                bucket: "bucket-a".to_string(),
26841                ..Default::default()
26842            },
26843            "bucket-a",
26844            "prefix/",
26845            1,
26846            2,
26847            3,
26848            generation,
26849        )
26850        .expect_err("unresolved partial listing must fail closed");
26851
26852        assert_eq!(unresolved_entry.bucket, "bucket-a");
26853        assert_eq!(unresolved_entry.object, "prefix/");
26854        assert_eq!(unresolved_entry.pool_index, 2);
26855        assert_eq!(unresolved_entry.set_index, 3);
26856        assert_eq!(unresolved_entry.source_generation, generation);
26857        assert_eq!(unresolved_entry.candidate_count, 0);
26858        assert_eq!(unresolved_entry.disk_error_count, 1);
26859        assert_eq!(unresolved_entry.reason, "metadata_resolution_failed");
26860
26861        let message = decommission_unresolved_listing_error(&unresolved_entry).to_string();
26862        assert!(message.contains("decommission listing could not resolve metadata"));
26863        assert!(message.contains("bucket-a/prefix/"));
26864        assert!(message.contains("pool 2 set 3"));
26865        assert!(message.contains("1 disk error(s)"));
26866    }
26867
26868    #[test]
26869    fn unresolved_entry_ledger_requires_individual_verification_before_completion() {
26870        let generation = OffsetDateTime::now_utc();
26871        let mut pool_meta = PoolMeta {
26872            version: POOL_META_VERSION,
26873            pools: vec![PoolStatus {
26874                id: 0,
26875                cmd_line: "/data/pool".to_string(),
26876                last_update: generation,
26877                decommission: Some(PoolDecommissionInfo {
26878                    start_time: Some(generation),
26879                    ..Default::default()
26880                }),
26881            }],
26882            dont_save: true,
26883        };
26884        let unresolved_entry = DecommissionUnresolvedEntry {
26885            bucket: "bucket-a".to_string(),
26886            object: "object-a".to_string(),
26887            pool_index: 0,
26888            set_index: 0,
26889            source_generation: generation,
26890            candidate_count: 1,
26891            disk_error_count: 0,
26892            observed_at: generation,
26893            reason: "metadata_resolution_failed".to_string(),
26894        };
26895
26896        assert!(
26897            record_decommission_unresolved_entry(
26898                &mut pool_meta,
26899                0,
26900                generation,
26901                unresolved_entry.clone(),
26902                generation + Duration::nanoseconds(1),
26903                None,
26904            )
26905            .expect("active generation should accept unresolved entry")
26906        );
26907        let err = reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, None, None)
26908            .expect_err("unverified completion must retain unresolved entries");
26909        assert!(err.to_string().contains("1 unresolved listing entries remain"));
26910        assert_eq!(
26911            pool_meta.pools[0]
26912                .decommission
26913                .as_ref()
26914                .expect("decommission should exist")
26915                .unresolved_entries
26916                .len(),
26917            1
26918        );
26919
26920        let err = reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), None)
26921            .expect_err("a same-generation sweep without entry proof must retain the ledger");
26922        assert!(err.to_string().contains("not individually verified"));
26923
26924        let stale_verified = vec![unresolved_entry.clone()];
26925        let mut replacement = unresolved_entry;
26926        replacement.disk_error_count = 2;
26927        replacement.observed_at = generation + Duration::seconds(1);
26928        assert!(
26929            record_decommission_unresolved_entry(
26930                &mut pool_meta,
26931                0,
26932                generation,
26933                replacement.clone(),
26934                generation + Duration::nanoseconds(2),
26935                None,
26936            )
26937            .expect("a newer observation should replace the ledger entry")
26938        );
26939        let err =
26940            reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), Some(&stale_verified))
26941                .expect_err("stale entry verification must not clear a concurrent replacement");
26942        assert!(err.to_string().contains("not individually verified"));
26943
26944        let verified = vec![replacement];
26945        reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), Some(&verified))
26946            .expect("individually verified entries should reconcile");
26947        assert!(
26948            pool_meta.pools[0]
26949                .decommission
26950                .as_ref()
26951                .expect("decommission should exist")
26952                .unresolved_entries
26953                .is_empty()
26954        );
26955    }
26956
26957    #[tokio::test]
26958    async fn final_sweep_persists_unresolved_entry_from_real_listing() {
26959        let (dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
26960        let bucket = "decommission-final-sweep-unresolved";
26961        let object = "corrupt-object";
26962        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
26963        store
26964            .make_bucket(bucket, &MakeBucketOptions::default())
26965            .await
26966            .expect("test bucket should be created");
26967
26968        let generation = OffsetDateTime::now_utc();
26969        {
26970            let mut pool_meta = store.pool_meta.write().await;
26971            pool_meta.dont_save = false;
26972            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
26973                start_time: Some(generation),
26974                ..Default::default()
26975            });
26976        }
26977        store
26978            .save_current_pool_meta_for_test(&[0])
26979            .await
26980            .expect("active decommission metadata should be persisted before the final sweep");
26981
26982        for (disk_index, dir) in dirs.iter().enumerate() {
26983            let object_dir = dir.path().join(bucket).join(object);
26984            tokio::fs::create_dir_all(&object_dir)
26985                .await
26986                .expect("corrupt object directory should be created");
26987            tokio::fs::write(object_dir.join(STORAGE_FORMAT_FILE), format!("corrupt-xl-meta-{disk_index}").into_bytes())
26988                .await
26989                .expect("divergent corrupt metadata should be written");
26990        }
26991
26992        let err = store
26993            .check_after_decommission(0, &CancellationToken::new(), generation)
26994            .await
26995            .expect_err("real final sweep must fail closed on unresolved listing metadata");
26996        assert!(
26997            err.to_string().contains("decommission listing could not resolve metadata"),
26998            "unexpected final sweep error: {err:?}"
26999        );
27000
27001        let in_memory_entries = store.pool_meta.read().await.pools[0]
27002            .decommission
27003            .as_ref()
27004            .expect("decommission should remain active")
27005            .unresolved_entries
27006            .clone();
27007        assert_eq!(in_memory_entries.len(), 1);
27008        let unresolved_entry = &in_memory_entries[0];
27009        assert_eq!(unresolved_entry.bucket, bucket);
27010        assert_eq!(unresolved_entry.object, object);
27011        assert_eq!(unresolved_entry.pool_index, 0);
27012        assert_eq!(unresolved_entry.set_index, 0);
27013        assert_eq!(unresolved_entry.source_generation, generation);
27014        assert_eq!(unresolved_entry.candidate_count, 4);
27015        assert_eq!(unresolved_entry.disk_error_count, 0);
27016        assert_eq!(unresolved_entry.reason, "metadata_resolution_failed");
27017
27018        let mut restored = PoolMeta::default();
27019        restored
27020            .load_no_lock_from_replicas(vec![store.pools[0].clone()])
27021            .await
27022            .expect("persisted pool metadata should reload after the final-sweep failure");
27023        assert_eq!(
27024            restored.pools[0]
27025                .decommission
27026                .as_ref()
27027                .expect("reloaded decommission should exist")
27028                .unresolved_entries,
27029            in_memory_entries
27030        );
27031    }
27032
27033    #[tokio::test]
27034    async fn decommission_metadata_saves_stay_monotonic_across_clock_rollback_before_terminal_restart() {
27035        let (_dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
27036        let persisted_floor = store.pool_meta.read().await.pools[0].last_update;
27037        let generation = persisted_floor
27038            .checked_add(Duration::seconds(1))
27039            .expect("test generation should advance the initialized pool metadata");
27040        let earlier_tick = generation - Duration::nanoseconds(10);
27041        {
27042            let mut pool_meta = store.pool_meta.write().await;
27043            pool_meta.dont_save = false;
27044            pool_meta.pools[0].last_update = generation;
27045            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
27046                start_time: Some(generation),
27047                ..Default::default()
27048            });
27049        }
27050        store
27051            .save_current_pool_meta_for_test(&[0])
27052            .await
27053            .expect("active decommission metadata should persist before rollback checkpoints");
27054
27055        let unresolved_entry = DecommissionUnresolvedEntry {
27056            bucket: "bucket-a".to_string(),
27057            object: "object-a".to_string(),
27058            pool_index: 0,
27059            set_index: 0,
27060            source_generation: generation,
27061            candidate_count: 1,
27062            disk_error_count: 1,
27063            observed_at: earlier_tick,
27064            reason: "metadata_resolution_failed".to_string(),
27065        };
27066        {
27067            let mut pool_meta = store.pool_meta.write().await;
27068            assert!(
27069                record_decommission_unresolved_entry(&mut pool_meta, 0, generation, unresolved_entry, earlier_tick, None)
27070                    .expect("unresolved entry should record under active generation")
27071            );
27072            assert_eq!(pool_meta.pools[0].last_update, generation + Duration::nanoseconds(1));
27073        }
27074        store
27075            .save_current_pool_meta_for_test(&[0])
27076            .await
27077            .expect("rollback unresolved-entry save should not stale-reject");
27078        store.pool_meta.write().await.pools[0]
27079            .decommission
27080            .as_mut()
27081            .expect("decommission metadata should exist")
27082            .items_decommissioned = DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD;
27083
27084        assert!(
27085            store
27086                .save_decommission_progress_checkpoint_at(0, generation, earlier_tick)
27087                .await
27088                .expect("rollback progress checkpoint should use a monotonic durable identity")
27089        );
27090        let progress_at = generation + Duration::nanoseconds(2);
27091        assert_eq!(store.pool_meta.read().await.pools[0].last_update, progress_at);
27092
27093        let terminal_at = generation + Duration::nanoseconds(3);
27094        {
27095            let mut pool_meta = store.pool_meta.write().await;
27096            assert!(pool_meta.decommission_failed_at_for_test(0, earlier_tick, None));
27097            assert_eq!(pool_meta.pools[0].last_update, terminal_at);
27098        }
27099        store
27100            .save_current_pool_meta_for_test(&[0])
27101            .await
27102            .expect("terminal failure after rollback checkpoints should persist");
27103
27104        let mut restored = PoolMeta::default();
27105        restored
27106            .load_no_lock_from_replicas(vec![store.pools[0].clone()])
27107            .await
27108            .expect("terminal metadata should reload after restart");
27109        assert_eq!(restored.pools[0].last_update, terminal_at);
27110        assert!(
27111            restored.pools[0]
27112                .decommission
27113                .as_ref()
27114                .is_some_and(|info| info.failed && !info.complete && !info.canceled)
27115        );
27116
27117        let restarted = decommission_worker_test_store(restored, Vec::new());
27118        let status = restarted.scanner_data_movement_pause_status().await;
27119        let expected_generation =
27120            u64::try_from(terminal_at.unix_timestamp_nanos()).expect("fixed positive timestamp should fit generation");
27121
27122        assert_eq!(status.movement_generation, expected_generation);
27123        assert_eq!(status.reasons, vec![crate::store::ScannerDataMovementPauseReason::DecommissionFailed]);
27124        assert_eq!(restarted.scanner_data_movement_generation(), expected_generation);
27125    }
27126
27127    #[tokio::test]
27128    async fn unresolved_entry_probe_verifies_absence_on_every_source_disk() {
27129        let (_dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
27130        let bucket = "decommission-final-sweep-absent-ledger";
27131        store
27132            .peer_sys
27133            .make_bucket(bucket, &MakeBucketOptions::default())
27134            .await
27135            .expect("test bucket should be created");
27136        metadata_sys::init_bucket_metadata_sys(store.clone(), vec![bucket.to_string()]).await;
27137
27138        let generation = OffsetDateTime::now_utc();
27139        let unresolved_entry = DecommissionUnresolvedEntry {
27140            bucket: bucket.to_string(),
27141            object: "absent-object".to_string(),
27142            pool_index: 0,
27143            set_index: 0,
27144            source_generation: generation,
27145            candidate_count: 1,
27146            disk_error_count: 1,
27147            observed_at: generation,
27148            reason: "metadata_resolution_failed".to_string(),
27149        };
27150        {
27151            let mut pool_meta = store.pool_meta.write().await;
27152            pool_meta.version = POOL_META_VERSION;
27153            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
27154                start_time: Some(generation),
27155                unresolved_entries: vec![unresolved_entry.clone()],
27156                ..Default::default()
27157            });
27158        }
27159
27160        assert!(
27161            store.pools[0].disk_set[0]
27162                .decommission_unresolved_entry_absent_on_all_disks(0, &unresolved_entry)
27163                .await
27164                .expect("all source disks should be readable")
27165        );
27166        let verified = vec![unresolved_entry.clone()];
27167
27168        let mut pool_meta = store.pool_meta.write().await;
27169        reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), Some(&verified))
27170            .expect("the individually verified entry should reconcile");
27171        assert!(
27172            pool_meta.pools[0]
27173                .decommission
27174                .as_ref()
27175                .expect("decommission should remain present")
27176                .unresolved_entries
27177                .is_empty()
27178        );
27179    }
27180
27181    #[tokio::test]
27182    async fn test_record_decommission_entry_error_cancels_listing_and_preserves_first_error() {
27183        let entry_error = Arc::new(tokio::sync::Mutex::new(None));
27184        let rx = CancellationToken::new();
27185
27186        assert!(record_decommission_entry_error(&entry_error, &rx, Error::SlowDown).await);
27187        assert!(!record_decommission_entry_error(&entry_error, &rx, Error::OperationCanceled).await);
27188
27189        assert!(rx.is_cancelled());
27190        assert!(matches!(*entry_error.lock().await, Some(Error::SlowDown)));
27191    }
27192
27193    #[tokio::test]
27194    async fn test_record_decommission_entry_error_ignores_already_canceled_listing() {
27195        let entry_error = Arc::new(tokio::sync::Mutex::new(None));
27196        let rx = CancellationToken::new();
27197        rx.cancel();
27198
27199        assert!(!record_decommission_entry_error(&entry_error, &rx, Error::SlowDown).await);
27200
27201        assert!(entry_error.lock().await.is_none());
27202    }
27203
27204    #[test]
27205    fn unresolved_entry_verification_requires_every_source_disk() {
27206        assert!(ensure_decommission_unresolved_verification_disk_count(4, 4, 0, 1).is_ok());
27207
27208        let err = ensure_decommission_unresolved_verification_disk_count(4, 3, 0, 1)
27209            .expect_err("an offline source disk must block ledger reconciliation");
27210        assert!(err.to_string().contains("requires all 4 source disks, but only 3 are online"));
27211    }
27212
27213    #[test]
27214    fn test_resolve_decommission_listing_error_preserves_real_listing_failure() {
27215        let err = resolve_decommission_listing_error(Some(Error::SlowDown), Some(Error::OperationCanceled))
27216            .expect("listing failure should be returned");
27217        assert!(matches!(err, Error::SlowDown));
27218
27219        let err = resolve_decommission_listing_error(Some(Error::OperationCanceled), Some(Error::SlowDown))
27220            .expect("entry failure should be returned");
27221        assert!(matches!(err, Error::SlowDown));
27222    }
27223
27224    #[test]
27225    fn test_resolve_decommission_check_after_list_result_returns_list_result_without_entry_error() {
27226        let err = resolve_decommission_check_after_list_result(Err(Error::OperationCanceled), None)
27227            .expect_err("list result should be preserved without entry error");
27228        assert!(matches!(err, Error::OperationCanceled));
27229    }
27230
27231    #[test]
27232    fn test_resolve_decommission_pool_meta_reload_result_passthrough_ok() {
27233        assert!(resolve_decommission_pool_meta_reload_result(Ok(()), "start_decommission").is_ok());
27234    }
27235
27236    #[test]
27237    fn test_resolve_decommission_pool_meta_reload_result_wraps_error_context() {
27238        let err = resolve_decommission_pool_meta_reload_result(Err(Error::SlowDown), "decommission_failed for pool 3")
27239            .expect_err("reload failure should be wrapped with stage context");
27240        let message = err.to_string();
27241        assert!(message.contains("decommission pool meta reload failed during decommission_failed for pool 3"));
27242        assert!(message.contains(Error::SlowDown.to_string().as_str()));
27243    }
27244
27245    #[test]
27246    fn test_resolve_start_decommission_pool_meta_reload_result_returns_failure() {
27247        let err = resolve_start_decommission_pool_meta_reload_result(Err(Error::other(
27248            "reload_pool_meta encountered 1 failure(s): peer[0] reload_pool_meta failed",
27249        )))
27250        .expect_err("start_decommission must fail when peer pool meta reload fails");
27251        let message = err.to_string();
27252
27253        assert!(message.contains("decommission pool meta reload failed during start_decommission"));
27254        assert!(message.contains("reload_pool_meta encountered 1 failure(s)"));
27255        assert!(message.contains("peer[0]"));
27256    }
27257
27258    #[test]
27259    fn test_resolve_decommission_listing_worker_result_passthrough_ok() {
27260        assert!(resolve_decommission_listing_worker_result(2, Ok(Ok(()))).is_ok());
27261    }
27262
27263    #[test]
27264    fn test_resolve_decommission_listing_worker_result_passthrough_worker_error() {
27265        let err = resolve_decommission_listing_worker_result(2, Ok(Err(Error::SlowDown)))
27266            .expect_err("listing worker error should be returned");
27267
27268        assert!(matches!(err, Error::SlowDown));
27269    }
27270
27271    #[tokio::test]
27272    async fn test_resolve_decommission_listing_worker_result_wraps_join_error_context() {
27273        let join_error = tokio::spawn(async {
27274            panic!("listing worker panic");
27275        })
27276        .await
27277        .expect_err("panic task should return JoinError");
27278
27279        let err = resolve_decommission_listing_worker_result(4, Err(join_error))
27280            .expect_err("join error should be wrapped with context");
27281        let message = err.to_string();
27282        assert!(message.contains("decommission listing worker 4 task join error"));
27283        assert!(message.contains("panic"));
27284    }
27285
27286    #[test]
27287    fn test_should_retry_decommission_listing_respects_attempt_limit_and_bucket_missing() {
27288        assert!(should_retry_decommission_listing(&Error::SlowDown, 0, 2));
27289        assert!(!should_retry_decommission_listing(&Error::SlowDown, 1, 2));
27290        assert!(!should_retry_decommission_listing(
27291            &StorageError::BucketNotFound("bucket".to_string()),
27292            0,
27293            2
27294        ));
27295    }
27296
27297    #[tokio::test]
27298    async fn test_wait_decommission_retry_backoff_reports_canceled_without_sleeping() {
27299        let token = CancellationToken::new();
27300        token.cancel();
27301
27302        assert!(wait_decommission_retry_backoff(&token, StdDuration::from_secs(30)).await);
27303    }
27304
27305    #[test]
27306    fn test_decommission_retry_backoff_delay_grows_linearly() {
27307        let base = StdDuration::from_millis(100);
27308
27309        assert_eq!(decommission_retry_backoff_delay(base, 1), base);
27310        assert_eq!(decommission_retry_backoff_delay(base, 3), StdDuration::from_millis(300));
27311        assert_eq!(decommission_retry_backoff_delay(base, usize::MAX), base.saturating_mul(u32::MAX));
27312    }
27313
27314    #[test]
27315    fn test_source_changed_exhaustion_fails_only_after_pool_limit() {
27316        assert!(!should_fail_decommission_pool_after_exhausted_source_changed(
27317            DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT
27318        ));
27319        assert!(should_fail_decommission_pool_after_exhausted_source_changed(
27320            DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT + 1
27321        ));
27322    }
27323
27324    #[tokio::test(start_paused = true)]
27325    async fn test_run_decommission_listing_with_retry_stops_after_attempt_limit() {
27326        let attempts = Arc::new(AtomicUsize::new(0));
27327        let err = run_decommission_listing_with_retry(
27328            CancellationToken::new(),
27329            "bucket-a".to_string(),
27330            noop_decommission_list_callback(),
27331            1,
27332            2,
27333            3,
27334            {
27335                let attempts = attempts.clone();
27336                move |_| {
27337                    let attempts = attempts.clone();
27338                    async move {
27339                        attempts.fetch_add(1, Ordering::SeqCst);
27340                        Err(Error::SlowDown)
27341                    }
27342                }
27343            },
27344        )
27345        .await
27346        .expect_err("permanent listing failure must not retry forever");
27347
27348        assert_eq!(attempts.load(Ordering::SeqCst), 3);
27349        assert!(err.to_string().contains("attempt 3/3"));
27350    }
27351
27352    #[tokio::test]
27353    async fn test_run_decommission_listing_with_retry_treats_bucket_missing_as_complete() {
27354        let attempts = Arc::new(AtomicUsize::new(0));
27355        run_decommission_listing_with_retry(
27356            CancellationToken::new(),
27357            "bucket-a".to_string(),
27358            noop_decommission_list_callback(),
27359            1,
27360            2,
27361            3,
27362            {
27363                let attempts = attempts.clone();
27364                move |_| {
27365                    let attempts = attempts.clone();
27366                    async move {
27367                        attempts.fetch_add(1, Ordering::SeqCst);
27368                        Err(StorageError::BucketNotFound("bucket-a".to_string()))
27369                    }
27370                }
27371            },
27372        )
27373        .await
27374        .expect("missing bucket should keep previous decommission listing behavior");
27375
27376        assert_eq!(attempts.load(Ordering::SeqCst), 1);
27377    }
27378
27379    #[test]
27380    fn test_should_count_decommission_version_complete_for_cleanup_safe_ignored_result() {
27381        assert!(should_count_decommission_version_complete(true, true, false));
27382    }
27383
27384    #[test]
27385    fn test_should_count_decommission_version_complete_rejects_skip_only_ignored_result() {
27386        assert!(!should_count_decommission_version_complete(true, false, false));
27387    }
27388
27389    #[test]
27390    fn test_should_count_decommission_version_complete_for_completed_result() {
27391        assert!(should_count_decommission_version_complete(false, false, false));
27392    }
27393
27394    #[test]
27395    fn test_should_count_decommission_version_complete_rejects_failed_result() {
27396        assert!(!should_count_decommission_version_complete(false, false, true));
27397    }
27398
27399    #[test]
27400    fn test_should_cleanup_decommission_source_entry_accepts_all_versions_completed() {
27401        assert!(should_cleanup_decommission_source_entry(3, 3, 0));
27402    }
27403
27404    #[test]
27405    fn test_should_cleanup_decommission_source_entry_accepts_migrated_and_safely_expired_versions() {
27406        assert!(should_cleanup_decommission_source_entry(1, 2, 1));
27407    }
27408
27409    #[test]
27410    fn test_should_cleanup_decommission_source_entry_accepts_versions_only_safely_expired_by_lifecycle() {
27411        assert!(should_cleanup_decommission_source_entry(0, 2, 2));
27412    }
27413
27414    #[test]
27415    fn test_should_cleanup_decommission_source_entry_rejects_object_lock_retained_version() {
27416        assert!(!should_cleanup_decommission_source_entry(1, 2, 0));
27417    }
27418
27419    #[test]
27420    fn test_should_cleanup_decommission_source_entry_rejects_replication_pending_version() {
27421        assert!(!should_cleanup_decommission_source_entry(2, 3, 0));
27422    }
27423
27424    #[test]
27425    fn test_should_cleanup_decommission_source_entry_rejects_counter_overrun() {
27426        assert!(!should_cleanup_decommission_source_entry(2, 2, 1));
27427    }
27428
27429    #[test]
27430    fn test_pool_meta_update_after_rejects_out_of_range_index() {
27431        let mut meta = PoolMeta::default();
27432        let err = meta
27433            .update_after(1, Duration::seconds(1))
27434            .expect_err("out-of-range index should fail");
27435        assert!(err.to_string().contains("invalid decommission pool index 1 for 0 pools"));
27436    }
27437
27438    #[test]
27439    fn test_pool_meta_update_after_rejects_when_decommission_missing() {
27440        let mut meta = PoolMeta {
27441            pools: vec![PoolStatus {
27442                id: 0,
27443                cmd_line: "pool-0".to_string(),
27444                last_update: OffsetDateTime::UNIX_EPOCH,
27445                decommission: None,
27446            }],
27447            ..Default::default()
27448        };
27449
27450        let err = meta
27451            .update_after(0, Duration::seconds(1))
27452            .expect_err("pool without decommission should fail");
27453        assert!(
27454            err.to_string()
27455                .contains("failed to update decommission metadata timestamp: decommission metadata not initialized")
27456        );
27457    }
27458
27459    #[test]
27460    fn test_track_decommission_stage_does_not_advance_checkpoint_state() {
27461        let mut meta = PoolMeta {
27462            pools: vec![PoolStatus {
27463                id: 0,
27464                cmd_line: "pool-0".to_string(),
27465                last_update: OffsetDateTime::UNIX_EPOCH,
27466                decommission: Some(PoolDecommissionInfo {
27467                    items_decommissioned: 3,
27468                    items_decommission_failed: 2,
27469                    ..Default::default()
27470                }),
27471            }],
27472            ..Default::default()
27473        };
27474
27475        track_decommission_current_object_stage(&mut meta, 0, "bucket", "object", "migrate_object")
27476            .expect("valid decommission progress should be tracked");
27477
27478        assert_eq!(meta.pools[0].last_update, OffsetDateTime::UNIX_EPOCH);
27479        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
27480        assert_eq!(info.items_since_last_progress_save(), 5);
27481        assert_eq!(info.stage, "migrate_object");
27482    }
27483
27484    #[test]
27485    fn test_pool_meta_update_after_skips_before_time_and_item_thresholds() {
27486        let mut meta = PoolMeta {
27487            pools: vec![PoolStatus {
27488                id: 0,
27489                cmd_line: "pool-0".to_string(),
27490                last_update: OffsetDateTime::now_utc(),
27491                decommission: Some(PoolDecommissionInfo {
27492                    items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD - 1,
27493                    ..Default::default()
27494                }),
27495            }],
27496            ..Default::default()
27497        };
27498
27499        let saved = meta
27500            .update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
27501            .expect("valid decommission state should update");
27502
27503        assert!(!saved);
27504        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
27505        assert_eq!(info.items_since_last_progress_save(), DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD - 1);
27506    }
27507
27508    #[test]
27509    fn test_pool_meta_update_after_requests_save_when_item_threshold_reached() {
27510        let mut meta = PoolMeta {
27511            pools: vec![PoolStatus {
27512                id: 0,
27513                cmd_line: "pool-0".to_string(),
27514                last_update: OffsetDateTime::now_utc(),
27515                decommission: Some(PoolDecommissionInfo {
27516                    items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
27517                    ..Default::default()
27518                }),
27519            }],
27520            ..Default::default()
27521        };
27522
27523        let saved = meta
27524            .update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
27525            .expect("item threshold should save progress");
27526
27527        assert!(saved);
27528        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
27529        assert_eq!(info.items_since_last_progress_save(), DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD);
27530    }
27531
27532    #[test]
27533    fn test_pool_meta_update_after_requests_save_when_time_threshold_reached() {
27534        let mut meta = PoolMeta {
27535            pools: vec![PoolStatus {
27536                id: 0,
27537                cmd_line: "pool-0".to_string(),
27538                last_update: OffsetDateTime::now_utc() - DECOMMISSION_PROGRESS_SAVE_INTERVAL,
27539                decommission: Some(PoolDecommissionInfo {
27540                    items_decommissioned: 1,
27541                    ..Default::default()
27542                }),
27543            }],
27544            ..Default::default()
27545        };
27546
27547        let saved = meta
27548            .update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
27549            .expect("time threshold should save progress");
27550
27551        assert!(saved);
27552        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
27553        assert_eq!(info.items_since_last_progress_save(), 1);
27554    }
27555
27556    #[test]
27557    fn test_pool_meta_update_after_does_not_advance_last_update_before_save() {
27558        let last_update = OffsetDateTime::UNIX_EPOCH;
27559        let mut meta = PoolMeta {
27560            pools: vec![PoolStatus {
27561                id: 0,
27562                cmd_line: "pool-0".to_string(),
27563                last_update,
27564                decommission: Some(PoolDecommissionInfo {
27565                    start_time: Some(last_update),
27566                    items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
27567                    ..Default::default()
27568                }),
27569            }],
27570            ..Default::default()
27571        };
27572
27573        assert!(
27574            meta.update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
27575                .expect("item threshold should request a checkpoint")
27576        );
27577        assert_eq!(meta.pools[0].last_update, last_update);
27578    }
27579
27580    #[test]
27581    fn test_decommission_progress_checkpoint_commits_exact_snapshot_watermark() {
27582        let start_time = OffsetDateTime::UNIX_EPOCH;
27583        let checkpoint_at = start_time + Duration::seconds(30);
27584        let mut meta = PoolMeta {
27585            pools: vec![PoolStatus {
27586                id: 0,
27587                cmd_line: "pool-0".to_string(),
27588                last_update: start_time,
27589                decommission: Some(PoolDecommissionInfo {
27590                    start_time: Some(start_time),
27591                    items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
27592                    ..Default::default()
27593                }),
27594            }],
27595            ..Default::default()
27596        };
27597
27598        let checkpoint = meta
27599            .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None)
27600            .expect("valid decommission state should produce a checkpoint")
27601            .expect("item threshold should produce a checkpoint");
27602        meta.count_item(0, 1, false);
27603
27604        assert!(meta.commit_decommission_progress_checkpoint(0, checkpoint));
27605        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
27606        assert_eq!(info.progress_save_item_baseline, checkpoint.counted_items);
27607        assert_eq!(info.items_since_last_progress_save(), 1);
27608        assert_eq!(meta.pools[0].last_update, checkpoint_at);
27609    }
27610
27611    #[test]
27612    fn test_decommission_progress_checkpoint_backoff_does_not_advance_baseline() {
27613        let start_time = OffsetDateTime::UNIX_EPOCH;
27614        let checkpoint_at = start_time + Duration::seconds(30);
27615        let retry_after = checkpoint_at + DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
27616        let mut meta = PoolMeta {
27617            pools: vec![PoolStatus {
27618                id: 0,
27619                cmd_line: "pool-0".to_string(),
27620                last_update: start_time,
27621                decommission: Some(PoolDecommissionInfo {
27622                    start_time: Some(start_time),
27623                    items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
27624                    ..Default::default()
27625                }),
27626            }],
27627            ..Default::default()
27628        };
27629
27630        let checkpoint = meta
27631            .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None)
27632            .expect("valid decommission state should produce a checkpoint")
27633            .expect("item threshold should produce a checkpoint");
27634        meta.defer_decommission_progress_checkpoint(0, checkpoint, retry_after);
27635
27636        assert!(
27637            meta.decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None)
27638                .expect("retry backoff check should succeed")
27639                .is_none()
27640        );
27641        assert_eq!(meta.pools[0].last_update, start_time);
27642        assert_eq!(
27643            meta.pools[0]
27644                .decommission
27645                .as_ref()
27646                .expect("decommission info should exist")
27647                .progress_save_item_baseline,
27648            0
27649        );
27650    }
27651
27652    #[test]
27653    fn test_decommission_progress_checkpoint_count_scales_with_threshold() {
27654        let start_time = OffsetDateTime::UNIX_EPOCH;
27655        let checkpoint_at = start_time;
27656        let mut meta = PoolMeta {
27657            pools: vec![PoolStatus {
27658                id: 0,
27659                cmd_line: "pool-0".to_string(),
27660                last_update: start_time,
27661                decommission: Some(PoolDecommissionInfo {
27662                    start_time: Some(start_time),
27663                    ..Default::default()
27664                }),
27665            }],
27666            ..Default::default()
27667        };
27668        let mut checkpoint_count = 0;
27669
27670        for _ in 0..(DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD * 10) {
27671            meta.count_item(0, 1, false);
27672            if let Some(checkpoint) = meta
27673                .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None)
27674                .expect("valid decommission state should produce a checkpoint")
27675            {
27676                checkpoint_count += 1;
27677                assert!(meta.commit_decommission_progress_checkpoint(0, checkpoint));
27678            }
27679        }
27680
27681        assert_eq!(checkpoint_count, 10);
27682    }
27683
27684    #[test]
27685    fn test_ensure_decommission_not_rebalancing_rejects_running_rebalance() {
27686        let err = ensure_decommission_not_rebalancing(true).expect_err("rebalance running should be rejected");
27687        assert!(matches!(err, Error::RebalanceAlreadyRunning));
27688    }
27689
27690    #[test]
27691    fn test_ensure_decommission_not_rebalancing_allows_idle() {
27692        assert!(ensure_decommission_not_rebalancing(false).is_ok());
27693    }
27694
27695    #[test]
27696    fn test_ensure_decommission_start_rebalance_meta_allowed_rejects_active_rebalance() {
27697        let meta = RebalanceMeta {
27698            pool_stats: vec![RebalanceStats {
27699                participating: true,
27700                info: RebalanceInfo {
27701                    status: RebalStatus::Started,
27702                    ..Default::default()
27703                },
27704                ..Default::default()
27705            }],
27706            ..Default::default()
27707        };
27708
27709        let err = ensure_decommission_start_rebalance_meta_allowed(Some(&meta))
27710            .expect_err("persisted active rebalance should block decommission start");
27711
27712        assert!(matches!(err, Error::RebalanceAlreadyRunning));
27713    }
27714
27715    #[test]
27716    fn test_ensure_decommission_start_rebalance_meta_allowed_rejects_stopping_rebalance() {
27717        let meta = RebalanceMeta {
27718            pool_stats: vec![RebalanceStats {
27719                info: RebalanceInfo {
27720                    status: RebalStatus::Started,
27721                    stopping: true,
27722                    ..Default::default()
27723                },
27724                ..Default::default()
27725            }],
27726            ..Default::default()
27727        };
27728
27729        let err = ensure_decommission_start_rebalance_meta_allowed(Some(&meta))
27730            .expect_err("persisted stopping rebalance should block decommission start");
27731
27732        assert!(matches!(err, Error::RebalanceAlreadyRunning));
27733    }
27734
27735    #[test]
27736    fn test_ensure_decommission_start_rebalance_meta_allowed_allows_terminal_or_missing_rebalance() {
27737        let terminal_meta = RebalanceMeta {
27738            pool_stats: vec![RebalanceStats {
27739                participating: true,
27740                info: RebalanceInfo {
27741                    status: RebalStatus::Completed,
27742                    ..Default::default()
27743                },
27744                ..Default::default()
27745            }],
27746            ..Default::default()
27747        };
27748
27749        assert!(ensure_decommission_start_rebalance_meta_allowed(Some(&terminal_meta)).is_ok());
27750        assert!(ensure_decommission_start_rebalance_meta_allowed(None).is_ok());
27751    }
27752
27753    #[test]
27754    fn test_ensure_local_decommission_pool_leaders_allows_local_first_endpoint() {
27755        let endpoints = EndpointServerPools::from(vec![
27756            decommission_test_pool_endpoint(0, false),
27757            decommission_test_pool_endpoint(1, true),
27758        ]);
27759
27760        assert!(ensure_local_decommission_pool_leaders(&endpoints, &[1]).is_ok());
27761    }
27762
27763    #[test]
27764    fn test_ensure_local_decommission_pool_leaders_rejects_remote_first_endpoint() {
27765        let endpoints = EndpointServerPools::from(vec![decommission_test_pool_endpoint(0, false)]);
27766
27767        let err = ensure_local_decommission_pool_leaders(&endpoints, &[0])
27768            .expect_err("remote first endpoint should reject local decommission start");
27769
27770        assert!(err.to_string().contains("must run on the pool first endpoint"));
27771    }
27772
27773    #[test]
27774    fn test_ensure_local_decommission_pool_leaders_rejects_empty_endpoints() {
27775        let endpoints = EndpointServerPools::from(vec![PoolEndpoints {
27776            legacy: false,
27777            set_count: 1,
27778            drives_per_set: 1,
27779            endpoints: Endpoints::from(Vec::<Endpoint>::new()),
27780            cmd_line: "pool-0".to_string(),
27781            platform: String::new(),
27782        }]);
27783
27784        let err = ensure_local_decommission_pool_leaders(&endpoints, &[0])
27785            .expect_err("pool without endpoints should reject local decommission start");
27786
27787        assert!(err.to_string().contains("has no configured endpoints"));
27788    }
27789
27790    #[test]
27791    fn test_decommission_meta_bucket_options_are_idempotent() {
27792        let opts = decommission_meta_bucket_options();
27793
27794        assert!(opts.force_create);
27795    }
27796
27797    #[test]
27798    fn test_is_decommission_active_true_only_when_not_terminal() {
27799        assert!(is_decommission_active(false, false, false));
27800        assert!(!is_decommission_active(true, false, false));
27801        assert!(!is_decommission_active(false, true, false));
27802        assert!(!is_decommission_active(false, false, true));
27803    }
27804
27805    #[test]
27806    fn test_ensure_decommission_generation_rejects_stale_or_queued_workers() {
27807        let generation = OffsetDateTime::UNIX_EPOCH;
27808        let mut meta = PoolMeta {
27809            pools: vec![PoolStatus {
27810                id: 0,
27811                cmd_line: "pool-0".to_string(),
27812                last_update: generation,
27813                decommission: Some(PoolDecommissionInfo {
27814                    start_time: Some(generation),
27815                    ..Default::default()
27816                }),
27817            }],
27818            ..Default::default()
27819        };
27820
27821        assert!(ensure_decommission_generation(&meta, 0, generation).is_ok());
27822        assert!(ensure_decommission_generation(&meta, 0, generation + Duration::seconds(1)).is_err());
27823
27824        meta.pools[0]
27825            .decommission
27826            .as_mut()
27827            .expect("decommission metadata should exist")
27828            .queued = true;
27829        assert!(ensure_decommission_generation(&meta, 0, generation).is_err());
27830
27831        let replacement_generation = generation + Duration::seconds(2);
27832        let info = meta.pools[0]
27833            .decommission
27834            .as_mut()
27835            .expect("decommission metadata should exist");
27836        info.queued = false;
27837        info.start_time = Some(replacement_generation);
27838        assert!(ensure_decommission_generation(&meta, 0, generation).is_err());
27839        assert!(ensure_decommission_generation(&meta, 0, replacement_generation).is_ok());
27840    }
27841
27842    #[test]
27843    fn test_pool_meta_has_active_decommission_counts_running_and_queued_states() {
27844        let active_meta = PoolMeta {
27845            pools: vec![PoolStatus {
27846                id: 0,
27847                cmd_line: "pool-0".to_string(),
27848                last_update: OffsetDateTime::UNIX_EPOCH,
27849                decommission: Some(PoolDecommissionInfo {
27850                    start_time: Some(OffsetDateTime::UNIX_EPOCH),
27851                    ..Default::default()
27852                }),
27853            }],
27854            ..Default::default()
27855        };
27856        let queued_meta = PoolMeta {
27857            pools: vec![PoolStatus {
27858                id: 0,
27859                cmd_line: "pool-0".to_string(),
27860                last_update: OffsetDateTime::UNIX_EPOCH,
27861                decommission: Some(PoolDecommissionInfo {
27862                    queued: true,
27863                    ..Default::default()
27864                }),
27865            }],
27866            ..Default::default()
27867        };
27868
27869        assert!(pool_meta_has_active_decommission(&active_meta));
27870        assert!(pool_meta_has_active_decommission(&queued_meta));
27871    }
27872
27873    #[test]
27874    fn test_pool_meta_has_active_decommission_ignores_capacity_placeholder() {
27875        let meta = PoolMeta {
27876            pools: vec![PoolStatus {
27877                id: 0,
27878                cmd_line: "pool-0".to_string(),
27879                last_update: OffsetDateTime::UNIX_EPOCH,
27880                decommission: Some(PoolDecommissionInfo {
27881                    total_size: 100,
27882                    current_size: 75,
27883                    ..Default::default()
27884                }),
27885            }],
27886            ..Default::default()
27887        };
27888
27889        assert!(!pool_meta_has_active_decommission(&meta));
27890        assert!(!meta.is_suspended(0));
27891        assert_eq!(decommission_start_pool_state(meta.pools.first()), DecommissionStartPoolState::Active);
27892    }
27893
27894    #[test]
27895    fn test_pool_meta_has_active_decommission_ignores_terminal_states() {
27896        let terminal_meta = PoolMeta {
27897            pools: vec![
27898                PoolStatus {
27899                    id: 0,
27900                    cmd_line: "pool-0".to_string(),
27901                    last_update: OffsetDateTime::UNIX_EPOCH,
27902                    decommission: Some(PoolDecommissionInfo {
27903                        complete: true,
27904                        ..Default::default()
27905                    }),
27906                },
27907                PoolStatus {
27908                    id: 1,
27909                    cmd_line: "pool-1".to_string(),
27910                    last_update: OffsetDateTime::UNIX_EPOCH,
27911                    decommission: Some(PoolDecommissionInfo {
27912                        failed: true,
27913                        ..Default::default()
27914                    }),
27915                },
27916                PoolStatus {
27917                    id: 2,
27918                    cmd_line: "pool-2".to_string(),
27919                    last_update: OffsetDateTime::UNIX_EPOCH,
27920                    decommission: Some(PoolDecommissionInfo {
27921                        canceled: true,
27922                        ..Default::default()
27923                    }),
27924                },
27925            ],
27926            ..Default::default()
27927        };
27928
27929        assert!(!pool_meta_has_active_decommission(&terminal_meta));
27930    }
27931
27932    #[test]
27933    fn test_decommission_request_rejections_preserve_invalid_argument_type() {
27934        for result in [
27935            ensure_decommission_start_allowed(DecommissionStartPoolState::Missing),
27936            ensure_decommission_start_allowed(DecommissionStartPoolState::Decommissioned),
27937            ensure_decommission_start_allowed(DecommissionStartPoolState::Blocked),
27938            ensure_decommission_cancel_allowed(false, false, false),
27939            ensure_decommission_clear_allowed(false, false, false, false, false, 0),
27940            ensure_decommission_clear_allowed(true, true, false, true, false, 1),
27941            ensure_decommission_terminal_operation_supported(true, "cancel decommission"),
27942            validate_start_decommission_request(&[], false),
27943            validate_start_decommission_request(&[0], true),
27944            ensure_decommission_start_keeps_active_pool(&PoolMeta::default(), &[]),
27945        ] {
27946            let err = result.expect_err("invalid lifecycle requests must be rejected before mutation");
27947            assert!(matches!(&err, Error::InvalidArgument(_, _, reason) if !reason.is_empty()), "{err:?}");
27948        }
27949    }
27950
27951    #[test]
27952    fn test_ensure_decommission_start_allowed_rejects_missing_pool() {
27953        let err =
27954            ensure_decommission_start_allowed(DecommissionStartPoolState::Missing).expect_err("missing pool should be invalid");
27955        assert!(
27956            err.to_string()
27957                .contains("failed to start decommission: target pool was not found")
27958        );
27959    }
27960
27961    #[test]
27962    fn test_ensure_decommission_start_allowed_rejects_running_state() {
27963        let err = ensure_decommission_start_allowed(DecommissionStartPoolState::Decommissioning)
27964            .expect_err("active decommission should be rejected");
27965        assert!(matches!(err, Error::DecommissionAlreadyRunning));
27966    }
27967
27968    #[test]
27969    fn test_ensure_decommission_start_allowed_rejects_completed_state() {
27970        let err = ensure_decommission_start_allowed(DecommissionStartPoolState::Decommissioned)
27971            .expect_err("completed decommission should be rejected");
27972        assert!(err.to_string().contains("target pool is already decommissioned"));
27973    }
27974
27975    #[test]
27976    fn test_ensure_decommission_start_allowed_rejects_blocked_state() {
27977        let err = ensure_decommission_start_allowed(DecommissionStartPoolState::Blocked)
27978            .expect_err("blocked decommission should be rejected");
27979        assert!(err.to_string().contains("target pool decommission is blocked"));
27980    }
27981
27982    #[test]
27983    fn test_ensure_decommission_start_allowed_allows_active_state() {
27984        assert!(ensure_decommission_start_allowed(DecommissionStartPoolState::Active).is_ok());
27985    }
27986
27987    #[test]
27988    fn test_ensure_decommission_start_allowed_allows_retryable_state() {
27989        assert!(ensure_decommission_start_allowed(DecommissionStartPoolState::Retryable).is_ok());
27990    }
27991
27992    #[test]
27993    fn test_decommission_start_pool_state_reports_missing_pool() {
27994        assert_eq!(decommission_start_pool_state(None), DecommissionStartPoolState::Missing);
27995    }
27996
27997    #[test]
27998    fn test_decommission_start_pool_state_reports_idle_pool_without_decommission_info() {
27999        let pool = PoolStatus {
28000            id: 0,
28001            cmd_line: "pool-0".to_string(),
28002            last_update: OffsetDateTime::UNIX_EPOCH,
28003            decommission: None,
28004        };
28005
28006        assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Active);
28007    }
28008
28009    #[test]
28010    fn test_decommission_start_pool_state_reports_decommissioning_pool_when_not_terminal() {
28011        let pool = PoolStatus {
28012            id: 0,
28013            cmd_line: "pool-0".to_string(),
28014            last_update: OffsetDateTime::UNIX_EPOCH,
28015            decommission: Some(PoolDecommissionInfo {
28016                start_time: Some(OffsetDateTime::UNIX_EPOCH),
28017                complete: false,
28018                failed: false,
28019                canceled: false,
28020                ..Default::default()
28021            }),
28022        };
28023
28024        assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Decommissioning);
28025    }
28026
28027    #[test]
28028    fn test_decommission_start_pool_state_reports_canceled_pool_as_blocked() {
28029        let pool = PoolStatus {
28030            id: 0,
28031            cmd_line: "pool-0".to_string(),
28032            last_update: OffsetDateTime::UNIX_EPOCH,
28033            decommission: Some(PoolDecommissionInfo {
28034                complete: false,
28035                failed: false,
28036                canceled: true,
28037                ..Default::default()
28038            }),
28039        };
28040
28041        assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Blocked);
28042    }
28043
28044    #[test]
28045    fn test_decommission_start_pool_state_reports_failed_pool_as_blocked() {
28046        let pool = PoolStatus {
28047            id: 0,
28048            cmd_line: "pool-0".to_string(),
28049            last_update: OffsetDateTime::UNIX_EPOCH,
28050            decommission: Some(PoolDecommissionInfo {
28051                failed: true,
28052                ..Default::default()
28053            }),
28054        };
28055
28056        assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Blocked);
28057    }
28058
28059    #[test]
28060    fn test_decommission_start_pool_state_reports_terminal_pool_with_unresolved_entries_as_retryable() {
28061        let pool = PoolStatus {
28062            id: 0,
28063            cmd_line: "pool-0".to_string(),
28064            last_update: OffsetDateTime::UNIX_EPOCH,
28065            decommission: Some(PoolDecommissionInfo {
28066                failed: true,
28067                unresolved_entries: vec![DecommissionUnresolvedEntry {
28068                    bucket: "bucket-a".to_string(),
28069                    object: "object-a".to_string(),
28070                    pool_index: 0,
28071                    set_index: 0,
28072                    source_generation: OffsetDateTime::UNIX_EPOCH,
28073                    candidate_count: 1,
28074                    disk_error_count: 1,
28075                    observed_at: OffsetDateTime::UNIX_EPOCH,
28076                    reason: "metadata_resolution_failed".to_string(),
28077                }],
28078                ..Default::default()
28079            }),
28080        };
28081
28082        assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Retryable);
28083    }
28084
28085    #[test]
28086    fn test_decommission_start_pool_state_reports_completed_pool() {
28087        let pool = PoolStatus {
28088            id: 0,
28089            cmd_line: "pool-0".to_string(),
28090            last_update: OffsetDateTime::UNIX_EPOCH,
28091            decommission: Some(PoolDecommissionInfo {
28092                complete: true,
28093                ..Default::default()
28094            }),
28095        };
28096
28097        assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Decommissioned);
28098    }
28099
28100    #[test]
28101    fn test_ensure_decommission_start_keeps_active_pool_rejects_last_active_pool() {
28102        let meta = PoolMeta {
28103            pools: vec![PoolStatus {
28104                id: 0,
28105                cmd_line: "pool-0".to_string(),
28106                last_update: OffsetDateTime::UNIX_EPOCH,
28107                decommission: None,
28108            }],
28109            ..Default::default()
28110        };
28111
28112        let err = ensure_decommission_start_keeps_active_pool(&meta, &[0]).expect_err("last active pool should be rejected");
28113
28114        assert!(err.to_string().contains("at least one active pool must remain"));
28115    }
28116
28117    #[test]
28118    fn decommission_capacity_model_selection_uses_v2_only_with_a_live_fleet_proof() {
28119        let meta = PoolMeta::default();
28120
28121        assert_eq!(
28122            select_decommission_capacity_model(&meta, false).expect("a proofless empty cohort should remain compatible"),
28123            DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION
28124        );
28125        assert_eq!(
28126            select_decommission_capacity_model(&meta, true).expect("an all-v4 proof should authorize the target fence"),
28127            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
28128        );
28129    }
28130
28131    #[test]
28132    fn decommission_capacity_model_selection_keeps_an_active_v1_cohort_sticky() {
28133        let meta = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION]);
28134
28135        assert_eq!(
28136            select_decommission_capacity_model(&meta, true).expect("a fleet upgrade must not change an active v1 lock model"),
28137            DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION
28138        );
28139    }
28140
28141    #[test]
28142    fn decommission_capacity_model_selection_never_downgrades_an_active_v2_cohort() {
28143        let meta = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION]);
28144
28145        assert_eq!(
28146            select_decommission_capacity_model(&meta, true).expect("a live proof should admit another v2 reservation"),
28147            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
28148        );
28149        let err = select_decommission_capacity_model(&meta, false)
28150            .expect_err("proof loss must block new admission instead of falling back to the global lock model");
28151        assert!(err.to_string().contains("active per-target capacity cohort"));
28152    }
28153
28154    #[test]
28155    fn decommission_capacity_model_selection_rejects_a_mixed_active_cohort() {
28156        let meta = decommission_test_active_model_meta(&[
28157            DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION,
28158            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
28159        ]);
28160
28161        let err = select_decommission_capacity_model(&meta, true)
28162            .expect_err("mixed global and per-target lock models cannot be made safe by a fleet proof");
28163        assert!(err.to_string().contains("mixed lock models"));
28164    }
28165
28166    #[test]
28167    fn test_ensure_decommission_start_target_capacity_allows_sufficient_free_space() {
28168        let meta = PoolMeta {
28169            version: POOL_META_VERSION,
28170            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
28171            ..Default::default()
28172        };
28173        let capacity_infos = vec![
28174            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 100, 700, 600),
28175            DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 1_600, 2_000, 400),
28176        ];
28177
28178        assert!(ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos, true).is_ok());
28179    }
28180
28181    #[test]
28182    fn test_ensure_decommission_start_target_capacity_rejects_insufficient_free_space() {
28183        let meta = PoolMeta {
28184            version: POOL_META_VERSION,
28185            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
28186            ..Default::default()
28187        };
28188        let capacity_infos = vec![
28189            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 100, 700, 600),
28190            DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 1_599, 2_000, 401),
28191        ];
28192
28193        let err = ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos, true)
28194            .expect_err("target physical free capacity below the modeled peak should be rejected");
28195
28196        assert!(err.to_string().contains("insufficient reserved physical target capacity"));
28197        assert!(err.to_string().contains("requires 1600 bytes, but 1599 bytes are available"));
28198    }
28199
28200    #[test]
28201    fn test_ensure_decommission_start_target_capacity_ignores_non_active_target_pool() {
28202        let meta = PoolMeta {
28203            version: POOL_META_VERSION,
28204            pools: vec![
28205                decommission_test_pool_status(0, None),
28206                decommission_test_pool_status(
28207                    1,
28208                    Some(PoolDecommissionInfo {
28209                        complete: true,
28210                        ..Default::default()
28211                    }),
28212                ),
28213                decommission_test_pool_status(2, None),
28214            ],
28215            ..Default::default()
28216        };
28217        let capacity_infos = vec![
28218            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 100, 700, 600),
28219            DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 10_000, 10_000, 0),
28220            DecommissionPoolCapacityInfo::for_test(2, DecommissionErasureLayout { data: 4, parity: 4 }, 1_599, 2_000, 401),
28221        ];
28222
28223        let err = ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos, true)
28224            .expect_err("completed pools must not contribute target free capacity");
28225
28226        assert!(err.to_string().contains("requires 1600 bytes, but 1599 bytes are available"));
28227    }
28228
28229    #[test]
28230    fn decommission_capacity_model_converts_different_source_and_target_parity() {
28231        let now = OffsetDateTime::UNIX_EPOCH;
28232        let high_target_parity = build_decommission_capacity_reservation(
28233            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 0, 600, 600),
28234            DecommissionErasureLayout { data: 4, parity: 4 },
28235            uuid::Uuid::new_v4(),
28236            1,
28237            now,
28238        )
28239        .expect("the source and target layouts should be valid");
28240        assert_eq!(high_target_parity.source_data_equivalent_bytes, 400);
28241        assert_eq!(high_target_parity.predicted_physical_bytes, 800);
28242        assert_eq!(high_target_parity.temporary_physical_bytes, 800);
28243        assert_eq!(high_target_parity.peak_physical_bytes, 1_600);
28244
28245        let low_target_parity = build_decommission_capacity_reservation(
28246            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 4 }, 0, 800, 800),
28247            DecommissionErasureLayout { data: 4, parity: 2 },
28248            uuid::Uuid::new_v4(),
28249            2,
28250            now,
28251        )
28252        .expect("the inverse source and target layouts should be valid");
28253        assert_eq!(low_target_parity.source_data_equivalent_bytes, 400);
28254        assert_eq!(low_target_parity.predicted_physical_bytes, 600);
28255        assert_eq!(low_target_parity.peak_physical_bytes, 1_200);
28256    }
28257
28258    #[test]
28259    fn decommission_batch_persists_one_replayable_operation_identity_and_generation() {
28260        let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(1);
28261        let operation_id = uuid::Uuid::new_v4();
28262        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
28263        let capacity_infos = vec![
28264            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
28265            DecommissionPoolCapacityInfo::for_test(1, layout, 0, 20, 20),
28266            DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
28267        ];
28268        let mut meta = PoolMeta {
28269            version: POOL_META_VERSION,
28270            pools: vec![
28271                decommission_test_pool_status(0, None),
28272                decommission_test_pool_status(1, None),
28273                decommission_test_pool_status(2, None),
28274            ],
28275            ..Default::default()
28276        };
28277        meta.decommission(0, capacity_infos[0].space).unwrap();
28278        meta.queue_decommission(1, capacity_infos[1].space).unwrap();
28279
28280        reserve_decommission_start_target_capacity(
28281            &mut meta,
28282            &[0, 1],
28283            &capacity_infos,
28284            operation_id,
28285            17,
28286            now,
28287            DECOMMISSION_CAPACITY_MODEL_VERSION,
28288        )
28289        .expect("the batch reservation should fit exactly");
28290        for idx in [0, 1] {
28291            let reservation = meta.pools[idx]
28292                .decommission
28293                .as_ref()
28294                .and_then(|info| info.capacity_reservation.as_ref())
28295                .expect("each source pool should carry the batch reservation identity");
28296            assert_eq!(reservation.operation_id, operation_id);
28297            assert_eq!(reservation.generation, 17);
28298        }
28299    }
28300
28301    #[test]
28302    fn decommission_capacity_model_reserves_versions_delete_markers_and_temporary_copies_from_physical_usage() {
28303        let reservation = build_decommission_capacity_reservation(
28304            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 2, parity: 2 }, 0, 1_024, 1_024),
28305            DecommissionErasureLayout { data: 2, parity: 2 },
28306            uuid::Uuid::new_v4(),
28307            1,
28308            OffsetDateTime::UNIX_EPOCH,
28309        )
28310        .expect("physical source usage should produce a reservation");
28311
28312        assert_eq!(reservation.source_physical_bytes, 1_024);
28313        assert_eq!(reservation.predicted_physical_bytes, 1_024);
28314        assert_eq!(reservation.temporary_copies, 1);
28315        assert_eq!(reservation.peak_physical_bytes, 2_048);
28316    }
28317
28318    #[test]
28319    fn decommission_runtime_capacity_tracks_own_target_consumption_but_rejects_external_drop() {
28320        let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
28321        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
28322        let initial = vec![
28323            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
28324            DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
28325        ];
28326        let mut meta = PoolMeta {
28327            version: POOL_META_VERSION,
28328            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
28329            ..Default::default()
28330        };
28331        meta.decommission(0, initial[0].space).unwrap();
28332        reserve_decommission_start_target_capacity(
28333            &mut meta,
28334            &[0],
28335            &initial,
28336            uuid::Uuid::new_v4(),
28337            1,
28338            now,
28339            DECOMMISSION_CAPACITY_MODEL_VERSION,
28340        )
28341        .expect("the initial target capacity should fit exactly");
28342
28343        let mutation_id = uuid::Uuid::from_u128(1);
28344        reserve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id, now + Duration::seconds(1))
28345            .expect("the target intent should be durable before its write");
28346        let own_consumption = vec![initial[0], DecommissionPoolCapacityInfo::for_test(1, layout, 50, 60, 10)];
28347        ensure_decommission_capacity_reservations_available(&meta, &own_consumption, "restart")
28348            .expect("a persisted target intent must recognize its own write after restart");
28349        resolve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id)
28350            .expect("the persisted target intent should resolve at commit");
28351        record_decommission_target_consumption(
28352            &mut meta,
28353            0,
28354            1,
28355            DecommissionTargetConsumption {
28356                committed_data_bytes: 10,
28357                target_physical_bytes: 10,
28358                observed_physical_bytes: 10,
28359            },
28360            mutation_id,
28361            now + Duration::seconds(1),
28362        )
28363        .expect("the operation's own target consumption should be persisted");
28364        ensure_decommission_capacity_reservations_available(&meta, &own_consumption, "migration")
28365            .expect("the operation's own committed target bytes must not look like external capacity loss");
28366
28367        let dropped = vec![initial[0], DecommissionPoolCapacityInfo::for_test(1, layout, 39, 60, 21)];
28368        let err = ensure_decommission_capacity_reservations_available(&meta, &dropped, "migration")
28369            .expect_err("runtime migration must stop after a sudden competing write consumes reserved capacity");
28370        assert!(
28371            err.to_string()
28372                .contains("requires 40 physical bytes, but only 39 bytes remain")
28373        );
28374        assert!(is_decommission_capacity_blocked_error(&err));
28375    }
28376
28377    #[test]
28378    fn decommission_target_capacity_reuses_only_matching_pending_intent() {
28379        let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
28380        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
28381        let initial = vec![
28382            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
28383            DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
28384        ];
28385        let mut meta = PoolMeta {
28386            version: POOL_META_VERSION,
28387            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
28388            ..Default::default()
28389        };
28390        meta.decommission(0, initial[0].space).unwrap();
28391        reserve_decommission_start_target_capacity(
28392            &mut meta,
28393            &[0],
28394            &initial,
28395            uuid::Uuid::new_v4(),
28396            1,
28397            now,
28398            DECOMMISSION_CAPACITY_MODEL_VERSION,
28399        )
28400        .expect("the initial target capacity should fit exactly");
28401
28402        let first_mutation_id = uuid::Uuid::from_u128(1);
28403        let second_mutation_id = uuid::Uuid::from_u128(2);
28404        reserve_decommission_target_pending(&mut meta, 0, 1, 10, first_mutation_id, now + Duration::seconds(1))
28405            .expect("the first mutation should persist its target intent");
28406        assert_eq!(
28407            reserve_decommission_target_pending(&mut meta, 0, 1, 10, first_mutation_id, now + Duration::seconds(2))
28408                .expect("the same mutation should reuse its persisted target intent"),
28409            0
28410        );
28411        let err = reserve_decommission_target_pending(&mut meta, 0, 1, 10, second_mutation_id, now + Duration::seconds(2))
28412            .expect_err("a second mutation must not reuse the first mutation's pending intent");
28413        assert!(is_decommission_capacity_blocked_error(&err));
28414        assert!(err.to_string().contains("unresolved target capacity intent"));
28415
28416        resolve_decommission_target_pending(&mut meta, 0, 1, 10, first_mutation_id)
28417            .expect("the first mutation should finalize its intent");
28418        record_decommission_target_consumption(
28419            &mut meta,
28420            0,
28421            1,
28422            DecommissionTargetConsumption {
28423                committed_data_bytes: 10,
28424                target_physical_bytes: 10,
28425                observed_physical_bytes: 10,
28426            },
28427            first_mutation_id,
28428            now + Duration::seconds(2),
28429        )
28430        .expect("the first mutation's consumption should be durable");
28431        reserve_decommission_target_pending(&mut meta, 0, 1, 10, second_mutation_id, now + Duration::seconds(3))
28432            .expect("the second mutation should retry only after the first intent is finalized");
28433        assert_eq!(
28434            meta.pools[0]
28435                .decommission
28436                .as_ref()
28437                .and_then(|info| info.capacity_reservation.as_ref())
28438                .expect("the reservation should remain present")
28439                .pending_target_physical_bytes,
28440            10
28441        );
28442    }
28443
28444    #[test]
28445    fn decommission_target_capacity_allows_same_mutation_to_grow_its_pending_stage() {
28446        let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
28447        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
28448        let initial = vec![
28449            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
28450            DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
28451        ];
28452        let mut meta = PoolMeta {
28453            version: POOL_META_VERSION,
28454            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
28455            ..Default::default()
28456        };
28457        meta.decommission(0, initial[0].space).unwrap();
28458        reserve_decommission_start_target_capacity(
28459            &mut meta,
28460            &[0],
28461            &initial,
28462            uuid::Uuid::new_v4(),
28463            1,
28464            now,
28465            DECOMMISSION_CAPACITY_MODEL_VERSION,
28466        )
28467        .expect("the initial target capacity should fit exactly");
28468
28469        let mutation_id = uuid::Uuid::from_u128(1);
28470        assert_eq!(
28471            reserve_decommission_target_pending(&mut meta, 0, 1, 1, mutation_id, now + Duration::seconds(1))
28472                .expect("the initial multipart stage should persist its one-byte intent"),
28473            1
28474        );
28475        assert_eq!(
28476            reserve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id, now + Duration::seconds(2))
28477                .expect("the same mutation should grow its intent for the commit stage"),
28478            9
28479        );
28480        assert_eq!(
28481            meta.pools[0]
28482                .decommission
28483                .as_ref()
28484                .and_then(|info| info.capacity_reservation.as_ref())
28485                .expect("the reservation should remain present")
28486                .targets[0]
28487                .pending_physical_bytes,
28488            10
28489        );
28490        assert_eq!(
28491            reserve_decommission_target_pending(&mut meta, 0, 1, 1, mutation_id, now + Duration::seconds(3))
28492                .expect("a retry of an earlier stage must reuse the larger same-mutation intent"),
28493            0
28494        );
28495    }
28496
28497    #[test]
28498    fn decommission_capacity_mutation_identity_survives_lease_nonce_rotation() {
28499        let owner = DecommissionCapacityOwner {
28500            source_pool_index: 0,
28501            operation_id: uuid::Uuid::from_u128(1),
28502            generation: 2,
28503            owner_nonce: uuid::Uuid::from_u128(3),
28504            mutation_id: None,
28505        };
28506        let recovered_owner = DecommissionCapacityOwner {
28507            owner_nonce: uuid::Uuid::from_u128(4),
28508            ..owner
28509        };
28510        let first = decommission_capacity_mutation_id(owner, "bucket", "object", Some("version"), false, None);
28511        let recovered = decommission_capacity_mutation_id(recovered_owner, "bucket", "object", Some("version"), false, None);
28512        assert_eq!(first, recovered, "a lease nonce rotation must not change the mutation identity");
28513    }
28514
28515    #[test]
28516    fn exact_delete_capacity_plan_requires_identity_and_exact_size() {
28517        let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
28518        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
28519        let capacity_infos = vec![
28520            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
28521            DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
28522        ];
28523        let mut meta = PoolMeta {
28524            version: POOL_META_VERSION,
28525            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
28526            ..Default::default()
28527        };
28528        meta.decommission(0, capacity_infos[0].space).unwrap();
28529        reserve_decommission_start_target_capacity(
28530            &mut meta,
28531            &[0],
28532            &capacity_infos,
28533            uuid::Uuid::new_v4(),
28534            1,
28535            now,
28536            DECOMMISSION_CAPACITY_MODEL_VERSION,
28537        )
28538        .expect("the exact-delete test reservation should fit");
28539        let exact = ObjectInfo {
28540            bucket: "bucket".to_string(),
28541            name: "object".to_string(),
28542            version_id: Some(uuid::Uuid::from_u128(7)),
28543            mod_time: Some(now),
28544            size: 10,
28545            ..Default::default()
28546        };
28547        let owner = {
28548            let reservation = meta.pools[0]
28549                .decommission
28550                .as_ref()
28551                .and_then(|info| info.capacity_reservation.as_ref())
28552                .expect("the exact-delete test reservation should exist");
28553            DecommissionCapacityOwner {
28554                source_pool_index: 0,
28555                operation_id: reservation.operation_id,
28556                generation: reservation.generation,
28557                owner_nonce: reservation.owner_nonce,
28558                mutation_id: None,
28559            }
28560        };
28561        let version_id = exact.version_id.map(|version_id| version_id.to_string());
28562        let mutation_id = decommission_capacity_mutation_id(
28563            owner,
28564            &exact.bucket,
28565            &exact.name,
28566            version_id.as_deref(),
28567            exact.delete_marker,
28568            exact.mod_time,
28569        );
28570        reserve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id, now + Duration::seconds(1))
28571            .expect("the exact-delete test intent should be reserved");
28572
28573        let plan = plan_exact_delete_capacity_reconciliations(&meta, &exact.name, &exact)
28574            .expect("the exact identity should match the pending intent");
28575        assert_eq!(plan.len(), 1);
28576        assert_eq!(plan[0].source_pool_index, 0);
28577        assert_eq!(plan[0].target_pool_index, 1);
28578        assert_eq!(plan[0].expected_data_bytes, 10);
28579        assert_eq!(plan[0].expected_target_physical_bytes, 10);
28580
28581        let mismatched_size = ObjectInfo {
28582            size: 9,
28583            ..exact.clone()
28584        };
28585        let mismatched_size = plan_exact_delete_capacity_reconciliations(&meta, &mismatched_size.name, &mismatched_size)
28586            .expect_err("a different exact size must not consume the pending intent");
28587        assert!(mismatched_size.to_string().contains("does not match the exact object size"));
28588
28589        let directory_exact = ObjectInfo {
28590            name: "directory/".to_string(),
28591            ..exact.clone()
28592        };
28593        let internal_directory = rustfs_utils::path::encode_dir_object(&directory_exact.name);
28594        let internal_directory_mutation_id = decommission_capacity_mutation_id(
28595            owner,
28596            &directory_exact.bucket,
28597            &internal_directory,
28598            version_id.as_deref(),
28599            directory_exact.delete_marker,
28600            directory_exact.mod_time,
28601        );
28602        meta.pools[0]
28603            .decommission
28604            .as_mut()
28605            .and_then(|info| info.capacity_reservation.as_mut())
28606            .expect("the exact-delete test reservation should exist")
28607            .targets[0]
28608            .pending_mutation_id = Some(internal_directory_mutation_id);
28609        let directory_plan = plan_exact_delete_capacity_reconciliations(&meta, &internal_directory, &directory_exact)
28610            .expect("an internally encoded directory intent should match its logical exact object");
28611        assert_eq!(directory_plan[0].mutation_id, internal_directory_mutation_id);
28612
28613        let logical_directory_mutation_id = decommission_capacity_mutation_id(
28614            owner,
28615            &directory_exact.bucket,
28616            &directory_exact.name,
28617            version_id.as_deref(),
28618            directory_exact.delete_marker,
28619            directory_exact.mod_time,
28620        );
28621        meta.pools[0]
28622            .decommission
28623            .as_mut()
28624            .and_then(|info| info.capacity_reservation.as_mut())
28625            .expect("the exact-delete test reservation should exist")
28626            .targets[0]
28627            .pending_mutation_id = Some(logical_directory_mutation_id);
28628        let directory_plan = plan_exact_delete_capacity_reconciliations(&meta, &internal_directory, &directory_exact)
28629            .expect("a logical directory intent should match its internally encoded delete path");
28630        assert_eq!(directory_plan[0].mutation_id, logical_directory_mutation_id);
28631
28632        meta.pools[0]
28633            .decommission
28634            .as_mut()
28635            .and_then(|info| info.capacity_reservation.as_mut())
28636            .expect("the exact-delete test reservation should exist")
28637            .targets[0]
28638            .pending_mutation_id = None;
28639        let unidentified = plan_exact_delete_capacity_reconciliations(&meta, &exact.name, &exact)
28640            .expect_err("an unidentified pending intent must fail closed");
28641        assert!(unidentified.to_string().contains("without an object identity"));
28642
28643        let mut opts = ObjectOptions::default();
28644        let unfenced = ensure_exact_delete_capacity_namespace_fences(&opts, &exact.bucket, &exact.name)
28645            .expect_err("capacity reconciliation must reject a missing object namespace fence");
28646        assert!(unfenced.to_string().contains("requires an object namespace fence"));
28647        opts.ensure_namespace_lock_fence();
28648        ensure_exact_delete_capacity_namespace_fences(&opts, &exact.bucket, &exact.name)
28649            .expect("a live object namespace fence should admit capacity reconciliation");
28650    }
28651
28652    #[test]
28653    fn ordinary_write_admission_shares_a_reserved_target_without_becoming_its_owner() {
28654        let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
28655        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
28656        let capacity_infos = vec![
28657            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
28658            DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
28659        ];
28660        let mut meta = PoolMeta {
28661            version: POOL_META_VERSION,
28662            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
28663            ..Default::default()
28664        };
28665        meta.decommission(0, capacity_infos[0].space).unwrap();
28666        reserve_decommission_start_target_capacity(
28667            &mut meta,
28668            &[0],
28669            &capacity_infos,
28670            uuid::Uuid::new_v4(),
28671            1,
28672            now,
28673            DECOMMISSION_CAPACITY_MODEL_VERSION,
28674        )
28675        .expect("the decommission reservation should fit");
28676
28677        for admission in [
28678            DecommissionCapacityAdmission::Mutation,
28679            DecommissionCapacityAdmission::BatchDelete,
28680            DecommissionCapacityAdmission::ScannerBacklog,
28681        ] {
28682            ensure_external_decommission_target_admission(&meta, 1, admission)
28683                .expect("a healthy target must remain writable while sharing capacity with migration");
28684        }
28685        assert!(matches!(
28686            ensure_external_decommission_target_admission(&meta, 1, DecommissionCapacityAdmission::Heal),
28687            Err(Error::SlowDown)
28688        ));
28689        let rebalance_opts = ObjectOptions {
28690            data_movement: true,
28691            src_pool_idx: 0,
28692            ..Default::default()
28693        };
28694        assert!(
28695            DecommissionCapacityOwner::from_options(&rebalance_opts).is_none(),
28696            "rebalance data movement must remain an external capacity consumer"
28697        );
28698        let reservation = meta.pools[0]
28699            .decommission
28700            .as_ref()
28701            .and_then(|info| info.capacity_reservation.as_ref())
28702            .expect("the active reservation should remain available");
28703        let expected_owner = DecommissionCapacityOwner {
28704            source_pool_index: 0,
28705            operation_id: reservation.operation_id,
28706            generation: reservation.generation,
28707            owner_nonce: reservation.owner_nonce,
28708            mutation_id: None,
28709        };
28710        ensure_decommission_target_owner_admission(&meta, expected_owner, 1, 10, now)
28711            .expect("the reservation owner should consume its own allocation under the shared boundary");
28712        let mut decommission_opts = rebalance_opts;
28713        expected_owner.apply_to(&mut decommission_opts);
28714        assert_eq!(DecommissionCapacityOwner::from_options(&decommission_opts), Some(expected_owner));
28715
28716        meta.pools[0]
28717            .decommission
28718            .as_mut()
28719            .expect("active source")
28720            .capacity_reservation = None;
28721        for admission in [
28722            DecommissionCapacityAdmission::Mutation,
28723            DecommissionCapacityAdmission::BatchDelete,
28724            DecommissionCapacityAdmission::ScannerBacklog,
28725        ] {
28726            assert!(
28727                matches!(ensure_external_decommission_target_admission(&meta, 1, admission), Err(Error::SlowDown)),
28728                "shared capacity must not bypass an active source's missing durable ledger"
28729            );
28730        }
28731    }
28732
28733    #[test]
28734    fn external_decommission_admission_fences_suspended_sources_but_preserves_repair() {
28735        let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
28736        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
28737        let capacity_infos = vec![
28738            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
28739            DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
28740        ];
28741        let mut active = PoolMeta {
28742            version: POOL_META_VERSION,
28743            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
28744            ..Default::default()
28745        };
28746        active
28747            .decommission(0, capacity_infos[0].space)
28748            .expect("start the source admission fixture");
28749        reserve_decommission_start_target_capacity(
28750            &mut active,
28751            &[0],
28752            &capacity_infos,
28753            uuid::Uuid::new_v4(),
28754            1,
28755            now,
28756            DECOMMISSION_CAPACITY_MODEL_VERSION,
28757        )
28758        .expect("the active source must have a valid reservation to isolate its write fence");
28759
28760        for (state, queued, failed, canceled, complete) in [
28761            ("running", false, false, false, false),
28762            ("queued", true, false, false, false),
28763            ("failed", false, true, false, false),
28764            ("canceled", false, false, true, false),
28765            ("completed", false, false, false, true),
28766        ] {
28767            let mut meta = active.clone();
28768            let info = meta.pools[0].decommission.as_mut().expect("the source fixture must exist");
28769            info.queued = queued;
28770            info.failed = failed;
28771            info.canceled = canceled;
28772            info.complete = complete;
28773            if queued || failed || canceled || complete {
28774                info.start_time = None;
28775            }
28776            for admission in [
28777                DecommissionCapacityAdmission::Mutation,
28778                DecommissionCapacityAdmission::BatchDelete,
28779            ] {
28780                assert!(
28781                    matches!(ensure_external_decommission_target_admission(&meta, 0, admission), Err(Error::SlowDown)),
28782                    "{state} source must reject new publication until its decommission metadata is cleared"
28783                );
28784            }
28785            let existing_multipart =
28786                ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::ExistingMultipart);
28787            if active_decommission_source_indices(&meta).contains(&0) {
28788                existing_multipart
28789                    .unwrap_or_else(|err| panic!("{state} source must allow an existing multipart upload to drain: {err}"));
28790            } else {
28791                assert!(
28792                    matches!(existing_multipart, Err(Error::SlowDown)),
28793                    "{state} terminal source must reject an existing multipart publication"
28794                );
28795            }
28796            ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::Heal)
28797                .unwrap_or_else(|err| panic!("{state} source repair must retain its capacity-only admission: {err}"));
28798            let scanner_result =
28799                ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::ScannerBacklog);
28800            assert_eq!(
28801                meta.scanner_pause_backlog_pool_writable(0),
28802                failed || canceled,
28803                "{state} scanner selection"
28804            );
28805            if failed || canceled {
28806                scanner_result.unwrap_or_else(|err| panic!("{state} scanner membership repair must remain writable: {err}"));
28807            } else {
28808                assert!(
28809                    matches!(scanner_result, Err(Error::SlowDown)),
28810                    "{state} scanner publication must reject its source"
28811                );
28812            }
28813            meta.pools[0].decommission = None;
28814            ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::Mutation)
28815                .unwrap_or_else(|err| panic!("cleared {state} source must become writable again: {err}"));
28816            ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::ScannerBacklog)
28817                .unwrap_or_else(|err| panic!("cleared {state} scanner source must rejoin membership: {err}"));
28818        }
28819        assert!(!active.scanner_pause_backlog_pool_writable(active.pools.len()));
28820        assert!(matches!(
28821            ensure_external_decommission_target_admission(
28822                &active,
28823                active.pools.len(),
28824                DecommissionCapacityAdmission::ScannerBacklog
28825            ),
28826            Err(Error::SlowDown)
28827        ));
28828    }
28829
28830    #[test]
28831    fn decommission_physical_capacity_uses_per_set_bottleneck_and_widest_usage() {
28832        let disks = [10_u64, 20, 30, 40]
28833            .into_iter()
28834            .enumerate()
28835            .map(|(disk_index, available_space)| rustfs_madmin::Disk {
28836                endpoint: format!("http://node-{disk_index}"),
28837                drive_path: format!("/disk-{disk_index}"),
28838                state: "ok".to_string(),
28839                total_space: 100,
28840                used_space: 100 - available_space,
28841                available_space,
28842                pool_index: 0,
28843                set_index: 0,
28844                disk_index: disk_index as i32,
28845                ..Default::default()
28846            })
28847            .collect::<Vec<_>>();
28848        let capacity = decommission_physical_pool_capacity(
28849            &disks,
28850            0,
28851            DecommissionErasureLayout { data: 2, parity: 2 },
28852            PoolSpaceInfo {
28853                free: 0,
28854                total: 0,
28855                used: 0,
28856            },
28857        );
28858
28859        assert_eq!(capacity, (400, 40, 360));
28860    }
28861
28862    #[test]
28863    fn decommission_terminal_transitions_release_capacity_reservations() {
28864        let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(1);
28865        let reservation = build_decommission_capacity_reservation(
28866            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 1, parity: 0 }, 0, 50, 50),
28867            DecommissionErasureLayout { data: 1, parity: 0 },
28868            uuid::Uuid::new_v4(),
28869            7,
28870            now,
28871        )
28872        .expect("test reservation should be valid");
28873        let active = PoolMeta {
28874            version: POOL_META_VERSION,
28875            pools: vec![decommission_test_pool_status(
28876                0,
28877                Some(PoolDecommissionInfo {
28878                    start_time: Some(now),
28879                    capacity_reservation: Some(reservation),
28880                    ..Default::default()
28881                }),
28882            )],
28883            ..Default::default()
28884        };
28885
28886        for (reason, transition) in [
28887            (
28888                DECOMMISSION_CAPACITY_RELEASE_CANCELED,
28889                PoolMeta::decommission_cancel as fn(&mut PoolMeta, usize) -> bool,
28890            ),
28891            (DECOMMISSION_CAPACITY_RELEASE_FAILED, PoolMeta::decommission_failed),
28892            (DECOMMISSION_CAPACITY_RELEASE_COMPLETED, PoolMeta::decommission_complete),
28893        ] {
28894            let mut meta = active.clone();
28895            assert!(transition(&mut meta, 0));
28896            let released = meta.pools[0]
28897                .decommission
28898                .as_ref()
28899                .and_then(|info| info.capacity_reservation.as_ref())
28900                .expect("terminal metadata should retain reservation observability");
28901            assert!(released.released_at.is_some());
28902            assert_eq!(released.release_reason.as_deref(), Some(reason));
28903            assert_eq!(
28904                released.operation_id,
28905                active.pools[0]
28906                    .decommission
28907                    .as_ref()
28908                    .unwrap()
28909                    .capacity_reservation
28910                    .as_ref()
28911                    .unwrap()
28912                    .operation_id
28913            );
28914        }
28915    }
28916
28917    #[test]
28918    fn decommission_capacity_target_round_trip_preserves_temporary_mutation_without_pending_intent() {
28919        let mutation_id = uuid::Uuid::new_v4();
28920        let target = DecommissionCapacityTarget {
28921            pool_index: 1,
28922            layout: DecommissionErasureLayout { data: 2, parity: 2 },
28923            physical_total_at_reservation: 200,
28924            physical_free_at_reservation: 200,
28925            reserved_physical_bytes: 200,
28926            consumed_physical_bytes: 0,
28927            observed_physical_bytes: 1,
28928            inflight_physical_bytes: 1,
28929            pending_physical_bytes: 0,
28930            pending_mutation_id: None,
28931            temporary_mutations: vec![DecommissionCapacityTemporaryMutation {
28932                mutation_id,
28933                physical_bytes: 1,
28934            }],
28935        };
28936        let mut encoded = Vec::new();
28937        target
28938            .serialize(&mut Serializer::new(&mut encoded))
28939            .expect("capacity target should serialize");
28940
28941        let restored: DecommissionCapacityTarget =
28942            rmp_serde::from_slice(&encoded).expect("capacity target with a released pending intent should deserialize");
28943
28944        assert_eq!(restored, target);
28945    }
28946
28947    #[test]
28948    fn temporary_cleanup_releases_only_its_exact_scoped_mutation_and_is_idempotent() {
28949        let mutation_id = uuid::Uuid::new_v4();
28950        let foreign_id = uuid::Uuid::new_v4();
28951        let mut meta = decommission_test_cleanup_meta(
28952            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
28953            vec![
28954                DecommissionCapacityTemporaryMutation {
28955                    mutation_id,
28956                    physical_bytes: 10,
28957                },
28958                DecommissionCapacityTemporaryMutation {
28959                    mutation_id: foreign_id,
28960                    physical_bytes: 20,
28961                },
28962            ],
28963            30,
28964            None,
28965        );
28966
28967        assert!(
28968            release_decommission_target_inflight(
28969                &mut meta,
28970                0,
28971                1,
28972                0,
28973                mutation_id,
28974                DecommissionCapacityReleaseProof::confirmed_absence(true),
28975                OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
28976            )
28977            .expect("confirmed absence should release the exact mutation")
28978        );
28979        let reservation = meta.pools[0]
28980            .decommission
28981            .as_ref()
28982            .and_then(|info| info.capacity_reservation.as_ref())
28983            .expect("cleanup reservation should remain present");
28984        assert_eq!(reservation.inflight_target_physical_bytes, 20);
28985        assert_eq!(
28986            reservation.targets[0].temporary_mutations,
28987            vec![DecommissionCapacityTemporaryMutation {
28988                mutation_id: foreign_id,
28989                physical_bytes: 20,
28990            }]
28991        );
28992
28993        let before_replay = reservation.clone();
28994        assert!(
28995            !release_decommission_target_inflight(
28996                &mut meta,
28997                0,
28998                1,
28999                0,
29000                mutation_id,
29001                DecommissionCapacityReleaseProof::confirmed_absence(true),
29002                OffsetDateTime::UNIX_EPOCH + Duration::seconds(2),
29003            )
29004            .expect("repeated confirmed absence should be a metadata no-op")
29005        );
29006        let after_replay = meta.pools[0]
29007            .decommission
29008            .as_ref()
29009            .and_then(|info| info.capacity_reservation.as_ref())
29010            .expect("cleanup reservation should remain present");
29011        assert_eq!(after_replay, &before_replay);
29012    }
29013
29014    #[test]
29015    fn temporary_cleanup_uses_aggregate_fallback_only_for_legacy_unscoped_state() {
29016        let mutation_id = uuid::Uuid::new_v4();
29017        let mut meta = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, Vec::new(), 30, None);
29018
29019        assert!(
29020            release_decommission_target_inflight(
29021                &mut meta,
29022                0,
29023                1,
29024                12,
29025                mutation_id,
29026                DecommissionCapacityReleaseProof::confirmed_absence(true),
29027                OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
29028            )
29029            .expect("legacy unscoped cleanup should use its observed release delta")
29030        );
29031        let reservation = meta.pools[0]
29032            .decommission
29033            .as_ref()
29034            .and_then(|info| info.capacity_reservation.as_ref())
29035            .expect("legacy cleanup reservation should remain present");
29036        assert_eq!(reservation.inflight_target_physical_bytes, 18);
29037        assert_eq!(reservation.targets[0].inflight_physical_bytes, 18);
29038    }
29039
29040    #[test]
29041    fn temporary_cleanup_clears_only_a_matching_pending_identity_after_confirmed_absence() {
29042        let mutation_id = uuid::Uuid::new_v4();
29043        let mut meta = decommission_test_cleanup_meta(
29044            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
29045            Vec::new(),
29046            0,
29047            Some((mutation_id, 15)),
29048        );
29049
29050        assert!(
29051            release_decommission_target_inflight(
29052                &mut meta,
29053                0,
29054                1,
29055                7,
29056                mutation_id,
29057                DecommissionCapacityReleaseProof::confirmed_absence(true),
29058                OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
29059            )
29060            .expect("a confirmed absent upload should clear its matching pending identity")
29061        );
29062        let reservation = meta.pools[0]
29063            .decommission
29064            .as_ref()
29065            .and_then(|info| info.capacity_reservation.as_ref())
29066            .expect("cleanup reservation should remain present");
29067        assert_eq!(reservation.pending_target_physical_bytes, 0);
29068        assert_eq!(reservation.targets[0].pending_mutation_id, None);
29069        assert_eq!(reservation.targets[0].pending_physical_bytes, 0);
29070    }
29071
29072    #[test]
29073    fn temporary_cleanup_preserves_pending_for_an_already_published_target() {
29074        let mutation_id = uuid::Uuid::new_v4();
29075        let mut meta = decommission_test_cleanup_meta(
29076            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
29077            vec![DecommissionCapacityTemporaryMutation {
29078                mutation_id,
29079                physical_bytes: 5,
29080            }],
29081            5,
29082            Some((mutation_id, 15)),
29083        );
29084
29085        assert!(
29086            release_decommission_target_inflight(
29087                &mut meta,
29088                0,
29089                1,
29090                0,
29091                mutation_id,
29092                DecommissionCapacityReleaseProof::confirmed_absence(false),
29093                OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
29094            )
29095            .expect("published-target cleanup should release only its temporary staging state")
29096        );
29097        let reservation = meta.pools[0]
29098            .decommission
29099            .as_ref()
29100            .and_then(|info| info.capacity_reservation.as_ref())
29101            .expect("cleanup reservation should remain present");
29102        assert_eq!(reservation.inflight_target_physical_bytes, 0);
29103        assert_eq!(reservation.pending_target_physical_bytes, 15);
29104        assert_eq!(reservation.targets[0].pending_mutation_id, Some(mutation_id));
29105        assert_eq!(reservation.targets[0].pending_physical_bytes, 15);
29106        assert!(reservation.targets[0].temporary_mutations.is_empty());
29107    }
29108
29109    #[test]
29110    fn published_pending_ignores_a_delayed_release_observation() {
29111        let mutation_id = uuid::Uuid::new_v4();
29112        let mut meta = decommission_test_cleanup_meta(
29113            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
29114            Vec::new(),
29115            0,
29116            Some((mutation_id, 15)),
29117        );
29118        let before = meta.pools[0]
29119            .decommission
29120            .as_ref()
29121            .and_then(|info| info.capacity_reservation.as_ref())
29122            .expect("published-target reservation should remain present")
29123            .clone();
29124
29125        assert!(
29126            !release_decommission_target_inflight(
29127                &mut meta,
29128                0,
29129                1,
29130                7,
29131                mutation_id,
29132                DecommissionCapacityReleaseProof::confirmed_absence(false),
29133                OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
29134            )
29135            .expect("a delayed statfs release must not block published-target reconciliation")
29136        );
29137        let after = meta.pools[0]
29138            .decommission
29139            .as_ref()
29140            .and_then(|info| info.capacity_reservation.as_ref())
29141            .expect("published-target reservation should remain present");
29142        assert_eq!(after, &before);
29143    }
29144
29145    #[test]
29146    fn temporary_cleanup_fails_closed_on_a_foreign_scoped_release_delta() {
29147        let foreign_id = uuid::Uuid::new_v4();
29148        let mut meta = decommission_test_cleanup_meta(
29149            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
29150            vec![DecommissionCapacityTemporaryMutation {
29151                mutation_id: foreign_id,
29152                physical_bytes: 20,
29153            }],
29154            20,
29155            None,
29156        );
29157        let before = meta.pools[0]
29158            .decommission
29159            .as_ref()
29160            .and_then(|info| info.capacity_reservation.as_ref())
29161            .expect("cleanup reservation should remain present")
29162            .clone();
29163
29164        let err = release_decommission_target_inflight(
29165            &mut meta,
29166            0,
29167            1,
29168            5,
29169            uuid::Uuid::new_v4(),
29170            DecommissionCapacityReleaseProof::confirmed_absence(true),
29171            OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
29172        )
29173        .expect_err("an observed release cannot be charged to a foreign scoped mutation");
29174        assert!(err.to_string().contains("cannot be attributed"));
29175        let after = meta.pools[0]
29176            .decommission
29177            .as_ref()
29178            .and_then(|info| info.capacity_reservation.as_ref())
29179            .expect("cleanup reservation should remain present");
29180        assert_eq!(after, &before);
29181    }
29182
29183    #[test]
29184    fn temporary_cleanup_missing_identity_and_nonzero_delta_is_a_metadata_noop() {
29185        let mut meta = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, Vec::new(), 0, None);
29186        let before = meta.pools[0]
29187            .decommission
29188            .as_ref()
29189            .and_then(|info| info.capacity_reservation.as_ref())
29190            .expect("cleanup reservation should remain present")
29191            .clone();
29192
29193        assert!(
29194            !release_decommission_target_inflight(
29195                &mut meta,
29196                0,
29197                1,
29198                7,
29199                uuid::Uuid::new_v4(),
29200                DecommissionCapacityReleaseProof::confirmed_absence(true),
29201                OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
29202            )
29203            .expect("confirmed absence without tracked state should ignore a delayed capacity observation")
29204        );
29205        let after = meta.pools[0]
29206            .decommission
29207            .as_ref()
29208            .and_then(|info| info.capacity_reservation.as_ref())
29209            .expect("cleanup reservation should remain present");
29210        assert_eq!(after, &before);
29211    }
29212
29213    #[test]
29214    fn temporary_cleanup_missing_identity_fails_closed_on_foreign_pending_state() {
29215        let foreign_id = uuid::Uuid::new_v4();
29216        let mut meta = decommission_test_cleanup_meta(
29217            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
29218            Vec::new(),
29219            0,
29220            Some((foreign_id, 11)),
29221        );
29222        let before = meta.pools[0]
29223            .decommission
29224            .as_ref()
29225            .and_then(|info| info.capacity_reservation.as_ref())
29226            .expect("cleanup reservation should remain present")
29227            .clone();
29228
29229        let err = release_decommission_target_inflight(
29230            &mut meta,
29231            0,
29232            1,
29233            7,
29234            uuid::Uuid::new_v4(),
29235            DecommissionCapacityReleaseProof::confirmed_absence(true),
29236            OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
29237        )
29238        .expect_err("a delayed release observation must not clear a foreign pending mutation");
29239        assert!(err.to_string().contains("cannot be attributed"));
29240        let after = meta.pools[0]
29241            .decommission
29242            .as_ref()
29243            .and_then(|info| info.capacity_reservation.as_ref())
29244            .expect("cleanup reservation should remain present");
29245        assert_eq!(after, &before);
29246    }
29247
29248    #[test]
29249    fn pool_meta_v2_round_trip_preserves_each_supported_capacity_lock_model() {
29250        for model_version in [
29251            DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION,
29252            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
29253        ] {
29254            let meta = decommission_test_cleanup_meta(model_version, Vec::new(), 0, None);
29255            let encoded = meta
29256                .encode_config_data_for_test()
29257                .expect("supported capacity lock model should encode in pool metadata V2");
29258            let mut restored = PoolMeta::default();
29259            restored
29260                .load_from_config_data(encoded)
29261                .expect("supported capacity lock model should decode from pool metadata V2");
29262            assert_eq!(
29263                restored.pools[0]
29264                    .decommission
29265                    .as_ref()
29266                    .and_then(|info| info.capacity_reservation.as_ref())
29267                    .expect("round-tripped reservation should remain present")
29268                    .model_version,
29269                model_version
29270            );
29271        }
29272    }
29273
29274    #[test]
29275    fn pool_meta_v2_decode_rejects_mixed_active_capacity_lock_models() {
29276        let mut mixed = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, Vec::new(), 0, None);
29277        let mut target_fence_source =
29278            decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, Vec::new(), 0, None)
29279                .pools
29280                .remove(0);
29281        target_fence_source.id = 2;
29282        target_fence_source.cmd_line = "pool-2".to_string();
29283        target_fence_source
29284            .decommission
29285            .as_mut()
29286            .and_then(|info| info.capacity_reservation.as_mut())
29287            .expect("second active reservation should remain present")
29288            .source_pool_index = 2;
29289        mixed.pools.push(target_fence_source);
29290
29291        let encoded = mixed
29292            .encode_config_data_for_test()
29293            .expect("the decode test must be able to construct a mixed persisted payload");
29294        let mut restored = PoolMeta::default();
29295        let err = restored
29296            .load_from_config_data(encoded)
29297            .expect_err("mixed active lock models must fail closed while loading pool metadata");
29298        assert!(err.to_string().contains("mixed lock models"));
29299    }
29300
29301    #[test]
29302    fn decommission_capacity_reservation_recovers_expired_lease_after_restart_round_trip() {
29303        let created_at = OffsetDateTime::UNIX_EPOCH + Duration::hours(1);
29304        let recovered_at = created_at + DECOMMISSION_CAPACITY_RESERVATION_TTL + Duration::seconds(1);
29305        let mut reservation = build_decommission_capacity_reservation(
29306            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 2, parity: 2 }, 0, 100, 100),
29307            DecommissionErasureLayout { data: 2, parity: 2 },
29308            uuid::Uuid::new_v4(),
29309            41,
29310            created_at,
29311        )
29312        .expect("test reservation should be valid");
29313        reservation.targets.push(DecommissionCapacityTarget {
29314            pool_index: 1,
29315            layout: DecommissionErasureLayout { data: 2, parity: 2 },
29316            physical_total_at_reservation: 200,
29317            physical_free_at_reservation: 200,
29318            reserved_physical_bytes: 200,
29319            consumed_physical_bytes: 0,
29320            observed_physical_bytes: 0,
29321            inflight_physical_bytes: 0,
29322            pending_physical_bytes: 0,
29323            pending_mutation_id: None,
29324            temporary_mutations: Vec::new(),
29325        });
29326        let operation_id = reservation.operation_id;
29327        let owner_nonce = reservation.owner_nonce;
29328        let mut meta = PoolMeta {
29329            version: POOL_META_VERSION,
29330            pools: vec![decommission_test_pool_status(
29331                0,
29332                Some(PoolDecommissionInfo {
29333                    start_time: Some(created_at),
29334                    capacity_reservation: Some(reservation),
29335                    ..Default::default()
29336                }),
29337            )],
29338            ..Default::default()
29339        };
29340
29341        let encoded = meta
29342            .encode_config_data_for_test()
29343            .expect("capacity reservation should persist in V2");
29344        let mut restored = PoolMeta::default();
29345        restored
29346            .load_from_config_data(encoded)
29347            .expect("capacity reservation should survive a restart round trip");
29348        let info = restored.pools[0]
29349            .decommission
29350            .as_mut()
29351            .expect("active decommission should restore");
29352        let recovered = info.capacity_reservation.as_mut().expect("reservation should remain present");
29353        assert!(renew_decommission_capacity_reservation(recovered, recovered_at, true));
29354        assert_eq!(recovered.operation_id, operation_id);
29355        assert_eq!(recovered.generation, 41);
29356        assert_ne!(recovered.owner_nonce, owner_nonce);
29357        assert_eq!(recovered.recovered_at, Some(recovered_at));
29358        assert_eq!(recovered.expires_at, recovered_at + DECOMMISSION_CAPACITY_RESERVATION_TTL);
29359        assert_eq!(next_decommission_capacity_generation(&restored).unwrap(), 42);
29360
29361        meta.decommission_failed(0);
29362        let terminal = meta.pools[0]
29363            .decommission
29364            .as_ref()
29365            .and_then(|info| info.capacity_reservation.as_ref())
29366            .expect("failed metadata should retain the released reservation");
29367        assert!(!terminal.active());
29368    }
29369
29370    #[test]
29371    fn decommission_restart_reconstructs_a_missing_capacity_reservation_fail_closed() {
29372        let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(2);
29373        let mut meta = PoolMeta {
29374            version: POOL_META_VERSION,
29375            pools: vec![
29376                decommission_test_pool_status(
29377                    0,
29378                    Some(PoolDecommissionInfo {
29379                        start_time: Some(now - Duration::minutes(1)),
29380                        ..Default::default()
29381                    }),
29382                ),
29383                decommission_test_pool_status(1, None),
29384            ],
29385            ..Default::default()
29386        };
29387        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
29388        let capacity_infos = vec![
29389            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
29390            DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
29391        ];
29392
29393        let recovered_indices = recover_decommission_capacity_reservations(&mut meta, &capacity_infos, now, true)
29394            .expect("restart recovery should rebuild the missing reservation while capacity still fits");
29395        assert_eq!(recovered_indices, vec![0]);
29396        let reservation = meta.pools[0]
29397            .decommission
29398            .as_ref()
29399            .and_then(|info| info.capacity_reservation.as_ref())
29400            .expect("restart recovery should persist a replacement reservation");
29401        assert_eq!(reservation.peak_physical_bytes, 60);
29402        assert_eq!(reservation.recovered_at, Some(now));
29403        assert_eq!(reservation.generation, 1);
29404    }
29405
29406    #[test]
29407    fn test_ensure_decommission_start_pool_states_rejects_blocked_pool() {
29408        let meta = PoolMeta {
29409            pools: vec![
29410                PoolStatus {
29411                    id: 0,
29412                    cmd_line: "pool-0".to_string(),
29413                    last_update: OffsetDateTime::UNIX_EPOCH,
29414                    decommission: Some(PoolDecommissionInfo {
29415                        failed: true,
29416                        ..Default::default()
29417                    }),
29418                },
29419                PoolStatus {
29420                    id: 1,
29421                    cmd_line: "pool-1".to_string(),
29422                    last_update: OffsetDateTime::UNIX_EPOCH,
29423                    decommission: None,
29424                },
29425            ],
29426            ..Default::default()
29427        };
29428
29429        let err = ensure_decommission_start_pool_states(&meta, &[0]).expect_err("blocked pool should be rejected");
29430
29431        assert!(err.to_string().contains("target pool decommission is blocked"));
29432    }
29433
29434    #[test]
29435    fn test_ensure_decommission_start_pool_states_allows_active_pool_with_remaining_active_pool() {
29436        let meta = PoolMeta {
29437            pools: vec![
29438                PoolStatus {
29439                    id: 0,
29440                    cmd_line: "pool-0".to_string(),
29441                    last_update: OffsetDateTime::UNIX_EPOCH,
29442                    decommission: None,
29443                },
29444                PoolStatus {
29445                    id: 1,
29446                    cmd_line: "pool-1".to_string(),
29447                    last_update: OffsetDateTime::UNIX_EPOCH,
29448                    decommission: None,
29449                },
29450            ],
29451            ..Default::default()
29452        };
29453
29454        assert!(ensure_decommission_start_pool_states(&meta, &[0]).is_ok());
29455    }
29456
29457    #[test]
29458    fn test_ensure_decommission_start_pool_states_allows_retryable_pool_with_active_peer() {
29459        let meta = PoolMeta {
29460            pools: vec![
29461                PoolStatus {
29462                    id: 0,
29463                    cmd_line: "pool-0".to_string(),
29464                    last_update: OffsetDateTime::UNIX_EPOCH,
29465                    decommission: Some(PoolDecommissionInfo {
29466                        failed: true,
29467                        unresolved_entries: vec![DecommissionUnresolvedEntry {
29468                            bucket: "bucket-a".to_string(),
29469                            object: "object-a".to_string(),
29470                            pool_index: 0,
29471                            set_index: 0,
29472                            source_generation: OffsetDateTime::UNIX_EPOCH,
29473                            candidate_count: 1,
29474                            disk_error_count: 1,
29475                            observed_at: OffsetDateTime::UNIX_EPOCH,
29476                            reason: "metadata_resolution_failed".to_string(),
29477                        }],
29478                        ..Default::default()
29479                    }),
29480                },
29481                PoolStatus {
29482                    id: 1,
29483                    cmd_line: "pool-1".to_string(),
29484                    last_update: OffsetDateTime::UNIX_EPOCH,
29485                    decommission: None,
29486                },
29487            ],
29488            ..Default::default()
29489        };
29490
29491        assert!(ensure_decommission_start_pool_states(&meta, &[0]).is_ok());
29492    }
29493
29494    #[test]
29495    fn test_ensure_valid_decommission_pool_index_accepts_in_range_index() {
29496        assert!(ensure_valid_decommission_pool_index(4, 3).is_ok());
29497    }
29498
29499    #[test]
29500    fn test_ensure_valid_decommission_pool_index_rejects_out_of_range_index() {
29501        let err = ensure_valid_decommission_pool_index(2, 2).expect_err("out-of-range index should fail");
29502        assert!(err.to_string().contains("invalid decommission pool index 2 for 2 pools"));
29503    }
29504
29505    #[test]
29506    fn test_ensure_valid_decommission_pool_index_rejects_when_pool_count_zero() {
29507        let err = ensure_valid_decommission_pool_index(0, 0).expect_err("empty pool list should reject all indices");
29508        assert!(err.to_string().contains("invalid decommission pool index 0 for 0 pools"));
29509    }
29510
29511    #[test]
29512    fn test_classify_decommission_terminal_state_completed_when_no_failures() {
29513        assert_eq!(classify_decommission_terminal_state(false), DecommissionTerminalState::Completed);
29514    }
29515
29516    #[test]
29517    fn test_classify_decommission_terminal_state_failed_when_failures_present() {
29518        assert_eq!(classify_decommission_terminal_state(true), DecommissionTerminalState::Failed);
29519    }
29520
29521    #[test]
29522    fn test_should_preserve_decommission_canceled_state_when_meta_canceled() {
29523        assert!(should_preserve_decommission_canceled_state(true, false));
29524    }
29525
29526    #[test]
29527    fn test_should_preserve_decommission_canceled_state_when_signal_canceled() {
29528        assert!(!should_preserve_decommission_canceled_state(false, true));
29529    }
29530
29531    #[test]
29532    fn test_should_preserve_decommission_canceled_state_when_not_canceled() {
29533        assert!(!should_preserve_decommission_canceled_state(false, false));
29534    }
29535
29536    #[test]
29537    fn test_should_continue_decommission_queue_requires_clean_completion() {
29538        let meta = PoolMeta {
29539            pools: vec![
29540                PoolStatus {
29541                    id: 0,
29542                    cmd_line: "pool-0".to_string(),
29543                    last_update: OffsetDateTime::UNIX_EPOCH,
29544                    decommission: Some(PoolDecommissionInfo {
29545                        complete: true,
29546                        ..Default::default()
29547                    }),
29548                },
29549                PoolStatus {
29550                    id: 1,
29551                    cmd_line: "pool-1".to_string(),
29552                    last_update: OffsetDateTime::UNIX_EPOCH,
29553                    decommission: Some(PoolDecommissionInfo::default()),
29554                },
29555                PoolStatus {
29556                    id: 2,
29557                    cmd_line: "pool-2".to_string(),
29558                    last_update: OffsetDateTime::UNIX_EPOCH,
29559                    decommission: Some(PoolDecommissionInfo {
29560                        failed: true,
29561                        ..Default::default()
29562                    }),
29563                },
29564                PoolStatus {
29565                    id: 3,
29566                    cmd_line: "pool-3".to_string(),
29567                    last_update: OffsetDateTime::UNIX_EPOCH,
29568                    decommission: Some(PoolDecommissionInfo {
29569                        canceled: true,
29570                        ..Default::default()
29571                    }),
29572                },
29573                PoolStatus {
29574                    id: 4,
29575                    cmd_line: "pool-4".to_string(),
29576                    last_update: OffsetDateTime::UNIX_EPOCH,
29577                    decommission: None,
29578                },
29579            ],
29580            ..Default::default()
29581        };
29582
29583        assert!(should_continue_decommission_queue(&meta, 0));
29584        assert!(!should_continue_decommission_queue(&meta, 1));
29585        assert!(!should_continue_decommission_queue(&meta, 2));
29586        assert!(!should_continue_decommission_queue(&meta, 3));
29587        assert!(!should_continue_decommission_queue(&meta, 4));
29588        assert!(!should_continue_decommission_queue(&meta, 5));
29589    }
29590
29591    #[test]
29592    fn test_decommission_cancel_signal_result_returns_err_when_canceled() {
29593        let err = decommission_cancel_signal_result(true).expect_err("canceled signal should return operation-canceled");
29594        assert!(matches!(err, Error::OperationCanceled));
29595    }
29596
29597    #[test]
29598    fn test_decommission_cancel_signal_result_returns_ok_when_not_canceled() {
29599        assert!(decommission_cancel_signal_result(false).is_ok());
29600    }
29601
29602    #[test]
29603    fn test_is_decommission_cancel_requested_accepts_signal_or_metadata() {
29604        let pool = PoolStatus {
29605            id: 0,
29606            cmd_line: "pool-0".to_string(),
29607            last_update: OffsetDateTime::UNIX_EPOCH,
29608            decommission: Some(PoolDecommissionInfo {
29609                canceled: true,
29610                ..Default::default()
29611            }),
29612        };
29613
29614        assert!(is_decommission_cancel_requested(false, Some(&pool)));
29615        assert!(is_decommission_cancel_requested(true, None));
29616    }
29617
29618    #[test]
29619    fn test_is_decommission_cancel_requested_rejects_active_without_signal() {
29620        let pool = PoolStatus {
29621            id: 0,
29622            cmd_line: "pool-0".to_string(),
29623            last_update: OffsetDateTime::UNIX_EPOCH,
29624            decommission: Some(PoolDecommissionInfo::default()),
29625        };
29626
29627        assert!(!is_decommission_cancel_requested(false, Some(&pool)));
29628        assert!(!is_decommission_cancel_requested(false, None));
29629    }
29630
29631    #[test]
29632    fn test_skip_canceled_decommission_routine_only_for_terminal_canceled_state() {
29633        let canceled = PoolStatus {
29634            id: 0,
29635            cmd_line: "pool-0".to_string(),
29636            last_update: OffsetDateTime::UNIX_EPOCH,
29637            decommission: Some(PoolDecommissionInfo {
29638                canceled: true,
29639                ..Default::default()
29640            }),
29641        };
29642        let active = PoolStatus {
29643            id: 1,
29644            cmd_line: "pool-1".to_string(),
29645            last_update: OffsetDateTime::UNIX_EPOCH,
29646            decommission: Some(PoolDecommissionInfo::default()),
29647        };
29648
29649        assert!(should_skip_canceled_decommission_routine(true, Some(&canceled)));
29650        assert!(!should_skip_canceled_decommission_routine(false, Some(&canceled)));
29651        assert!(!should_skip_canceled_decommission_routine(true, Some(&active)));
29652        assert!(!should_skip_canceled_decommission_routine(true, None));
29653    }
29654
29655    #[test]
29656    fn test_ensure_decommission_cancel_allowed_rejects_missing_pool() {
29657        let err = ensure_decommission_cancel_allowed(false, false, false).expect_err("missing pool should be invalid");
29658        assert!(
29659            err.to_string()
29660                .contains("failed to cancel decommission: target pool was not found")
29661        );
29662    }
29663
29664    #[test]
29665    fn test_should_reject_decommission_cancel_as_terminal_true_when_completed() {
29666        assert!(should_reject_decommission_cancel_as_terminal(true, false));
29667    }
29668
29669    #[test]
29670    fn test_should_reject_decommission_cancel_as_terminal_true_when_failed() {
29671        assert!(should_reject_decommission_cancel_as_terminal(false, true));
29672    }
29673
29674    #[test]
29675    fn test_should_reject_decommission_cancel_as_terminal_false_when_active_or_canceled() {
29676        assert!(!should_reject_decommission_cancel_as_terminal(false, false));
29677    }
29678
29679    #[test]
29680    fn test_should_retry_decommission_cancel_reload_when_changed_or_already_canceled() {
29681        assert!(should_retry_decommission_cancel_reload(true, false));
29682        assert!(should_retry_decommission_cancel_reload(false, true));
29683        assert!(!should_retry_decommission_cancel_reload(false, false));
29684    }
29685
29686    #[test]
29687    fn test_ensure_decommission_cancel_allowed_rejects_not_started() {
29688        let err =
29689            ensure_decommission_cancel_allowed(true, false, false).expect_err("not-started decommission should be rejected");
29690        assert!(matches!(err, Error::DecommissionNotStarted));
29691    }
29692
29693    #[test]
29694    fn test_ensure_decommission_cancel_allowed_rejects_terminal() {
29695        let err = ensure_decommission_cancel_allowed(true, true, true).expect_err("terminal decommission should be rejected");
29696        assert!(matches!(err, Error::DecommissionNotStarted));
29697    }
29698
29699    #[test]
29700    fn test_ensure_decommission_cancel_allowed_allows_active() {
29701        assert!(ensure_decommission_cancel_allowed(true, true, false).is_ok());
29702    }
29703
29704    #[test]
29705    fn test_ensure_decommission_clear_allowed_allows_failed_or_canceled() {
29706        assert!(ensure_decommission_clear_allowed(true, true, false, true, false, 0).is_ok());
29707        assert!(ensure_decommission_clear_allowed(true, true, false, false, true, 0).is_ok());
29708    }
29709
29710    #[test]
29711    fn test_ensure_decommission_clear_allowed_rejects_active_or_completed() {
29712        let active = ensure_decommission_clear_allowed(true, true, false, false, false, 0)
29713            .expect_err("active decommission should not be clearable");
29714        assert!(matches!(active, Error::DecommissionAlreadyRunning));
29715
29716        let complete = ensure_decommission_clear_allowed(true, true, true, false, false, 0)
29717            .expect_err("completed decommission should not be clearable");
29718        assert!(matches!(complete, Error::DecommissionNotStarted));
29719    }
29720
29721    #[test]
29722    fn test_ensure_decommission_clear_allowed_rejects_unresolved_entries() {
29723        let err = ensure_decommission_clear_allowed(true, true, false, true, false, 1)
29724            .expect_err("unresolved entries must survive until a retry reconciles them");
29725
29726        assert!(err.to_string().contains("must be reconciled by retrying decommission"));
29727    }
29728
29729    #[test]
29730    fn test_pool_meta_clear_decommission_restores_failed_or_canceled_pool() {
29731        for decommission in [
29732            PoolDecommissionInfo {
29733                failed: true,
29734                ..Default::default()
29735            },
29736            PoolDecommissionInfo {
29737                canceled: true,
29738                ..Default::default()
29739            },
29740        ] {
29741            let mut meta = PoolMeta {
29742                pools: vec![PoolStatus {
29743                    id: 0,
29744                    cmd_line: "pool-0".to_string(),
29745                    last_update: OffsetDateTime::UNIX_EPOCH,
29746                    decommission: Some(decommission),
29747                }],
29748                ..Default::default()
29749            };
29750
29751            assert!(meta.is_suspended(0));
29752            assert!(meta.clear_decommission(0).expect("terminal decommission should clear"));
29753            assert!(
29754                meta.pools[0]
29755                    .decommission
29756                    .as_ref()
29757                    .is_some_and(|info| !info.has_decommission_state())
29758            );
29759            assert!(!meta.is_suspended(0));
29760        }
29761    }
29762
29763    #[test]
29764    fn test_pool_meta_clear_decommission_preserves_unresolved_entries_for_retry() {
29765        let generation = OffsetDateTime::UNIX_EPOCH;
29766        let unresolved_entry = DecommissionUnresolvedEntry {
29767            bucket: "bucket-a".to_string(),
29768            object: "object-a".to_string(),
29769            pool_index: 0,
29770            set_index: 0,
29771            source_generation: generation,
29772            candidate_count: 1,
29773            disk_error_count: 1,
29774            observed_at: generation,
29775            reason: "metadata_resolution_failed".to_string(),
29776        };
29777        let mut meta = PoolMeta {
29778            pools: vec![PoolStatus {
29779                id: 0,
29780                cmd_line: "pool-0".to_string(),
29781                last_update: generation,
29782                decommission: Some(PoolDecommissionInfo {
29783                    failed: true,
29784                    unresolved_entries: vec![unresolved_entry.clone()],
29785                    ..Default::default()
29786                }),
29787            }],
29788            ..Default::default()
29789        };
29790
29791        let err = meta
29792            .clear_decommission(0)
29793            .expect_err("clear must not discard the unresolved-entry recovery ledger");
29794
29795        assert!(err.to_string().contains("must be reconciled by retrying decommission"));
29796        assert_eq!(
29797            meta.pools[0]
29798                .decommission
29799                .as_ref()
29800                .expect("failed state should remain retryable")
29801                .unresolved_entries,
29802            vec![unresolved_entry]
29803        );
29804    }
29805
29806    #[test]
29807    fn test_pool_meta_clear_decommission_rejects_active_or_completed_pool() {
29808        for decommission in [
29809            PoolDecommissionInfo::default(),
29810            PoolDecommissionInfo {
29811                complete: true,
29812                ..Default::default()
29813            },
29814        ] {
29815            let mut meta = PoolMeta {
29816                pools: vec![PoolStatus {
29817                    id: 0,
29818                    cmd_line: "pool-0".to_string(),
29819                    last_update: OffsetDateTime::UNIX_EPOCH,
29820                    decommission: Some(decommission),
29821                }],
29822                ..Default::default()
29823            };
29824
29825            assert!(meta.clear_decommission(0).is_err());
29826            assert!(meta.pools[0].decommission.is_some());
29827        }
29828    }
29829
29830    #[test]
29831    fn test_contextualized_decommission_terminal_operation_supported_rejects_single_pool() {
29832        let err = ensure_decommission_terminal_operation_supported(true, "complete decommission")
29833            .expect_err("single-pool decommission terminal operations should be rejected");
29834        assert!(
29835            err.to_string()
29836                .contains("failed to complete decommission: single pool deployments do not support decommission")
29837        );
29838    }
29839
29840    #[test]
29841    fn test_contextualized_decommission_terminal_operation_supported_allows_multi_pool() {
29842        assert!(ensure_decommission_terminal_operation_supported(false, "mark decommission failed").is_ok());
29843    }
29844
29845    #[test]
29846    fn test_contextualized_decommission_start_request_rejects_empty_indices() {
29847        let err = validate_start_decommission_request(&[], false).expect_err("empty decommission target list should be rejected");
29848        assert!(
29849            err.to_string()
29850                .contains("failed to start decommission: no target pools were provided")
29851        );
29852    }
29853
29854    #[test]
29855    fn test_contextualized_decommission_start_request_rejects_single_pool() {
29856        let err = validate_start_decommission_request(&[0], true)
29857            .expect_err("single-pool deployments should reject decommission start");
29858        assert!(
29859            err.to_string()
29860                .contains("failed to start decommission: single pool deployments do not support decommission")
29861        );
29862    }
29863
29864    #[test]
29865    fn test_contextualized_decommission_start_request_allows_multiple_target_pools() {
29866        assert!(validate_start_decommission_request(&[0, 1], false).is_ok());
29867    }
29868
29869    #[test]
29870    fn test_contextualized_decommission_start_request_allows_one_target_pool() {
29871        assert!(validate_start_decommission_request(&[0], false).is_ok());
29872    }
29873
29874    #[test]
29875    fn test_decommission_retry_preserves_and_rebinds_unresolved_entries() {
29876        let previous_generation = OffsetDateTime::UNIX_EPOCH;
29877        let unresolved_entry = DecommissionUnresolvedEntry {
29878            bucket: "bucket-a".to_string(),
29879            object: "object-a".to_string(),
29880            pool_index: 0,
29881            set_index: 0,
29882            source_generation: previous_generation,
29883            candidate_count: 1,
29884            disk_error_count: 1,
29885            observed_at: previous_generation,
29886            reason: "metadata_resolution_failed".to_string(),
29887        };
29888        let mut meta = PoolMeta {
29889            pools: vec![PoolStatus {
29890                id: 0,
29891                cmd_line: "pool-0".to_string(),
29892                last_update: previous_generation,
29893                decommission: Some(PoolDecommissionInfo {
29894                    failed: true,
29895                    decommissioned_buckets: vec!["bucket-a".to_string()],
29896                    items_decommissioned: 7,
29897                    bytes_done: 1024,
29898                    unresolved_entries: vec![unresolved_entry],
29899                    ..Default::default()
29900                }),
29901            }],
29902            ..Default::default()
29903        };
29904
29905        meta.decommission(
29906            0,
29907            PoolSpaceInfo {
29908                total: 200,
29909                free: 50,
29910                used: 150,
29911            },
29912        )
29913        .expect("failed decommission with unresolved entries should retry atomically");
29914
29915        let info = meta.pools[0]
29916            .decommission
29917            .as_ref()
29918            .expect("retried decommission metadata should exist");
29919        let next_generation = info.start_time.expect("retry should assign a new generation");
29920        assert!(next_generation > previous_generation);
29921        assert_eq!(info.decommissioned_buckets, vec!["bucket-a".to_string()]);
29922        assert_eq!(info.items_decommissioned, 7);
29923        assert_eq!(info.bytes_done, 1024);
29924        assert_eq!(info.unresolved_entries.len(), 1);
29925        assert_eq!(info.unresolved_entries[0].source_generation, next_generation);
29926    }
29927
29928    #[test]
29929    fn test_queued_decommission_retry_rebinds_unresolved_entries_when_promoted() {
29930        let previous_generation = OffsetDateTime::UNIX_EPOCH;
29931        let mut meta = PoolMeta {
29932            pools: vec![PoolStatus {
29933                id: 0,
29934                cmd_line: "pool-0".to_string(),
29935                last_update: previous_generation,
29936                decommission: Some(PoolDecommissionInfo {
29937                    canceled: true,
29938                    unresolved_entries: vec![DecommissionUnresolvedEntry {
29939                        bucket: "bucket-a".to_string(),
29940                        object: "object-a".to_string(),
29941                        pool_index: 0,
29942                        set_index: 0,
29943                        source_generation: previous_generation,
29944                        candidate_count: 1,
29945                        disk_error_count: 1,
29946                        observed_at: previous_generation,
29947                        reason: "metadata_resolution_failed".to_string(),
29948                    }],
29949                    ..Default::default()
29950                }),
29951            }],
29952            ..Default::default()
29953        };
29954
29955        meta.queue_decommission(
29956            0,
29957            PoolSpaceInfo {
29958                total: 100,
29959                free: 25,
29960                used: 75,
29961            },
29962        )
29963        .expect("canceled decommission with unresolved entries should queue an atomic retry");
29964
29965        assert!(meta.is_suspended(0));
29966        assert!(meta.promote_queued_decommission(0));
29967        let promoted = meta.pools[0]
29968            .decommission
29969            .as_ref()
29970            .expect("promoted decommission metadata should exist");
29971        let generation = promoted.start_time.expect("promotion should assign a generation");
29972        assert_eq!(promoted.unresolved_entries.len(), 1);
29973        assert_eq!(promoted.unresolved_entries[0].source_generation, generation);
29974    }
29975
29976    #[test]
29977    fn test_queued_decommission_promotion_advances_generation_after_clock_rollback() {
29978        let queued_at = OffsetDateTime::from_unix_timestamp(1_260).expect("fixed timestamp should be valid");
29979        let earlier_tick = queued_at - Duration::nanoseconds(10);
29980        let rebalance_floor = queued_at + Duration::nanoseconds(5);
29981        let rebalance = RebalanceMeta {
29982            stopped_at: Some(rebalance_floor),
29983            id: "completed-rebalance".to_string(),
29984            ..Default::default()
29985        };
29986        let mut meta = PoolMeta {
29987            pools: vec![PoolStatus {
29988                id: 0,
29989                cmd_line: "pool-0".to_string(),
29990                last_update: queued_at,
29991                decommission: Some(PoolDecommissionInfo {
29992                    queued: true,
29993                    ..Default::default()
29994                }),
29995            }],
29996            ..Default::default()
29997        };
29998
29999        assert!(meta.promote_queued_decommission_at_for_test(0, earlier_tick, Some(&rebalance)));
30000        let expected_generation = rebalance_floor + Duration::nanoseconds(1);
30001        let promoted = meta.pools[0]
30002            .decommission
30003            .as_ref()
30004            .expect("promoted decommission metadata should exist");
30005        assert_eq!(meta.pools[0].last_update, expected_generation);
30006        assert_eq!(promoted.start_time, Some(expected_generation));
30007        assert!(!promoted.queued);
30008    }
30009
30010    #[test]
30011    fn test_pool_meta_queued_decommission_is_suspended_to_preserve_reserved_capacity() {
30012        let mut meta = PoolMeta {
30013            pools: vec![PoolStatus {
30014                id: 0,
30015                cmd_line: "pool-0".to_string(),
30016                last_update: OffsetDateTime::UNIX_EPOCH,
30017                decommission: None,
30018            }],
30019            ..Default::default()
30020        };
30021
30022        meta.queue_decommission(
30023            0,
30024            PoolSpaceInfo {
30025                total: 100,
30026                free: 10,
30027                used: 90,
30028            },
30029        )
30030        .expect("queued decommission should be stored");
30031
30032        assert!(meta.is_suspended(0));
30033        assert!(meta.promote_queued_decommission(0));
30034        assert!(meta.is_suspended(0));
30035    }
30036
30037    #[test]
30038    fn test_pool_meta_promoted_queued_decommission_can_be_canceled() {
30039        let mut meta = PoolMeta {
30040            pools: vec![PoolStatus {
30041                id: 0,
30042                cmd_line: "pool-0".to_string(),
30043                last_update: OffsetDateTime::UNIX_EPOCH,
30044                decommission: None,
30045            }],
30046            ..Default::default()
30047        };
30048
30049        meta.queue_decommission(
30050            0,
30051            PoolSpaceInfo {
30052                total: 100,
30053                free: 10,
30054                used: 90,
30055            },
30056        )
30057        .expect("queued decommission should be stored");
30058
30059        assert!(pool_meta_has_active_decommission(&meta));
30060        assert!(meta.promote_queued_decommission(0));
30061        assert!(meta.decommission_cancel(0));
30062
30063        let info = meta.pools[0]
30064            .decommission
30065            .as_ref()
30066            .expect("canceled decommission state should be kept for clear");
30067        assert!(info.canceled);
30068        assert!(!info.queued);
30069        assert!(!info.failed);
30070        assert!(!info.complete);
30071        assert!(!pool_meta_has_active_decommission(&meta));
30072    }
30073
30074    #[test]
30075    fn test_pool_meta_failed_decommission_requires_clear_before_restart() {
30076        let mut meta = PoolMeta {
30077            pools: vec![PoolStatus {
30078                id: 0,
30079                cmd_line: "pool-0".to_string(),
30080                last_update: OffsetDateTime::UNIX_EPOCH,
30081                decommission: Some(PoolDecommissionInfo {
30082                    failed: true,
30083                    decommissioned_buckets: vec!["bucket-done".to_string()],
30084                    queued_buckets: vec!["bucket-pending".to_string()],
30085                    bucket: "bucket-pending".to_string(),
30086                    prefix: "prefix".to_string(),
30087                    object: "object.txt".to_string(),
30088                    items_decommissioned: 7,
30089                    items_decommission_failed: 3,
30090                    bytes_done: 1024,
30091                    bytes_failed: 256,
30092                    progress_save_item_baseline: 10,
30093                    ..Default::default()
30094                }),
30095            }],
30096            ..Default::default()
30097        };
30098
30099        let err = meta
30100            .decommission(
30101                0,
30102                PoolSpaceInfo {
30103                    total: 200,
30104                    free: 50,
30105                    used: 150,
30106                },
30107            )
30108            .expect_err("failed decommission should be blocked until cleared");
30109        assert!(err.to_string().contains("target pool decommission is blocked"));
30110        let blocked = meta.pools[0]
30111            .decommission
30112            .as_ref()
30113            .expect("blocked metadata should remain until clear");
30114        assert!(blocked.failed);
30115        assert_eq!(blocked.decommissioned_buckets, vec!["bucket-done".to_string()]);
30116        assert_eq!(blocked.items_decommissioned, 7);
30117        assert_eq!(blocked.bytes_done, 1024);
30118
30119        assert!(meta.clear_decommission(0).expect("failed decommission should clear"));
30120        assert!(
30121            meta.pools[0]
30122                .decommission
30123                .as_ref()
30124                .is_some_and(|info| !info.has_decommission_state())
30125        );
30126
30127        meta.decommission(
30128            0,
30129            PoolSpaceInfo {
30130                total: 200,
30131                free: 50,
30132                used: 150,
30133            },
30134        )
30135        .expect("cleared decommission should be restartable");
30136        meta.queue_buckets(
30137            0,
30138            vec![
30139                DecomBucketInfo {
30140                    name: "bucket-done".to_string(),
30141                    prefix: String::new(),
30142                },
30143                DecomBucketInfo {
30144                    name: "bucket-pending".to_string(),
30145                    prefix: String::new(),
30146                },
30147            ],
30148        );
30149
30150        let info = meta.pools[0]
30151            .decommission
30152            .as_ref()
30153            .expect("decommission info should be rebuilt");
30154        assert!(!info.failed);
30155        assert!(!info.canceled);
30156        assert!(!info.complete);
30157        assert!(info.decommissioned_buckets.is_empty());
30158        assert_eq!(info.queued_buckets, vec!["bucket-done".to_string(), "bucket-pending".to_string()]);
30159        assert_eq!(info.items_decommissioned, 0);
30160        assert_eq!(info.items_decommission_failed, 0);
30161        assert_eq!(info.bytes_done, 0);
30162        assert_eq!(info.bytes_failed, 0);
30163        assert_eq!(info.items_since_last_progress_save(), 0);
30164        assert_eq!(info.start_size, 50);
30165        assert_eq!(info.total_size, 200);
30166        assert_eq!(info.current_size, 50);
30167        assert_eq!(info.bucket, "bucket-pending");
30168        assert!(info.prefix.is_empty());
30169        assert!(info.object.is_empty());
30170        assert!(info.start_time.is_some());
30171    }
30172
30173    #[test]
30174    fn test_pool_meta_canceled_queued_decommission_requires_clear_before_restart() {
30175        let mut meta = PoolMeta {
30176            pools: vec![PoolStatus {
30177                id: 0,
30178                cmd_line: "pool-0".to_string(),
30179                last_update: OffsetDateTime::UNIX_EPOCH,
30180                decommission: Some(PoolDecommissionInfo {
30181                    canceled: true,
30182                    decommissioned_buckets: vec!["bucket-done".to_string()],
30183                    items_decommissioned: 5,
30184                    bytes_done: 512,
30185                    ..Default::default()
30186                }),
30187            }],
30188            ..Default::default()
30189        };
30190
30191        let err = meta
30192            .queue_decommission(
30193                0,
30194                PoolSpaceInfo {
30195                    total: 100,
30196                    free: 25,
30197                    used: 75,
30198                },
30199            )
30200            .expect_err("canceled queued decommission should be blocked until cleared");
30201        assert!(err.to_string().contains("target pool decommission is blocked"));
30202        let blocked = meta.pools[0]
30203            .decommission
30204            .as_ref()
30205            .expect("blocked metadata should remain until clear");
30206        assert!(blocked.canceled);
30207        assert_eq!(blocked.decommissioned_buckets, vec!["bucket-done".to_string()]);
30208        assert_eq!(blocked.items_decommissioned, 5);
30209        assert_eq!(blocked.bytes_done, 512);
30210
30211        assert!(meta.clear_decommission(0).expect("canceled decommission should clear"));
30212        assert!(
30213            meta.pools[0]
30214                .decommission
30215                .as_ref()
30216                .is_some_and(|info| !info.has_decommission_state())
30217        );
30218
30219        meta.queue_decommission(
30220            0,
30221            PoolSpaceInfo {
30222                total: 100,
30223                free: 25,
30224                used: 75,
30225            },
30226        )
30227        .expect("cleared queued decommission should be restartable");
30228
30229        let info = meta.pools[0]
30230            .decommission
30231            .as_ref()
30232            .expect("decommission info should be rebuilt");
30233        assert!(info.queued);
30234        assert!(info.start_time.is_none());
30235        assert!(info.decommissioned_buckets.is_empty());
30236        assert_eq!(info.items_decommissioned, 0);
30237        assert_eq!(info.bytes_done, 0);
30238    }
30239
30240    #[test]
30241    fn test_contextualized_decommission_listing_disks_available_rejects_empty_set() {
30242        let err = ensure_decommission_listing_disks_available(false, "bucket-a")
30243            .expect_err("missing online disks should be reported with bucket context");
30244        assert!(
30245            err.to_string()
30246                .contains("failed to list objects to decommission for bucket bucket-a: no disks available")
30247        );
30248    }
30249
30250    #[test]
30251    fn test_contextualized_decommission_listing_disks_available_allows_online_disks() {
30252        assert!(ensure_decommission_listing_disks_available(true, "bucket-a").is_ok());
30253    }
30254
30255    #[test]
30256    fn test_require_decommission_store_returns_value_when_present() {
30257        let store = require_decommission_store(Some(7_u8), "start decommission").expect("present store should be returned");
30258        assert_eq!(store, 7);
30259    }
30260
30261    #[test]
30262    fn test_require_decommission_store_returns_error_when_missing() {
30263        let err = require_decommission_store::<u8>(None, "start decommission").expect_err("missing store should return error");
30264        assert!(
30265            err.to_string()
30266                .contains("failed to start decommission: store not initialized")
30267        );
30268    }
30269
30270    #[test]
30271    fn test_bind_decommission_cancelers_binds_existing_slots_only() {
30272        let parent = CancellationToken::new();
30273        let mut cancelers = vec![None, None];
30274
30275        let bound = bind_decommission_cancelers(&[0, 3, 1], &parent, cancelers.as_mut_slice());
30276
30277        assert_eq!(bound.len(), 2);
30278        assert_eq!(bound[0].0, 0);
30279        assert_eq!(bound[1].0, 1);
30280        assert!(cancelers[0].is_some());
30281        assert!(cancelers[1].is_some());
30282    }
30283
30284    #[test]
30285    fn test_bind_decommission_cancelers_child_tokens_follow_parent_cancel() {
30286        let parent = CancellationToken::new();
30287        let mut cancelers = vec![None];
30288
30289        let bound = bind_decommission_cancelers(&[0], &parent, cancelers.as_mut_slice());
30290        assert_eq!(bound.len(), 1);
30291        assert!(!bound[0].1.is_cancelled());
30292
30293        parent.cancel();
30294        assert!(bound[0].1.is_cancelled());
30295    }
30296
30297    #[test]
30298    fn test_bind_decommission_cancelers_replaces_existing_slot() {
30299        let parent = CancellationToken::new();
30300        let existing = DecommissionCanceler::new(CancellationToken::new());
30301        let mut cancelers = vec![Some(existing.clone())];
30302
30303        let bound = bind_decommission_cancelers(&[0], &parent, cancelers.as_mut_slice());
30304
30305        assert_eq!(bound.len(), 1);
30306        assert_eq!(bound[0].0, 0);
30307        assert!(existing.is_cancelled());
30308        let replacement = cancelers[0].as_ref().expect("replacement token should be stored");
30309        assert!(!replacement.is_cancelled());
30310        parent.cancel();
30311        assert!(replacement.is_cancelled());
30312    }
30313
30314    #[test]
30315    fn test_bind_missing_decommission_cancelers_stops_at_existing_slot() {
30316        let parent = CancellationToken::new();
30317        let existing = DecommissionCanceler::new(CancellationToken::new());
30318        let mut cancelers = vec![None, Some(existing.clone()), None];
30319
30320        let bound = bind_missing_decommission_cancelers(&[0, 1, 2], &parent, cancelers.as_mut_slice());
30321
30322        assert_eq!(bound.len(), 1);
30323        assert_eq!(bound[0].0, 0);
30324        assert!(cancelers[0].is_some());
30325        assert!(cancelers[1].is_some());
30326        assert!(cancelers[2].is_none());
30327        assert!(!existing.is_cancelled());
30328    }
30329
30330    #[test]
30331    fn test_serialized_decommission_double_start_preserves_first_operation() {
30332        let mut pool_meta = PoolMeta {
30333            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
30334            ..Default::default()
30335        };
30336        let first_parent = CancellationToken::new();
30337        let second_parent = CancellationToken::new();
30338        let mut cancelers = vec![None, None];
30339
30340        let first = reserve_decommission_start_cancelers(&pool_meta, &[0], &[0], &first_parent, cancelers.as_mut_slice())
30341            .expect("first start should reserve its worker");
30342        pool_meta
30343            .decommission(
30344                0,
30345                PoolSpaceInfo {
30346                    total: 100,
30347                    free: 40,
30348                    used: 60,
30349                },
30350            )
30351            .expect("first start should install active metadata");
30352
30353        let second = reserve_decommission_start_cancelers(&pool_meta, &[0], &[0], &second_parent, cancelers.as_mut_slice());
30354
30355        assert!(matches!(second, Err(Error::DecommissionAlreadyRunning)));
30356        let current = cancelers[0].as_ref().expect("first operation should retain the slot");
30357        assert!(current.owns_same_operation(first[0].1.canceler()));
30358        assert!(current.is_active());
30359        assert!(!first_parent.is_cancelled());
30360    }
30361
30362    #[test]
30363    fn test_local_decommission_queue_prefix_stops_at_remote_leader() {
30364        let endpoints = EndpointServerPools::from(vec![
30365            decommission_test_pool_endpoint(0, true),
30366            decommission_test_pool_endpoint(1, true),
30367            decommission_test_pool_endpoint(2, false),
30368            decommission_test_pool_endpoint(3, true),
30369        ]);
30370
30371        let local = local_decommission_queue_prefix(&endpoints, &[0, 1, 2, 3]).expect("prefix should resolve");
30372
30373        assert_eq!(local, vec![0, 1]);
30374    }
30375
30376    #[test]
30377    fn test_local_decommission_queue_prefix_empty_when_first_leader_remote() {
30378        let endpoints = EndpointServerPools::from(vec![
30379            decommission_test_pool_endpoint(0, false),
30380            decommission_test_pool_endpoint(1, true),
30381        ]);
30382
30383        let local = local_decommission_queue_prefix(&endpoints, &[0, 1]).expect("prefix should resolve");
30384
30385        assert!(local.is_empty());
30386    }
30387
30388    #[test]
30389    fn test_decommission_start_local_leader_allows_remote_queued_pool() {
30390        let endpoints = EndpointServerPools::from(vec![
30391            decommission_test_pool_endpoint(0, true),
30392            decommission_test_pool_endpoint(1, false),
30393        ]);
30394
30395        assert!(ensure_decommission_start_local_leader(&endpoints, &[0, 1]).is_ok());
30396    }
30397
30398    #[test]
30399    fn test_decommission_start_local_leader_rejects_remote_active_pool() {
30400        let endpoints = EndpointServerPools::from(vec![decommission_test_pool_endpoint(0, false)]);
30401
30402        let err = ensure_decommission_start_local_leader(&endpoints, &[0]).expect_err("remote active pool should be rejected");
30403
30404        assert!(
30405            err.to_string()
30406                .contains("decommission for pool 0 must run on the pool first endpoint")
30407        );
30408    }
30409
30410    #[test]
30411    fn test_missing_decommission_worker_prefix_stops_at_active_worker() {
30412        let cancelers = vec![None, Some(DecommissionCanceler::new(CancellationToken::new())), None];
30413
30414        let missing = missing_decommission_worker_prefix(&[0, 1, 2], cancelers.as_slice());
30415
30416        assert_eq!(missing, vec![0]);
30417    }
30418
30419    #[test]
30420    fn test_resumable_decommission_queue_indices_skip_terminal_predecessors() {
30421        let meta = PoolMeta {
30422            pools: vec![
30423                decommission_test_pool_status(
30424                    0,
30425                    Some(PoolDecommissionInfo {
30426                        complete: true,
30427                        ..Default::default()
30428                    }),
30429                ),
30430                decommission_test_pool_status(
30431                    1,
30432                    Some(PoolDecommissionInfo {
30433                        canceled: true,
30434                        ..Default::default()
30435                    }),
30436                ),
30437                decommission_test_pool_status(
30438                    2,
30439                    Some(PoolDecommissionInfo {
30440                        failed: true,
30441                        ..Default::default()
30442                    }),
30443                ),
30444                decommission_test_pool_status(
30445                    3,
30446                    Some(PoolDecommissionInfo {
30447                        queued: true,
30448                        ..Default::default()
30449                    }),
30450                ),
30451                decommission_test_pool_status(4, None),
30452            ],
30453            ..Default::default()
30454        };
30455
30456        assert_eq!(resumable_decommission_queue_indices(&meta), vec![3]);
30457    }
30458
30459    #[test]
30460    fn test_resumable_decommission_queue_indices_preserve_active_predecessor_order() {
30461        let meta = PoolMeta {
30462            pools: vec![
30463                decommission_test_pool_status(
30464                    0,
30465                    Some(PoolDecommissionInfo {
30466                        complete: true,
30467                        ..Default::default()
30468                    }),
30469                ),
30470                decommission_test_pool_status(
30471                    1,
30472                    Some(PoolDecommissionInfo {
30473                        start_time: Some(OffsetDateTime::UNIX_EPOCH),
30474                        ..Default::default()
30475                    }),
30476                ),
30477                decommission_test_pool_status(
30478                    2,
30479                    Some(PoolDecommissionInfo {
30480                        queued: true,
30481                        ..Default::default()
30482                    }),
30483                ),
30484            ],
30485            ..Default::default()
30486        };
30487
30488        assert_eq!(resumable_decommission_queue_indices(&meta), vec![1, 2]);
30489    }
30490
30491    #[tokio::test]
30492    async fn test_runtime_recovery_reserves_the_startup_resumable_queue() {
30493        let meta = PoolMeta {
30494            version: super::POOL_META_VERSION,
30495            pools: vec![
30496                decommission_test_pool_status(
30497                    0,
30498                    Some(PoolDecommissionInfo {
30499                        failed: true,
30500                        ..Default::default()
30501                    }),
30502                ),
30503                decommission_test_pool_status(
30504                    1,
30505                    Some(PoolDecommissionInfo {
30506                        canceled: true,
30507                        ..Default::default()
30508                    }),
30509                ),
30510                decommission_test_pool_status(
30511                    2,
30512                    Some(PoolDecommissionInfo {
30513                        start_time: Some(OffsetDateTime::UNIX_EPOCH),
30514                        ..Default::default()
30515                    }),
30516                ),
30517                decommission_test_pool_status(
30518                    3,
30519                    Some(PoolDecommissionInfo {
30520                        queued: true,
30521                        ..Default::default()
30522                    }),
30523                ),
30524            ],
30525            ..Default::default()
30526        };
30527        let startup_ids = meta
30528            .return_resumable_pools()
30529            .into_iter()
30530            .map(|pool| pool.id)
30531            .collect::<Vec<_>>();
30532        let store = decommission_worker_test_store(meta, vec![None, None, None, None]);
30533        let endpoints = EndpointServerPools::from(vec![
30534            decommission_test_pool_endpoint(0, true),
30535            decommission_test_pool_endpoint(1, true),
30536            decommission_test_pool_endpoint(2, true),
30537            decommission_test_pool_endpoint(3, true),
30538        ]);
30539        let reserved = store
30540            .reserve_missing_local_decommission_routines(&CancellationToken::new(), &endpoints)
30541            .await
30542            .expect("runtime recovery reservation should succeed");
30543        let runtime_indices = reserved.iter().map(|(idx, _)| *idx).collect::<Vec<_>>();
30544
30545        assert_eq!(runtime_indices, vec![2, 3]);
30546        assert_eq!(startup_ids, vec![2, 3]);
30547    }
30548
30549    #[tokio::test]
30550    async fn test_runtime_recovery_does_not_reserve_behind_active_predecessor() {
30551        let meta = PoolMeta {
30552            version: super::POOL_META_VERSION,
30553            pools: vec![
30554                decommission_test_pool_status(
30555                    0,
30556                    Some(PoolDecommissionInfo {
30557                        failed: true,
30558                        ..Default::default()
30559                    }),
30560                ),
30561                decommission_test_pool_status(
30562                    1,
30563                    Some(PoolDecommissionInfo {
30564                        canceled: true,
30565                        ..Default::default()
30566                    }),
30567                ),
30568                decommission_test_pool_status(
30569                    2,
30570                    Some(PoolDecommissionInfo {
30571                        start_time: Some(OffsetDateTime::UNIX_EPOCH),
30572                        ..Default::default()
30573                    }),
30574                ),
30575                decommission_test_pool_status(
30576                    3,
30577                    Some(PoolDecommissionInfo {
30578                        queued: true,
30579                        ..Default::default()
30580                    }),
30581                ),
30582            ],
30583            ..Default::default()
30584        };
30585        let active = DecommissionCanceler::new(CancellationToken::new());
30586        let store = decommission_worker_test_store(meta, vec![None, None, Some(active.clone()), None]);
30587        let endpoints = EndpointServerPools::from(vec![
30588            decommission_test_pool_endpoint(0, true),
30589            decommission_test_pool_endpoint(1, true),
30590            decommission_test_pool_endpoint(2, true),
30591            decommission_test_pool_endpoint(3, true),
30592        ]);
30593
30594        let reserved = store
30595            .reserve_missing_local_decommission_routines(&CancellationToken::new(), &endpoints)
30596            .await
30597            .expect("runtime recovery reservation should succeed");
30598
30599        assert!(reserved.is_empty());
30600        assert!(active.is_active());
30601    }
30602
30603    #[test]
30604    fn test_take_decommission_canceler_takes_and_clears_slot() {
30605        let canceler = DecommissionCanceler::new(CancellationToken::new());
30606        let mut cancelers = vec![Some(canceler)];
30607
30608        let taken = take_decommission_canceler(cancelers.as_mut_slice(), 0);
30609        assert!(taken.is_some());
30610        assert!(cancelers[0].is_none());
30611    }
30612
30613    #[test]
30614    fn test_take_decommission_canceler_returns_none_for_missing_slot() {
30615        let mut cancelers: Vec<Option<DecommissionCanceler>> = Vec::new();
30616        assert!(take_decommission_canceler(cancelers.as_mut_slice(), 0).is_none());
30617    }
30618
30619    #[test]
30620    fn test_has_active_decommission_canceler_true_when_any_slot_present() {
30621        let cancelers = vec![None, Some(DecommissionCanceler::new(CancellationToken::new()))];
30622        assert!(has_active_decommission_canceler(cancelers.as_slice()));
30623    }
30624
30625    #[test]
30626    fn test_has_active_decommission_canceler_false_when_all_empty() {
30627        let cancelers = vec![None, None];
30628        assert!(!has_active_decommission_canceler(cancelers.as_slice()));
30629    }
30630
30631    #[test]
30632    fn test_cancel_decommission_canceler_cancels_when_present() {
30633        let canceler = DecommissionCanceler::new(CancellationToken::new());
30634        let canceled = cancel_decommission_canceler(Some(canceler.clone()));
30635
30636        assert!(canceled);
30637        assert!(canceler.is_cancelled());
30638        assert!(!canceler.is_active());
30639    }
30640
30641    #[test]
30642    fn test_cancel_decommission_canceler_returns_false_when_missing() {
30643        assert!(!cancel_decommission_canceler(None));
30644    }
30645
30646    #[test]
30647    fn test_take_and_cancel_decommission_canceler_clears_slot() {
30648        let canceler = DecommissionCanceler::new(CancellationToken::new());
30649        let mut cancelers = vec![Some(canceler.clone())];
30650
30651        assert!(take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), 0));
30652        assert!(cancelers[0].is_none());
30653        assert!(canceler.is_cancelled());
30654        assert!(!canceler.is_active());
30655    }
30656
30657    #[test]
30658    fn test_take_and_cancel_decommission_canceler_missing_slot_is_false() {
30659        let mut cancelers = vec![None];
30660
30661        assert!(!take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), 0));
30662        assert!(cancelers[0].is_none());
30663    }
30664
30665    #[test]
30666    fn test_guarded_decommission_future_releases_without_first_poll() {
30667        let canceler = DecommissionCanceler::new(CancellationToken::new());
30668        let cancelers = vec![Some(canceler.clone())];
30669        let guards = guard_decommission_cancelers(vec![(0, canceler.clone())]);
30670        let unpolled = async move {
30671            let _guards = guards;
30672            std::future::pending::<()>().await;
30673        };
30674
30675        drop(unpolled);
30676
30677        assert!(canceler.is_cancelled());
30678        assert!(!has_active_decommission_canceler(cancelers.as_slice()));
30679    }
30680
30681    #[test]
30682    fn test_partial_decommission_spawn_reservation_releases_bound_slot() {
30683        let parent = CancellationToken::new();
30684        let mut cancelers = vec![None];
30685        let bound = bind_decommission_cancelers(&[0, 1], &parent, cancelers.as_mut_slice());
30686        let guards = guard_decommission_cancelers(bound);
30687
30688        let result = super::ensure_decommission_routines_scheduled(guards.len(), 2);
30689        drop(guards);
30690
30691        assert!(result.is_err());
30692        assert!(!has_active_decommission_canceler(cancelers.as_slice()));
30693    }
30694
30695    #[tokio::test]
30696    async fn test_decommission_supervisor_observes_worker_abort() {
30697        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
30698        let worker = tokio::spawn(async move {
30699            started_tx.send(()).expect("worker start should be observed");
30700            std::future::pending::<()>().await;
30701            #[allow(unreachable_code)]
30702            Ok(())
30703        });
30704
30705        started_rx.await.expect("worker start should be observed");
30706        worker.abort();
30707        let err = await_decommission_worker(3, worker)
30708            .await
30709            .expect_err("supervisor should observe aborted worker");
30710
30711        assert!(err.to_string().contains("decommission worker 3 task join error"));
30712    }
30713
30714    #[tokio::test]
30715    async fn test_decommission_supervisor_observes_worker_panic() {
30716        let worker = tokio::spawn(async move {
30717            panic!("injected decommission worker panic");
30718        });
30719
30720        let err = await_decommission_worker(4, worker)
30721            .await
30722            .expect_err("supervisor should observe panicked worker");
30723
30724        assert!(err.to_string().contains("decommission worker 4 task join error"));
30725    }
30726
30727    #[tokio::test]
30728    async fn test_decommission_worker_metadata_missing_releases_owned_slot() {
30729        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
30730        let canceler = DecommissionCanceler::new(CancellationToken::new());
30731        *store.pool_meta.write().await = PoolMeta::default();
30732        store.decommission_cancelers.write().await[0] = Some(canceler.clone());
30733
30734        let err = store
30735            .do_decommission_in_routine(canceler.clone(), 0, Arc::new(Semaphore::new(1)))
30736            .await
30737            .expect_err("missing worker metadata should fail the routine");
30738
30739        assert!(err.to_string().contains("target pool was not found"));
30740        assert!(!canceler.is_active());
30741        assert!(store.decommission_cancelers.read().await[0].is_none());
30742    }
30743
30744    #[tokio::test]
30745    async fn test_decommission_supervisor_failure_cancels_queued_successor() {
30746        let first = DecommissionCanceler::new(CancellationToken::new());
30747        let queued = DecommissionCanceler::new(CancellationToken::new());
30748        let store = decommission_worker_test_store(PoolMeta::default(), vec![Some(first.clone()), Some(queued.clone())]);
30749        let guards = guard_decommission_cancelers(vec![(0, first.clone()), (1, queued.clone())]);
30750
30751        spawn_decommission_index_cancelers(store.clone(), CancellationToken::new(), guards, Arc::new(Semaphore::new(1)))
30752            .await
30753            .expect("decommission supervisor should finish after queued cleanup");
30754
30755        assert!(!first.is_active());
30756        assert!(!queued.is_active());
30757        assert!(queued.is_cancelled());
30758        assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
30759    }
30760
30761    #[tokio::test]
30762    async fn test_decommission_supervisor_releases_slot_without_terminal_retry_when_pool_meta_is_blocked() {
30763        let generation = OffsetDateTime::UNIX_EPOCH;
30764        let canceler = DecommissionCanceler::new(CancellationToken::new());
30765        let pool_meta = PoolMeta {
30766            version: super::POOL_META_VERSION,
30767            pools: vec![decommission_test_pool_status(
30768                0,
30769                Some(PoolDecommissionInfo {
30770                    start_time: Some(generation),
30771                    ..Default::default()
30772                }),
30773            )],
30774            ..Default::default()
30775        };
30776        let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
30777        store
30778            .pool_meta_save_gate
30779            .lock()
30780            .await
30781            .observe_replicas(super::PoolMetaReplicaState {
30782                needs_repair: true,
30783                repair_write_safe: false,
30784            });
30785        let guards = guard_decommission_cancelers(vec![(0, canceler.clone())]);
30786
30787        tokio::time::timeout(
30788            StdDuration::from_secs(1),
30789            spawn_decommission_index_cancelers(store.clone(), CancellationToken::new(), guards, Arc::new(Semaphore::new(1))),
30790        )
30791        .await
30792        .expect("blocked supervisor must not enter terminal retry")
30793        .expect("blocked supervisor task should not panic");
30794
30795        assert!(store.decommission_cancelers.read().await[0].is_none());
30796        assert!(!canceler.is_active());
30797        let pool_meta = store.pool_meta.read().await;
30798        let info = pool_meta.pools[0]
30799            .decommission
30800            .as_ref()
30801            .expect("blocked decommission metadata should remain active");
30802        assert!(!info.failed);
30803        assert!(!info.canceled);
30804        assert!(!info.complete);
30805        assert_eq!(info.start_time, Some(generation));
30806    }
30807
30808    #[tokio::test]
30809    async fn test_decommission_promotion_rechecks_sticky_gate_after_start_wait() {
30810        let pool_meta = PoolMeta {
30811            pools: vec![decommission_test_pool_status(
30812                0,
30813                Some(PoolDecommissionInfo {
30814                    queued: true,
30815                    ..Default::default()
30816                }),
30817            )],
30818            ..Default::default()
30819        };
30820        let store = decommission_worker_test_store(pool_meta, vec![None]);
30821        let start_guard = store.start_gate.lock().await;
30822        let mut promotion = tokio::spawn({
30823            let store = store.clone();
30824            async move { store.promote_queued_decommission_for_test(0).await }
30825        });
30826        tokio::task::yield_now().await;
30827        assert!(!promotion.is_finished(), "promotion should be waiting for the start gate");
30828
30829        store
30830            .pool_meta_save_gate
30831            .lock()
30832            .await
30833            .observe_replicas(super::PoolMetaReplicaState {
30834                needs_repair: true,
30835                repair_write_safe: false,
30836            });
30837        drop(start_guard);
30838
30839        let err = tokio::time::timeout(StdDuration::from_secs(1), &mut promotion)
30840            .await
30841            .expect("blocked promotion should finish")
30842            .expect("promotion task should not panic")
30843            .expect_err("promotion must recheck the sticky gate after waiting");
30844        assert!(
30845            err.to_string()
30846                .contains("restart after all replicas are readable and consistent")
30847        );
30848
30849        let pool_meta = store.pool_meta.read().await;
30850        let info = pool_meta.pools[0]
30851            .decommission
30852            .as_ref()
30853            .expect("queued metadata should remain present");
30854        assert!(info.queued);
30855        assert!(info.start_time.is_none());
30856    }
30857
30858    #[tokio::test]
30859    #[serial_test::serial(pool_meta_version_env)]
30860    async fn test_decommission_promotion_persists_all_recovered_capacity_sources() {
30861        let (_temp_dirs, store, _other_store) =
30862            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
30863        persist_v3_pool_meta_for_test(&store).await;
30864        let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(3);
30865        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
30866        let mut queued_reservation = build_decommission_capacity_reservation(
30867            DecommissionPoolCapacityInfo::for_test(1, layout, 90, 100, 10),
30868            layout,
30869            uuid::Uuid::new_v4(),
30870            7,
30871            now,
30872        )
30873        .expect("queued source reservation should be valid");
30874        queued_reservation.targets.push(DecommissionCapacityTarget {
30875            pool_index: 2,
30876            layout,
30877            physical_total_at_reservation: 200,
30878            physical_free_at_reservation: 100,
30879            reserved_physical_bytes: queued_reservation.peak_physical_bytes,
30880            consumed_physical_bytes: 0,
30881            observed_physical_bytes: 0,
30882            inflight_physical_bytes: 0,
30883            pending_physical_bytes: 0,
30884            pending_mutation_id: None,
30885            temporary_mutations: Vec::new(),
30886        });
30887        {
30888            let mut pool_meta = store.pool_meta.write().await;
30889            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
30890                start_time: Some(now),
30891                ..Default::default()
30892            });
30893            pool_meta.pools[1].decommission = Some(PoolDecommissionInfo {
30894                queued: true,
30895                start_time: Some(now),
30896                capacity_reservation: Some(queued_reservation),
30897                ..Default::default()
30898            });
30899        }
30900        let mut loaded_v3 = PoolMeta::default();
30901        loaded_v3
30902            .load_no_lock_from_replicas(store.pools.clone())
30903            .await
30904            .expect("the startup fixture must provide a durable V3 pool metadata replica");
30905        assert_eq!(loaded_v3.version, POOL_META_GENERATION_VERSION);
30906        store
30907            .save_current_pool_meta_for_test(&[0, 1, 2])
30908            .await
30909            .expect("active and queued source metadata should be persisted before promotion");
30910        set_decommission_capacity_info_overrides_for_test(
30911            store.id,
30912            vec![vec![
30913                DecommissionPoolCapacityInfo::for_test(0, layout, 90, 100, 10),
30914                DecommissionPoolCapacityInfo::for_test(1, layout, 90, 100, 10),
30915                DecommissionPoolCapacityInfo::for_test(2, layout, 100, 200, 100),
30916            ]],
30917        );
30918
30919        store
30920            .promote_queued_decommission_for_test(1)
30921            .await
30922            .expect("queued promotion should recover all active source reservations");
30923
30924        let mut persisted = PoolMeta::default();
30925        persisted
30926            .load_no_lock_from_replicas(store.pools.clone())
30927            .await
30928            .expect("promoted active and recovered source metadata should reload");
30929        assert_eq!(persisted.version, POOL_META_GENERATION_VERSION);
30930        assert!(
30931            persisted.pools[0]
30932                .decommission
30933                .as_ref()
30934                .and_then(|info| info.capacity_reservation.as_ref())
30935                .is_some_and(DecommissionCapacityReservation::active)
30936        );
30937        let promoted = persisted.pools[1]
30938            .decommission
30939            .as_ref()
30940            .expect("queued source should remain after promotion");
30941        assert!(!promoted.queued, "queued source should be promoted durably");
30942        assert!(
30943            promoted
30944                .capacity_reservation
30945                .as_ref()
30946                .is_some_and(DecommissionCapacityReservation::active)
30947        );
30948    }
30949
30950    #[tokio::test]
30951    async fn test_decommission_bucket_done_rechecks_sticky_gate_before_mutation() {
30952        let bucket = DecomBucketInfo {
30953            name: "bucket-a".to_string(),
30954            prefix: String::new(),
30955        };
30956        let pool_meta = PoolMeta {
30957            pools: vec![decommission_test_pool_status(
30958                0,
30959                Some(PoolDecommissionInfo {
30960                    queued_buckets: vec![bucket.to_string()],
30961                    ..Default::default()
30962                }),
30963            )],
30964            ..Default::default()
30965        };
30966        let store = decommission_worker_test_store(pool_meta, vec![None]);
30967        store
30968            .pool_meta_save_gate
30969            .lock()
30970            .await
30971            .observe_replicas(super::PoolMetaReplicaState {
30972                needs_repair: true,
30973                repair_write_safe: false,
30974            });
30975
30976        let err = store
30977            .mark_decommission_bucket_done_and_save(0, OffsetDateTime::UNIX_EPOCH, &bucket)
30978            .await
30979            .expect_err("bucket completion must stop before mutating sticky pool metadata");
30980        assert!(
30981            err.to_string()
30982                .contains("restart after all replicas are readable and consistent")
30983        );
30984
30985        let pool_meta = store.pool_meta.read().await;
30986        let info = pool_meta.pools[0]
30987            .decommission
30988            .as_ref()
30989            .expect("decommission metadata should remain present");
30990        assert_eq!(info.queued_buckets, vec![bucket.to_string()]);
30991        assert!(info.decommissioned_buckets.is_empty());
30992    }
30993
30994    #[tokio::test]
30995    async fn test_decommission_cancel_save_failure_preserves_generation_and_blocks_retry() {
30996        let generation = OffsetDateTime::UNIX_EPOCH;
30997        let canceler = DecommissionCanceler::new(CancellationToken::new());
30998        let pool_meta = PoolMeta {
30999            version: super::POOL_META_VERSION,
31000            pools: vec![decommission_test_pool_status(
31001                0,
31002                Some(PoolDecommissionInfo {
31003                    start_time: Some(generation),
31004                    ..Default::default()
31005                }),
31006            )],
31007            ..Default::default()
31008        };
31009        let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
31010
31011        let err = store
31012            .decommission_cancel_with_owner_and_save(0, Some(&canceler), |_, _| async { Err(Error::Timeout) })
31013            .await
31014            .expect_err("injected pool metadata timeout should fail cancel");
31015        assert!(matches!(err, Error::Timeout));
31016
31017        {
31018            let pool_meta = store.pool_meta.read().await;
31019            let info = pool_meta.pools[0]
31020                .decommission
31021                .as_ref()
31022                .expect("failed cancel must retain decommission metadata");
31023            assert_eq!(info.start_time, Some(generation));
31024            assert!(!info.canceled);
31025            assert!(!info.complete);
31026            assert!(!info.failed);
31027            assert!(ensure_decommission_generation(&pool_meta, 0, generation).is_ok());
31028        }
31029        {
31030            let cancelers = store.decommission_cancelers.read().await;
31031            let current = cancelers[0].as_ref().expect("failed cancel must retain the worker owner");
31032            assert!(current.owns_same_operation(&canceler));
31033            assert!(current.is_active());
31034        }
31035        assert!(!canceler.is_cancelled());
31036        assert!(store.decommission_terminal_retryable_for_operation(0, &canceler).await);
31037
31038        let retry_save_called = Arc::new(AtomicBool::new(false));
31039        let retry_save_called_by_closure = retry_save_called.clone();
31040        let retry_err = store
31041            .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |_, _| async move {
31042                retry_save_called_by_closure.store(true, Ordering::SeqCst);
31043                Ok(())
31044            })
31045            .await
31046            .expect_err("an ambiguous save failure must block same-process retry");
31047        assert!(
31048            retry_err
31049                .to_string()
31050                .contains("restart after all replicas are readable and consistent")
31051        );
31052        assert!(!retry_save_called.load(Ordering::SeqCst));
31053
31054        {
31055            let pool_meta = store.pool_meta.read().await;
31056            let info = pool_meta.pools[0]
31057                .decommission
31058                .as_ref()
31059                .expect("blocked retry must retain decommission metadata");
31060            assert_eq!(info.start_time, Some(generation));
31061            assert!(!info.canceled);
31062            assert!(!info.complete);
31063            assert!(!info.failed);
31064        }
31065        assert!(store.decommission_cancelers.read().await[0].is_some());
31066        assert!(canceler.is_active());
31067        assert!(!canceler.is_cancelled());
31068    }
31069
31070    #[tokio::test]
31071    async fn test_decommission_cancel_stays_blocked_after_unreadable_pool_meta_replica() {
31072        let generation = OffsetDateTime::UNIX_EPOCH;
31073        let canceler = DecommissionCanceler::new(CancellationToken::new());
31074        let pool_meta = PoolMeta {
31075            version: super::POOL_META_VERSION,
31076            pools: vec![decommission_test_pool_status(
31077                0,
31078                Some(PoolDecommissionInfo {
31079                    start_time: Some(generation),
31080                    ..Default::default()
31081                }),
31082            )],
31083            ..Default::default()
31084        };
31085        let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
31086        store
31087            .pool_meta_save_gate
31088            .lock()
31089            .await
31090            .observe_replicas(super::PoolMetaReplicaState {
31091                needs_repair: true,
31092                repair_write_safe: false,
31093            });
31094        let save_called = Arc::new(AtomicBool::new(false));
31095        let save_called_by_closure = save_called.clone();
31096
31097        let err = store
31098            .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |_, _| async move {
31099                save_called_by_closure.store(true, Ordering::SeqCst);
31100                Ok(())
31101            })
31102            .await
31103            .expect_err("cancel must remain blocked until restart after an unreadable replica");
31104
31105        assert!(
31106            err.to_string()
31107                .contains("restart after all replicas are readable and consistent")
31108        );
31109        assert!(!save_called.load(Ordering::SeqCst));
31110        let pool_meta = store.pool_meta.read().await;
31111        let info = pool_meta.pools[0]
31112            .decommission
31113            .as_ref()
31114            .expect("blocked cancel must preserve decommission metadata");
31115        assert_eq!(info.start_time, Some(generation));
31116        assert!(!info.canceled);
31117        drop(pool_meta);
31118        assert!(canceler.is_active());
31119        assert!(!canceler.is_cancelled());
31120        let cancelers = store.decommission_cancelers.read().await;
31121        assert!(
31122            cancelers[0]
31123                .as_ref()
31124                .is_some_and(|current| current.owns_same_operation(&canceler))
31125        );
31126    }
31127
31128    #[tokio::test]
31129    async fn test_decommission_cancel_serializes_reload_until_local_commit() {
31130        let generation = OffsetDateTime::UNIX_EPOCH;
31131        let canceler = DecommissionCanceler::new(CancellationToken::new());
31132        let pool_meta = PoolMeta {
31133            version: super::POOL_META_VERSION,
31134            pools: vec![decommission_test_pool_status(
31135                0,
31136                Some(PoolDecommissionInfo {
31137                    start_time: Some(generation),
31138                    stage: "migrate_object".to_string(),
31139                    ..Default::default()
31140                }),
31141            )],
31142            ..Default::default()
31143        };
31144        let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
31145        let (persisted_tx, persisted_rx) = tokio::sync::oneshot::channel();
31146        let save_release = Arc::new(tokio::sync::Notify::new());
31147
31148        let cancel = tokio::spawn({
31149            let store = store.clone();
31150            let canceler = canceler.clone();
31151            let save_release = save_release.clone();
31152            async move {
31153                store
31154                    .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move {
31155                        persisted_tx
31156                            .send(snapshot.encode_config_data()?)
31157                            .map_err(|_| Error::other("failed to expose saved cancel snapshot"))?;
31158                        save_release.notified().await;
31159                        Ok(())
31160                    })
31161                    .await
31162            }
31163        });
31164        let persisted = persisted_rx.await.expect("save should expose the canceled snapshot");
31165        let reload_started = Arc::new(tokio::sync::Notify::new());
31166        let reload = tokio::spawn({
31167            let store = store.clone();
31168            let reload_started = reload_started.clone();
31169            async move {
31170                let mut reloaded = PoolMeta::default();
31171                reloaded.load_from_config_data(persisted)?;
31172                reload_started.notify_one();
31173                *store.pool_meta.write().await = reloaded;
31174                Ok::<(), Error>(())
31175            }
31176        });
31177        reload_started.notified().await;
31178        assert!(!reload.is_finished(), "peer reload must wait for cancel publication");
31179
31180        save_release.notify_one();
31181        cancel
31182            .await
31183            .expect("cancel task should not panic")
31184            .expect("cancel should commit before releasing the reload");
31185        reload
31186            .await
31187            .expect("reload task should not panic")
31188            .expect("reload should install the saved cancel after publication");
31189
31190        let pool_meta = store.pool_meta.read().await;
31191        let info = pool_meta.pools[0]
31192            .decommission
31193            .as_ref()
31194            .expect("reloaded cancel metadata should remain present");
31195        assert!(info.canceled);
31196        assert!(!info.complete);
31197        assert!(!info.failed);
31198        assert!(info.start_time.is_none());
31199        assert!(info.stage.is_empty(), "the persisted snapshot should have been decoded before commit");
31200        drop(pool_meta);
31201        assert!(store.decommission_cancelers.read().await[0].is_none());
31202        assert!(!canceler.is_active());
31203        assert!(canceler.is_cancelled());
31204    }
31205
31206    #[tokio::test]
31207    async fn test_decommission_cancel_transaction_survives_caller_abort_after_durable_save() {
31208        let generation = OffsetDateTime::UNIX_EPOCH;
31209        let canceler = DecommissionCanceler::new(CancellationToken::new());
31210        let pool_meta = PoolMeta {
31211            version: super::POOL_META_VERSION,
31212            pools: vec![decommission_test_pool_status(
31213                0,
31214                Some(PoolDecommissionInfo {
31215                    start_time: Some(generation),
31216                    ..Default::default()
31217                }),
31218            )],
31219            ..Default::default()
31220        };
31221        let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
31222        let persisted = Arc::new(std::sync::Mutex::new(None));
31223        let (durable_tx, durable_rx) = tokio::sync::oneshot::channel();
31224        let save_release = Arc::new(tokio::sync::Notify::new());
31225        let cancel = tokio::spawn({
31226            let store = store.clone();
31227            let canceler = canceler.clone();
31228            let persisted = persisted.clone();
31229            let save_release = save_release.clone();
31230            async move {
31231                store
31232                    .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move {
31233                        assert!(
31234                            snapshot.pools[0]
31235                                .decommission
31236                                .as_ref()
31237                                .is_some_and(|info| info.canceled && info.start_time.is_none())
31238                        );
31239                        *persisted.lock().expect("persisted cancel lock should not be poisoned") =
31240                            Some(snapshot.encode_config_data()?);
31241                        durable_tx
31242                            .send(())
31243                            .map_err(|_| Error::other("failed to report durable cancel snapshot"))?;
31244                        save_release.notified().await;
31245                        Ok(())
31246                    })
31247                    .await
31248            }
31249        });
31250        durable_rx.await.expect("save hook should report the durable cancel snapshot");
31251        cancel.abort();
31252        let join_err = cancel.await.expect_err("caller cancel future should be aborted");
31253        assert!(join_err.is_cancelled());
31254        assert!(
31255            store.pool_meta.try_read().is_err(),
31256            "the detached transaction must retain its state guard"
31257        );
31258        assert!(
31259            store.decommission_cancelers.try_read().is_err(),
31260            "the detached transaction must retain its owner guard"
31261        );
31262
31263        save_release.notify_one();
31264        tokio::time::timeout(StdDuration::from_secs(1), canceler.token().cancelled())
31265            .await
31266            .expect("detached cancel transaction should terminate the old token");
31267
31268        let persisted = persisted
31269            .lock()
31270            .expect("persisted cancel lock should not be poisoned")
31271            .take()
31272            .expect("save should capture the durable cancel snapshot");
31273        let mut durable = PoolMeta::default();
31274        durable
31275            .load_from_config_data(persisted)
31276            .expect("durable cancel snapshot should decode");
31277        assert!(
31278            durable.pools[0]
31279                .decommission
31280                .as_ref()
31281                .is_some_and(|info| info.canceled && info.start_time.is_none())
31282        );
31283        assert!(store.decommission_cancelers.read().await[0].is_none());
31284        assert!(!canceler.is_active());
31285        assert!(canceler.is_cancelled());
31286
31287        let side_effect_ran = Arc::new(AtomicBool::new(false));
31288        let operation_gate = store.ctx.data_movement_operation_gate();
31289        let result = run_decommission_side_effect(canceler.token(), &operation_gate, {
31290            let side_effect_ran = side_effect_ran.clone();
31291            move || async move {
31292                side_effect_ran.store(true, Ordering::SeqCst);
31293                Ok(())
31294            }
31295        })
31296        .await;
31297        assert!(matches!(result, Err(Error::OperationCanceled)));
31298        assert!(!side_effect_ran.load(Ordering::SeqCst));
31299        assert!(
31300            store.pool_meta.read().await.pools[0]
31301                .decommission
31302                .as_ref()
31303                .is_some_and(|info| info.canceled && !info.failed)
31304        );
31305    }
31306
31307    #[tokio::test]
31308    async fn test_decommission_cancel_persists_before_signaling_and_quiesces_before_return() {
31309        let generation = OffsetDateTime::UNIX_EPOCH;
31310        let canceler = DecommissionCanceler::new(CancellationToken::new());
31311        let pool_meta = PoolMeta {
31312            pools: vec![decommission_test_pool_status(
31313                0,
31314                Some(PoolDecommissionInfo {
31315                    start_time: Some(generation),
31316                    ..Default::default()
31317                }),
31318            )],
31319            ..Default::default()
31320        };
31321        let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
31322        let operation_gate = store.ctx.data_movement_operation_gate();
31323        let side_effect = operation_gate.read().await;
31324        let save_started = Arc::new(tokio::sync::Notify::new());
31325        let save_release = Arc::new(tokio::sync::Notify::new());
31326        let save_entered = Arc::new(AtomicBool::new(false));
31327        let persisted = Arc::new(std::sync::Mutex::new(None));
31328
31329        let mut cancel = tokio::spawn({
31330            let store = store.clone();
31331            let canceler = canceler.clone();
31332            let save_started = save_started.clone();
31333            let save_release = save_release.clone();
31334            let save_entered = save_entered.clone();
31335            let persisted = persisted.clone();
31336            async move {
31337                store
31338                    .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move {
31339                        *persisted.lock().expect("persisted cancel lock should not be poisoned") =
31340                            Some(snapshot.encode_config_data()?);
31341                        save_entered.store(true, Ordering::SeqCst);
31342                        save_started.notify_one();
31343                        save_release.notified().await;
31344                        Ok(())
31345                    })
31346                    .await
31347            }
31348        });
31349
31350        tokio::time::timeout(StdDuration::from_secs(1), save_started.notified())
31351            .await
31352            .expect("cancel should persist while the side effect is still in flight");
31353        assert!(save_entered.load(Ordering::SeqCst));
31354        assert!(!cancel.is_finished(), "cancel must wait for the injected save");
31355        assert!(
31356            store.pool_meta_save_gate.try_lock().is_err(),
31357            "the cancel save must exclude stale full-document saves until publication"
31358        );
31359        assert!(
31360            store.pool_meta.try_read().is_err(),
31361            "the active generation must stay write-locked through persistence"
31362        );
31363        assert!(!canceler.is_cancelled(), "the token must remain live until persistence commits");
31364
31365        let mut fail = tokio::spawn({
31366            let store = store.clone();
31367            async move { store.decommission_failed(0).await }
31368        });
31369        tokio::task::yield_now().await;
31370        assert!(!fail.is_finished(), "fail must serialize behind the pending cancel");
31371
31372        save_release.notify_one();
31373        tokio::time::timeout(StdDuration::from_secs(1), canceler.token().cancelled())
31374            .await
31375            .expect("the durable cancel must signal the active worker");
31376        assert!(!cancel.is_finished(), "cancel must wait for in-flight movement after the durable signal");
31377        let canceled_at = {
31378            let pool_meta = store.pool_meta.read().await;
31379            let info = pool_meta.pools[0]
31380                .decommission
31381                .as_ref()
31382                .expect("the durable cancel must be published before quiescence");
31383            assert!(info.canceled);
31384            assert!(!info.complete);
31385            assert!(!info.failed);
31386            assert!(info.start_time.is_none());
31387            pool_meta.pools[0].last_update
31388        };
31389        let expected_generation = crate::store::scanner_data_movement_timestamp_generation(canceled_at);
31390        let scanner_status = store.scanner_data_movement_pause_snapshot_for_test().await;
31391        assert_eq!(scanner_status.movement_generation, expected_generation);
31392
31393        drop(side_effect);
31394        tokio::time::timeout(StdDuration::from_secs(1), &mut cancel)
31395            .await
31396            .expect("cancel should finish after in-flight movement quiesces")
31397            .expect("cancel task should not panic")
31398            .expect("cancel should commit");
31399        tokio::time::timeout(StdDuration::from_secs(1), &mut fail)
31400            .await
31401            .expect("fail should finish after cancel commits")
31402            .expect("fail task should not panic")
31403            .expect("stale fail should be a no-op");
31404
31405        let pool_meta = store.pool_meta.read().await;
31406        let info = pool_meta.pools[0]
31407            .decommission
31408            .as_ref()
31409            .expect("cancel metadata should remain present");
31410        assert!(info.canceled);
31411        assert!(!info.complete);
31412        assert!(!info.failed);
31413        assert!(info.start_time.is_none());
31414        drop(pool_meta);
31415        assert!(canceler.is_cancelled());
31416        assert!(!canceler.is_active());
31417        assert!(store.decommission_cancelers.read().await[0].is_none());
31418        assert_eq!(store.scanner_data_movement_generation(), expected_generation);
31419
31420        let persisted = persisted
31421            .lock()
31422            .expect("persisted cancel lock should not be poisoned")
31423            .take()
31424            .expect("cancel should persist a durable snapshot");
31425        let mut durable = PoolMeta::default();
31426        durable
31427            .load_from_config_data(persisted)
31428            .expect("durable cancel snapshot should decode after restart");
31429        let restarted = decommission_worker_test_store(durable.clone(), Vec::new());
31430        let restarted_status = restarted.scanner_data_movement_pause_status().await;
31431        assert_eq!(restarted_status.movement_generation, expected_generation);
31432
31433        assert!(
31434            durable
31435                .clear_decommission_at_for_test(0, canceled_at, None)
31436                .expect("same-tick clear should succeed")
31437        );
31438        assert!(durable.pools[0].last_update > canceled_at);
31439    }
31440
31441    #[test]
31442    fn test_decommission_cancel_commit_rejects_a_replaced_generation() {
31443        let old_generation = OffsetDateTime::UNIX_EPOCH;
31444        let new_generation = old_generation + Duration::seconds(1);
31445        let mut canceled = PoolMeta {
31446            pools: vec![decommission_test_pool_status(
31447                0,
31448                Some(PoolDecommissionInfo {
31449                    start_time: Some(old_generation),
31450                    ..Default::default()
31451                }),
31452            )],
31453            ..Default::default()
31454        };
31455        let previous_last_update = canceled.pools[0].last_update;
31456        assert!(canceled.decommission_cancel(0));
31457        let canceled_pool = canceled.pools.remove(0);
31458        let commit = super::DecommissionCancelCommit {
31459            previous_start_time: Some(old_generation),
31460            previous_queued: false,
31461            previous_last_update,
31462            canceled_pool,
31463        };
31464        let mut current = PoolMeta {
31465            pools: vec![decommission_test_pool_status(
31466                0,
31467                Some(PoolDecommissionInfo {
31468                    start_time: Some(new_generation),
31469                    ..Default::default()
31470                }),
31471            )],
31472            ..Default::default()
31473        };
31474
31475        let err = super::commit_decommission_cancel(&mut current, 0, commit)
31476            .expect_err("an old cancel must not publish over a replacement generation");
31477        assert!(err.to_string().contains("operation generation changed"));
31478        let info = current.pools[0]
31479            .decommission
31480            .as_ref()
31481            .expect("replacement generation should remain present");
31482        assert_eq!(info.start_time, Some(new_generation));
31483        assert!(!info.canceled);
31484    }
31485
31486    #[tokio::test]
31487    async fn test_decommission_cancel_rejects_stale_retry_after_queued_replacement() {
31488        let old_generation = OffsetDateTime::UNIX_EPOCH;
31489        let canceler = DecommissionCanceler::new(CancellationToken::new());
31490        let pool_meta = PoolMeta {
31491            pools: vec![decommission_test_pool_status(
31492                0,
31493                Some(PoolDecommissionInfo {
31494                    start_time: Some(old_generation),
31495                    ..Default::default()
31496                }),
31497            )],
31498            ..Default::default()
31499        };
31500        let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
31501        let queued_replacement = Arc::new(std::sync::Mutex::new(None));
31502        let queued_replacement_for_save = queued_replacement.clone();
31503
31504        store
31505            .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move {
31506                let saved_cancel = snapshot.pools[0]
31507                    .decommission
31508                    .as_ref()
31509                    .expect("saved snapshot should contain decommission metadata");
31510                assert!(saved_cancel.canceled);
31511                assert!(saved_cancel.start_time.is_none());
31512                let mut queued_replacement = snapshot.clone();
31513                let replacement = queued_replacement
31514                    .pools
31515                    .get_mut(0)
31516                    .expect("cancel snapshot should contain the pool");
31517                replacement.last_update += Duration::seconds(1);
31518                let info = replacement
31519                    .decommission
31520                    .as_mut()
31521                    .expect("cancel snapshot should contain decommission metadata");
31522                info.canceled = false;
31523                info.queued = true;
31524                *queued_replacement_for_save
31525                    .lock()
31526                    .expect("queued replacement lock should not be poisoned") = Some(queued_replacement);
31527                Ok(())
31528            })
31529            .await
31530            .expect("cancel should commit before a queued replacement is installed");
31531        assert!(store.decommission_cancelers.read().await[0].is_none());
31532        assert!(!canceler.is_active());
31533        assert!(canceler.is_cancelled());
31534
31535        let queued_replacement = queued_replacement
31536            .lock()
31537            .expect("queued replacement lock should not be poisoned")
31538            .take()
31539            .expect("save should prepare the queued replacement");
31540        *store.pool_meta.write().await = queued_replacement;
31541
31542        let replacement_revision = {
31543            let pool_meta = store.pool_meta.read().await;
31544            let info = pool_meta.pools[0]
31545                .decommission
31546                .as_ref()
31547                .expect("queued replacement should remain present");
31548            assert!(info.queued);
31549            assert!(!info.canceled);
31550            assert!(info.start_time.is_none());
31551            pool_meta.pools[0].last_update
31552        };
31553
31554        store.retry_decommission_cancel_for_operation(0, &canceler).await;
31555
31556        let pool_meta = store.pool_meta.read().await;
31557        let info = pool_meta.pools[0]
31558            .decommission
31559            .as_ref()
31560            .expect("stale retry must preserve the queued replacement");
31561        assert_eq!(pool_meta.pools[0].last_update, replacement_revision);
31562        assert!(info.queued);
31563        assert!(!info.canceled);
31564        assert!(info.start_time.is_none());
31565    }
31566
31567    #[tokio::test]
31568    async fn test_decommission_failed_save_failure_preserves_owner_until_retry_succeeds() {
31569        let canceler = DecommissionCanceler::new(CancellationToken::new());
31570        let pool_meta = PoolMeta {
31571            pools: vec![decommission_test_pool_status(
31572                0,
31573                Some(PoolDecommissionInfo {
31574                    start_time: Some(OffsetDateTime::UNIX_EPOCH),
31575                    ..Default::default()
31576                }),
31577            )],
31578            ..Default::default()
31579        };
31580        let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
31581
31582        store
31583            .decommission_failed_with_owner_and_save(0, Some(&canceler), async { Err(Error::SlowDown) })
31584            .await
31585            .expect_err("injected terminal save failure should be returned");
31586
31587        {
31588            let cancelers = store.decommission_cancelers.read().await;
31589            let current = cancelers[0].as_ref().expect("failed save must retain the exact owner slot");
31590            assert!(current.owns_same_operation(&canceler));
31591            assert!(current.is_active());
31592        }
31593        {
31594            let pool_meta = store.pool_meta.read().await;
31595            let info = pool_meta.pools[0]
31596                .decommission
31597                .as_ref()
31598                .expect("rollback must retain active decommission metadata");
31599            assert!(info.has_decommission_state());
31600            assert!(!info.failed);
31601            assert!(!info.complete);
31602            assert!(!info.canceled);
31603        }
31604        assert!(store.decommission_terminal_retryable_for_operation(0, &canceler).await);
31605
31606        store
31607            .decommission_failed_with_owner_and_save(0, Some(&canceler), async { Ok(()) })
31608            .await
31609            .expect("terminal retry should commit");
31610
31611        let pool_meta = store.pool_meta.read().await;
31612        assert!(
31613            pool_meta.pools[0]
31614                .decommission
31615                .as_ref()
31616                .expect("terminal metadata should remain")
31617                .failed
31618        );
31619        drop(pool_meta);
31620        assert!(store.decommission_cancelers.read().await[0].is_none());
31621        assert!(!canceler.is_active());
31622        assert!(canceler.is_cancelled());
31623        assert_eq!(store.ctx.data_movement_operation_epoch(), 1);
31624    }
31625
31626    #[test]
31627    fn test_stale_decommission_operation_cannot_cancel_replacement() {
31628        let stale = DecommissionCanceler::new(CancellationToken::new());
31629        let replacement = DecommissionCanceler::new(CancellationToken::new());
31630        let cancelers = vec![Some(replacement.clone())];
31631        let mut pool_meta = PoolMeta {
31632            pools: vec![decommission_test_pool_status(
31633                0,
31634                Some(PoolDecommissionInfo {
31635                    start_time: Some(OffsetDateTime::UNIX_EPOCH),
31636                    ..Default::default()
31637                }),
31638            )],
31639            ..Default::default()
31640        };
31641
31642        let changed = update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, 0, Some(&stale), |pool_meta| {
31643            pool_meta.decommission_cancel(0)
31644        });
31645
31646        assert!(changed.is_none());
31647        assert!(
31648            !pool_meta.pools[0]
31649                .decommission
31650                .as_ref()
31651                .expect("replacement metadata should remain")
31652                .canceled
31653        );
31654        assert!(replacement.is_active());
31655        assert!(!replacement.is_cancelled());
31656        assert!(!stale.is_active());
31657        assert!(stale.is_cancelled());
31658    }
31659
31660    #[test]
31661    fn test_ensure_decommission_routines_scheduled_accepts_positive_bound_count() {
31662        assert!(super::ensure_decommission_routines_scheduled(2, 2).is_ok());
31663    }
31664
31665    #[test]
31666    fn test_ensure_decommission_routines_scheduled_rejects_zero_bound_count() {
31667        let err = super::ensure_decommission_routines_scheduled(0, 1).expect_err("zero bound count should be rejected");
31668        assert!(
31669            err.to_string()
31670                .contains("failed to start decommission routines: scheduled 0 of 1 expected workers")
31671        );
31672    }
31673
31674    #[test]
31675    fn test_ensure_decommission_routines_scheduled_rejects_partial_binding() {
31676        let err = super::ensure_decommission_routines_scheduled(1, 2).expect_err("partial binding should be rejected");
31677        assert!(
31678            err.to_string()
31679                .contains("failed to start decommission routines: scheduled 1 of 2 expected workers")
31680        );
31681    }
31682
31683    #[test]
31684    #[cfg(windows)]
31685    fn test_path2_bucket_object_with_base_path_supports_windows_separators() {
31686        let (bucket, object) = super::path2_bucket_object_with_base_path("C:\\data", "C:\\data\\my-bucket\\nested\\object.txt");
31687
31688        assert_eq!(bucket, "my-bucket");
31689        assert_eq!(object, "nested/object.txt");
31690    }
31691}