Skip to main content

rustfs_ecstore/core/
pools.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use crate::bucket::replication::replication_state_from_filemeta;
16#[cfg(all(test, feature = "test-util"))]
17use crate::bucket::utils::is_meta_bucketname;
18use crate::bucket::versioning::VersioningApi as _;
19use crate::bucket::versioning_sys::BucketVersioningSys;
20use crate::bucket::{
21    lifecycle::{
22        DurableIlmRecordCheckpoint, ILM_META_PREFIX, LifecycleExpiryConfigs, TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE,
23        ValidatedDurableIlmRecord,
24        bucket_lifecycle_audit::LcEventSrc,
25        bucket_lifecycle_ops::{
26            LifecycleOps, apply_expiry_on_transitioned_object, apply_expiry_rule_for_data_movement, apply_expiry_rule_in,
27            eval_action_from_lifecycle, lifecycle_delete_all_versions_blocked_by_replication,
28        },
29        classify_durable_ilm_record, get_expiry_configs,
30        lifecycle::IlmAction,
31        tier_delete_journal::durable_ilm_v6_topology_generation,
32        validate_durable_ilm_record,
33    },
34    metadata_sys,
35};
36use crate::cache_value::metacache_set::{ListPathRawOptions, list_path_raw};
37use crate::config::com::{
38    CONFIG_PREFIX, delete_config, read_config_limited_preserve_empty, read_config_limited_preserve_empty_with_metadata,
39    read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty, save_config_with_opts,
40    save_config_with_opts_and_metadata,
41};
42use crate::data_movement;
43use crate::data_movement::backpressure::{self, DataMovementOperation};
44use crate::data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_cache};
45use crate::disk::error::DiskError;
46use crate::disk::{BUCKET_META_PREFIX, DiskAPI, RUSTFS_META_BUCKET};
47use crate::error::{Error, Result};
48use crate::error::{
49    StorageError, is_err_bucket_exists, is_err_bucket_not_found, is_err_data_movement_overwrite, is_err_invalid_upload_id,
50    is_err_object_not_found, is_err_operation_canceled, is_err_version_not_found,
51};
52use crate::layout::endpoints::EndpointServerPools;
53use crate::object_api::{DecommissionCapacityOptions, GetObjectReader, ObjectInfo, ObjectOptions};
54use crate::runtime::sources as runtime_sources;
55use crate::services::notification_sys::{
56    acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches, tier_delete_journal_topology_generation,
57};
58use crate::services::rebalance::{REBAL_META_NAME, RebalanceMeta, is_rebalance_conflicting_with_decommission};
59use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
60use crate::storage_api_contracts::{
61    admin::StorageAdminApi,
62    bucket::{BucketOperations, BucketOptions, MakeBucketOptions},
63    heal::HealOperations as _,
64    list::ListOperations as _,
65    namespace::NamespaceLocking as _,
66    object::{EcstoreObjectIO, HTTPPreconditions, ObjectIO as _, ObjectOperations as _},
67};
68use crate::store::PoolMetaWriteGateStatus;
69use crate::{core::sets::Sets, store::ECStore};
70use byteorder::{ByteOrder, LittleEndian, WriteBytesExt};
71use futures::{
72    StreamExt,
73    future::{BoxFuture, join_all},
74    stream::FuturesUnordered,
75};
76use http::HeaderMap;
77use rand::RngExt as _;
78#[cfg(test)]
79use rmp_serde::Deserializer;
80use rmp_serde::Serializer;
81use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams};
82use rustfs_heal_contracts::heal_channel::HealOpts;
83use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
84use rustfs_utils::path::{
85    decode_dir_object, encode_dir_object, path_join, path_to_bucket_object, path_to_bucket_object_with_base_path,
86};
87use s3s::dto::{BucketLifecycleConfiguration, ObjectLockConfiguration, ReplicationConfiguration};
88use serde::{Deserialize, Serialize};
89use sha2::{Digest, Sha256};
90use std::collections::{HashMap, HashSet};
91use std::fmt::Display;
92#[cfg(test)]
93use std::io::Cursor;
94use std::io::Write;
95use std::path::PathBuf;
96use std::sync::{
97    Arc,
98    atomic::{AtomicBool, AtomicUsize, Ordering},
99};
100use time::{Duration, OffsetDateTime};
101use tokio::sync::{OwnedSemaphorePermit, Semaphore, mpsc};
102use tokio_util::sync::CancellationToken;
103use tracing::{debug, error, info, warn};
104
105const LOG_COMPONENT_ECSTORE: &str = "ecstore";
106const LOG_SUBSYSTEM_POOLS: &str = "pools";
107const EVENT_DECOMMISSION_STATE: &str = "decommission_state";
108const EVENT_DECOMMISSION_BUCKET: &str = "decommission_bucket";
109const EVENT_DECOMMISSION_ENTRY: &str = "decommission_entry";
110const POOL_ACTIVATION_FLEET_PROOF_REQUIRED: &str = "pool activation requires a live fleet capability proof";
111const POOL_ACTIVATION_FLEET_PROOF_EXPIRED: &str = "pool activation fleet capability proof expired before commit";
112const DECOMMISSION_TARGET_FLEET_PROOF_EXPIRED: &str =
113    "decommission target fence fleet capability proof expired before reservation commit";
114const DECOMMISSION_STAGE_MIGRATE_OBJECT: &str = "migrate_object";
115const DECOMMISSION_STAGE_CLEANUP_PREFLIGHT: &str = "cleanup_preflight";
116const DECOMMISSION_STAGE_SOURCE_CLEANUP: &str = "source_cleanup";
117const DECOMMISSION_STAGE_ENTRY_FINISHED: &str = "entry_finished";
118const DECOMMISSION_PROGRESS_SAVE_INTERVAL: Duration = Duration::seconds(30);
119const DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD: usize = 1000;
120const DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF: Duration = Duration::seconds(1);
121const DECOMMISSION_BUCKET_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_BUCKET_CONCURRENCY";
122const DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP: usize = 4;
123const DECOMMISSION_ENTRY_CONCURRENCY_ENV: &str = "RUSTFS_DECOMMISSION_ENTRY_CONCURRENCY";
124const DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP: usize = 8;
125const DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP: usize = 64;
126const DECOMMISSION_ENTRY_WORKERS_PER_SET: usize = 2;
127const DECOMMISSION_META_PREFIXES: [&str; 3] = [CONFIG_PREFIX, BUCKET_META_PREFIX, ILM_META_PREFIX];
128const DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION: u16 = 1;
129const DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION: u16 = 2;
130#[cfg(test)]
131const DECOMMISSION_CAPACITY_MODEL_VERSION: u16 = DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION;
132const DECOMMISSION_CAPACITY_TEMPORARY_COPIES: usize = 1;
133const DECOMMISSION_CAPACITY_RESERVATION_TTL: Duration = Duration::minutes(10);
134const DECOMMISSION_CAPACITY_RELEASE_CANCELED: &str = "canceled";
135const DECOMMISSION_CAPACITY_RELEASE_FAILED: &str = "failed";
136const DECOMMISSION_CAPACITY_RELEASE_COMPLETED: &str = "completed";
137pub(crate) const DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX: &str = "decommission/capacity-target";
138const DECOMMISSION_CAPACITY_TARGET_LOCK_TIMEOUT: std::time::Duration = std::time::Duration::from_millis(250);
139const DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS: usize = 12;
140pub(crate) const DECOMMISSION_MUTATION_GATE_MAX_INLINE_ATTEMPTS: usize = 3;
141static DECOMMISSION_OBJECT_ATTEMPTS_MAX: std::sync::Mutex<u64> = std::sync::Mutex::new(0);
142const DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX: &str = "target pool ";
143const DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX: &str = " target capacity mutation gate is busy";
144const METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL: &str = "rustfs_decommission_capacity_conflicts_total";
145const METRIC_DECOMMISSION_CAPACITY_GATE_RETRIES_TOTAL: &str = "rustfs_decommission_capacity_gate_retries_total";
146/// Entry passes and repeated version-copy attempts within a listing and its
147/// deferred replay. Inline gate waits have their own retry counter.
148const METRIC_DECOMMISSION_OBJECT_ATTEMPTS_TOTAL: &str = "rustfs_decommission_object_attempts_total";
149/// Highest per-entry attempt count, including its deferred replay. A new
150/// listing after a durable pause starts a new observation.
151const METRIC_DECOMMISSION_OBJECT_ATTEMPTS_MAX: &str = "rustfs_decommission_object_attempts_max";
152/// Wait time spent inside a single target capacity gate retry.
153const METRIC_DECOMMISSION_CAPACITY_GATE_WAIT_SECONDS: &str = "rustfs_decommission_capacity_gate_wait_seconds";
154const METRIC_DECOMMISSION_CAPACITY_PREDICTED_BYTES: &str = "rustfs_decommission_capacity_predicted_physical_bytes";
155const METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES: &str = "rustfs_decommission_capacity_reserved_physical_bytes";
156const METRIC_DECOMMISSION_CAPACITY_PREDICTION_ERROR_BYTES: &str = "rustfs_decommission_capacity_prediction_error_bytes";
157const METRIC_DECOMMISSION_CAPACITY_PREDICTION_ABSOLUTE_ERROR_BYTES: &str =
158    "rustfs_decommission_capacity_prediction_absolute_error_bytes";
159const DECOMMISSION_LISTING_MAX_ATTEMPTS: usize = 3;
160const DECOMMISSION_LISTING_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(5);
161pub(crate) const DECOMMISSION_ENTRY_MAX_ATTEMPTS: usize = 3;
162const DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS: usize = 12;
163// A scanner pause backlog handoff conflict is a transient membership
164// transition: the native authority is expected to converge once surviving
165// replicas advance their durable commit. Bound the wait so a genuinely
166// divergent authority still reaches a terminal, attributable failure.
167const DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS: usize = 12;
168const DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(100);
169pub(crate) const DECOMMISSION_VERSION_COPY_ATTEMPTS: usize = 3;
170const DECOMMISSION_COPY_RETRY_DELAY: std::time::Duration = std::time::Duration::from_millis(50);
171const DECOMMISSION_CAPACITY_GATE_RETRY_BASE: std::time::Duration = std::time::Duration::from_millis(100);
172const DECOMMISSION_RETRY_BACKOFF_CAP: std::time::Duration = std::time::Duration::from_millis(1500);
173const DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT: usize = 100;
174const DECOMMISSION_TERMINAL_RETRY_DELAY: std::time::Duration = std::time::Duration::from_secs(1);
175const DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS: usize = 3;
176
177fn decommission_capacity_target_gate_retry_exhausted(attempt: usize) -> bool {
178    attempt.saturating_add(1) >= DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS
179}
180
181fn record_decommission_capacity_gate_retry(target_pool_index: usize) {
182    metrics::counter!(
183        METRIC_DECOMMISSION_CAPACITY_GATE_RETRIES_TOTAL,
184        "target_pool_index" => target_pool_index.to_string()
185    )
186    .increment(1);
187}
188
189/// Record one object-processing attempt plus the peak attempts any object
190/// reached, so amplification is visible without reconstructing it from logs.
191fn record_decommission_object_attempt(attempts: &mut usize) {
192    *attempts = attempts.saturating_add(1);
193    metrics::counter!(METRIC_DECOMMISSION_OBJECT_ATTEMPTS_TOTAL).increment(1);
194    let attempt = u64::try_from(*attempts).unwrap_or(u64::MAX);
195    // Serialize publication too: concurrent fetch_max + gauge.set can publish
196    // an older peak after a newer one. This guard never crosses an await.
197    let mut peak = DECOMMISSION_OBJECT_ATTEMPTS_MAX.lock().unwrap_or_else(|err| err.into_inner());
198    if *peak < attempt {
199        *peak = attempt;
200        metrics::gauge!(METRIC_DECOMMISSION_OBJECT_ATTEMPTS_MAX).set(attempt as f64);
201    }
202}
203
204fn record_decommission_capacity_gate_wait(delay: std::time::Duration) {
205    metrics::histogram!(METRIC_DECOMMISSION_CAPACITY_GATE_WAIT_SECONDS).record(delay.as_secs_f64());
206}
207const DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT: &str = "decommission/ilm-receipts";
208const DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT: &str = "decommission/ilm-manifests";
209const DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA: &str = "v2";
210const DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA: &str = "v1";
211const DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE: usize = 16 * 1024;
212const DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE: usize = 4 * 1024;
213const DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS: usize = 3;
214/// Background decommission walks must tolerate slow object migrations; the
215/// stall timeout is the drive-health bound, not the total listing duration.
216const DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60);
217
218pub const POOL_META_NAME: &str = "pool.bin";
219pub(crate) const POOL_META_IDENTITY_NAME: &str = "pool.bin.identity";
220pub const POOL_META_FORMAT: u16 = 1;
221const POOL_META_V1_VERSION: u16 = 1;
222pub const POOL_META_VERSION: u16 = 2;
223const POOL_META_GENERATION_VERSION: u16 = 3;
224const POOL_META_IDENTITY_FORMAT: u16 = 1;
225const POOL_META_IDENTITY_VERSION: u16 = 1;
226const POOL_META_INITIAL_EPOCH: u64 = 1;
227const POOL_META_CAS_MAX_ATTEMPTS: usize = 3;
228const METRIC_POOL_META_STALE_WRITE_REJECTIONS_TOTAL: &str = "rustfs_pool_meta_stale_write_rejections_total";
229
230fn record_pool_meta_stale_write_rejection(reason: &'static str) {
231    metrics::counter!(METRIC_POOL_META_STALE_WRITE_REJECTIONS_TOTAL, "reason" => reason).increment(1);
232}
233
234fn pool_meta_v2_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool {
235    requested && fleet_confirmed
236}
237
238fn pool_meta_v2_writer_enabled() -> bool {
239    pool_meta_v2_writer_enabled_for(
240        rustfs_utils::get_env_bool(rustfs_config::ENV_POOL_META_V2_WRITE, rustfs_config::DEFAULT_POOL_META_V2_WRITE),
241        rustfs_utils::get_env_bool(
242            rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
243            rustfs_config::DEFAULT_POOL_META_V2_FLEET_CONFIRMED,
244        ),
245    )
246}
247
248fn pool_meta_v3_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool {
249    requested && fleet_confirmed
250}
251
252fn pool_meta_v3_writer_enabled() -> bool {
253    pool_meta_v3_writer_enabled_for(
254        rustfs_utils::get_env_bool(rustfs_config::ENV_POOL_META_V3_WRITE, rustfs_config::DEFAULT_POOL_META_V3_WRITE),
255        rustfs_utils::get_env_bool(
256            rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED,
257            rustfs_config::DEFAULT_POOL_META_V3_FLEET_CONFIRMED,
258        ),
259    )
260}
261
262fn decommission_capacity_writer_supported_for(version: u16, v2_writer_enabled: bool, v3_writer_enabled: bool) -> bool {
263    matches!(version, POOL_META_VERSION | POOL_META_GENERATION_VERSION) || v2_writer_enabled || v3_writer_enabled
264}
265
266fn ensure_decommission_ledger_persistence_supported_for(
267    version: u16,
268    v2_writer_enabled: bool,
269    v3_writer_enabled: bool,
270) -> Result<()> {
271    if decommission_capacity_writer_supported_for(version, v2_writer_enabled, v3_writer_enabled) {
272        return Ok(());
273    }
274
275    Err(Error::InvalidArgument(
276        "decommission".to_string(),
277        "pool-metadata-version".to_string(),
278        format!(
279            "durable unresolved-entry recovery requires pool metadata V2 or V3; enable either the {} + {} V2 gate or the {} + {} V3 gate only after every reader and writer supports that format",
280            rustfs_config::ENV_POOL_META_V2_WRITE,
281            rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
282            rustfs_config::ENV_POOL_META_V3_WRITE,
283            rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED,
284        ),
285    ))
286}
287
288fn ensure_decommission_ledger_persistence_supported(pool_meta: &PoolMeta) -> Result<()> {
289    ensure_decommission_ledger_persistence_supported_for(
290        pool_meta.version,
291        pool_meta_v2_writer_enabled(),
292        pool_meta_v3_writer_enabled(),
293    )
294}
295
296#[derive(Clone, Debug)]
297pub struct DecommissionCanceler {
298    operation: Arc<DecommissionOperation>,
299}
300
301#[derive(Debug)]
302struct DecommissionOperation {
303    token: CancellationToken,
304    active: AtomicBool,
305}
306
307impl DecommissionCanceler {
308    pub(crate) fn new(token: CancellationToken) -> Self {
309        Self {
310            operation: Arc::new(DecommissionOperation {
311                token,
312                active: AtomicBool::new(true),
313            }),
314        }
315    }
316
317    #[cfg(test)]
318    pub(crate) fn new_for_test(token: CancellationToken) -> Self {
319        Self::new(token)
320    }
321
322    fn token(&self) -> &CancellationToken {
323        &self.operation.token
324    }
325
326    pub(crate) fn is_active(&self) -> bool {
327        self.operation.active.load(Ordering::Acquire)
328    }
329
330    #[cfg(test)]
331    fn is_cancelled(&self) -> bool {
332        self.token().is_cancelled()
333    }
334
335    fn cancel(&self) {
336        self.token().cancel();
337    }
338
339    fn release(&self) {
340        self.cancel();
341        self.operation.active.store(false, Ordering::Release);
342    }
343
344    fn owns_same_operation(&self, other: &Self) -> bool {
345        Arc::ptr_eq(&self.operation, &other.operation)
346    }
347}
348
349struct DecommissionCancelerGuard {
350    canceler: DecommissionCanceler,
351}
352
353impl DecommissionCancelerGuard {
354    fn new(canceler: DecommissionCanceler) -> Self {
355        Self { canceler }
356    }
357
358    fn canceler(&self) -> &DecommissionCanceler {
359        &self.canceler
360    }
361}
362
363impl Drop for DecommissionCancelerGuard {
364    fn drop(&mut self) {
365        self.canceler.release();
366    }
367}
368
369fn dedup_indices(indices: &[usize]) -> Vec<usize> {
370    let mut seen = HashSet::with_capacity(indices.len());
371    let mut output = Vec::with_capacity(indices.len());
372    for idx in indices {
373        if seen.insert(*idx) {
374            output.push(*idx);
375        }
376    }
377
378    output
379}
380
381fn bind_decommission_cancelers(
382    indices: &[usize],
383    parent: &CancellationToken,
384    cancelers: &mut [Option<DecommissionCanceler>],
385) -> Vec<(usize, DecommissionCanceler)> {
386    let mut bound = Vec::with_capacity(indices.len());
387
388    for idx in indices {
389        if let Some(slot) = cancelers.get_mut(*idx) {
390            if let Some(existing) = slot.take() {
391                existing.release();
392            }
393            let canceler = DecommissionCanceler::new(parent.child_token());
394            *slot = Some(canceler.clone());
395            bound.push((*idx, canceler));
396        }
397    }
398
399    bound
400}
401
402fn bind_missing_decommission_cancelers(
403    indices: &[usize],
404    parent: &CancellationToken,
405    cancelers: &mut [Option<DecommissionCanceler>],
406) -> Vec<(usize, DecommissionCanceler)> {
407    let mut bound = Vec::with_capacity(indices.len());
408
409    for idx in indices {
410        let Some(slot) = cancelers.get_mut(*idx) else {
411            continue;
412        };
413        if slot.as_ref().is_some_and(DecommissionCanceler::is_active) {
414            break;
415        }
416        if let Some(stale) = slot.take() {
417            stale.release();
418        }
419        let canceler = DecommissionCanceler::new(parent.child_token());
420        *slot = Some(canceler.clone());
421        bound.push((*idx, canceler));
422    }
423
424    bound
425}
426
427fn take_decommission_canceler(cancelers: &mut [Option<DecommissionCanceler>], idx: usize) -> Option<DecommissionCanceler> {
428    cancelers.get_mut(idx).and_then(Option::take)
429}
430
431fn take_decommission_canceler_for_operation(
432    cancelers: &mut [Option<DecommissionCanceler>],
433    idx: usize,
434    owner: &DecommissionCanceler,
435) -> Option<DecommissionCanceler> {
436    let slot = cancelers.get_mut(idx)?;
437    if slot.as_ref().is_some_and(|canceler| canceler.owns_same_operation(owner)) {
438        slot.take()
439    } else {
440        None
441    }
442}
443
444fn decommission_canceler_is_owned_by(
445    cancelers: &[Option<DecommissionCanceler>],
446    idx: usize,
447    owner: &DecommissionCanceler,
448) -> bool {
449    cancelers
450        .get(idx)
451        .and_then(Option::as_ref)
452        .is_some_and(|canceler| canceler.owns_same_operation(owner))
453}
454
455fn update_decommission_for_operation<T>(
456    cancelers: &[Option<DecommissionCanceler>],
457    pool_meta: &mut PoolMeta,
458    idx: usize,
459    owner: Option<&DecommissionCanceler>,
460    update: impl FnOnce(&mut PoolMeta) -> T,
461) -> Option<T> {
462    if let Some(owner) = owner
463        && !decommission_canceler_is_owned_by(cancelers, idx, owner)
464    {
465        owner.release();
466        return None;
467    }
468
469    Some(update(pool_meta))
470}
471
472fn has_active_decommission_canceler(cancelers: &[Option<DecommissionCanceler>]) -> bool {
473    cancelers.iter().flatten().any(DecommissionCanceler::is_active)
474}
475
476fn cancel_decommission_canceler(canceler: Option<DecommissionCanceler>) -> bool {
477    if let Some(canceler) = canceler {
478        canceler.release();
479        true
480    } else {
481        false
482    }
483}
484
485fn take_and_cancel_decommission_canceler(cancelers: &mut [Option<DecommissionCanceler>], idx: usize) -> bool {
486    let canceler = take_decommission_canceler(cancelers, idx);
487    cancel_decommission_canceler(canceler)
488}
489
490fn take_and_cancel_decommission_canceler_for_operation(
491    cancelers: &mut [Option<DecommissionCanceler>],
492    idx: usize,
493    owner: &DecommissionCanceler,
494) -> bool {
495    let canceler = take_decommission_canceler_for_operation(cancelers, idx, owner);
496    if canceler.is_none() {
497        owner.release();
498        return false;
499    }
500    cancel_decommission_canceler(canceler)
501}
502
503fn ensure_decommission_routines_scheduled(bound_count: usize, expected_count: usize) -> Result<()> {
504    if bound_count == 0 || bound_count != expected_count {
505        return Err(Error::other(format!(
506            "failed to start decommission routines: scheduled {bound_count} of {expected_count} expected workers"
507        )));
508    }
509
510    Ok(())
511}
512
513fn guard_decommission_cancelers(index_cancelers: Vec<(usize, DecommissionCanceler)>) -> Vec<(usize, DecommissionCancelerGuard)> {
514    index_cancelers
515        .into_iter()
516        .map(|(idx, canceler)| (idx, DecommissionCancelerGuard::new(canceler)))
517        .collect()
518}
519
520async fn await_decommission_worker(idx: usize, worker: tokio::task::JoinHandle<Result<()>>) -> Result<()> {
521    worker
522        .await
523        .map_err(|err| Error::other(format!("decommission worker {idx} task join error: {err}")))?
524}
525
526fn reserve_decommission_start_cancelers(
527    pool_meta: &PoolMeta,
528    indices: &[usize],
529    local_indices: &[usize],
530    parent: &CancellationToken,
531    cancelers: &mut [Option<DecommissionCanceler>],
532) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
533    ensure_decommission_start_pool_states(pool_meta, indices)?;
534    if local_indices.is_empty() {
535        return Ok(Vec::new());
536    }
537    let bound = bind_decommission_cancelers(local_indices, parent, cancelers);
538    let guards = guard_decommission_cancelers(bound);
539    ensure_decommission_routines_scheduled(guards.len(), local_indices.len())?;
540    Ok(guards)
541}
542
543fn default_decommission_bucket_concurrency(cpu_count: usize) -> usize {
544    cpu_count.clamp(1, DECOMMISSION_BUCKET_CONCURRENCY_DEFAULT_CAP)
545}
546
547fn decommission_bucket_concurrency_limit() -> usize {
548    let default_limit = default_decommission_bucket_concurrency(num_cpus::get());
549    rustfs_utils::get_env_usize(DECOMMISSION_BUCKET_CONCURRENCY_ENV, default_limit).max(1)
550}
551
552fn default_decommission_entry_concurrency(cpu_count: usize) -> usize {
553    cpu_count.clamp(1, DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP)
554}
555
556fn clamp_decommission_entry_concurrency(limit: usize) -> usize {
557    limit.clamp(1, DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP)
558}
559
560fn decommission_entry_concurrency_limit() -> usize {
561    let default_limit = default_decommission_entry_concurrency(num_cpus::get());
562    clamp_decommission_entry_concurrency(rustfs_utils::get_env_usize(DECOMMISSION_ENTRY_CONCURRENCY_ENV, default_limit))
563}
564
565fn is_decommission_meta_bucket(bucket: &DecomBucketInfo) -> bool {
566    bucket.name == RUSTFS_META_BUCKET
567}
568
569fn decommission_meta_buckets() -> [DecomBucketInfo; DECOMMISSION_META_PREFIXES.len()] {
570    DECOMMISSION_META_PREFIXES.map(|prefix| DecomBucketInfo {
571        name: RUSTFS_META_BUCKET.to_owned(),
572        prefix: prefix.to_owned(),
573    })
574}
575
576fn reconcile_decommission_meta_buckets(meta: &mut PoolMeta, idx: usize) -> bool {
577    let before = meta.pending_buckets(idx).len();
578    meta.queue_buckets(idx, decommission_meta_buckets().into());
579    meta.pending_buckets(idx).len() != before
580}
581
582fn split_decommission_buckets(buckets: Vec<DecomBucketInfo>) -> (Vec<DecomBucketInfo>, Vec<DecomBucketInfo>) {
583    let mut regular = Vec::with_capacity(buckets.len());
584    let mut meta = Vec::new();
585
586    for bucket in buckets {
587        if is_decommission_meta_bucket(&bucket) {
588            meta.push(bucket);
589        } else {
590            regular.push(bucket);
591        }
592    }
593
594    regular.shrink_to_fit();
595    (regular, meta)
596}
597
598fn ensure_decommission_not_rebalancing(rebalance_running: bool) -> Result<()> {
599    if rebalance_running {
600        return Err(Error::RebalanceAlreadyRunning);
601    }
602
603    Ok(())
604}
605
606fn ensure_decommission_start_rebalance_meta_allowed(meta: Option<&RebalanceMeta>) -> Result<()> {
607    ensure_decommission_not_rebalancing(meta.is_some_and(is_rebalance_conflicting_with_decommission))
608}
609
610#[allow(dead_code, reason = "leader precondition asserted by this file's tests (backlog#1823)")]
611fn ensure_local_decommission_pool_leaders(endpoints: &EndpointServerPools, indices: &[usize]) -> Result<()> {
612    for idx in indices {
613        ensure_local_decommission_pool_leader(endpoints, *idx)?;
614    }
615
616    Ok(())
617}
618
619fn ensure_local_decommission_pool_leader(endpoints: &EndpointServerPools, idx: usize) -> Result<()> {
620    let pool = endpoints
621        .as_ref()
622        .get(idx)
623        .ok_or_else(|| invalid_decommission_pool_index_error(endpoints.as_ref().len(), idx))?;
624    let endpoint = pool
625        .endpoints
626        .as_ref()
627        .first()
628        .ok_or_else(|| Error::other(format!("decommission pool {idx} has no configured endpoints")))?;
629
630    if !endpoint.is_local {
631        return Err(Error::other(format!(
632            "decommission for pool {idx} must run on the pool first endpoint {endpoint}"
633        )));
634    }
635
636    Ok(())
637}
638
639fn decommission_pool_first_endpoint_is_local(endpoints: &EndpointServerPools, idx: usize) -> Result<bool> {
640    let pool = endpoints
641        .as_ref()
642        .get(idx)
643        .ok_or_else(|| invalid_decommission_pool_index_error(endpoints.as_ref().len(), idx))?;
644    let endpoint = pool
645        .endpoints
646        .as_ref()
647        .first()
648        .ok_or_else(|| Error::other(format!("decommission pool {idx} has no configured endpoints")))?;
649
650    Ok(endpoint.is_local)
651}
652
653pub(crate) fn local_decommission_queue_prefix(endpoints: &EndpointServerPools, indices: &[usize]) -> Result<Vec<usize>> {
654    let mut local = Vec::with_capacity(indices.len());
655
656    for idx in indices {
657        if decommission_pool_first_endpoint_is_local(endpoints, *idx)? {
658            local.push(*idx);
659        } else {
660            break;
661        }
662    }
663
664    Ok(local)
665}
666
667fn resumable_decommission_queue_indices(meta: &PoolMeta) -> Vec<usize> {
668    let mut indices = Vec::new();
669    for (idx, pool) in meta.pools.iter().enumerate() {
670        if let Some(decommission) = &pool.decommission {
671            if !decommission.has_decommission_state() {
672                continue;
673            }
674            if decommission.complete || decommission.failed || decommission.canceled {
675                continue;
676            }
677            indices.push(idx);
678        }
679    }
680
681    indices
682}
683
684fn missing_decommission_worker_prefix(indices: &[usize], cancelers: &[Option<DecommissionCanceler>]) -> Vec<usize> {
685    let mut missing = Vec::with_capacity(indices.len());
686
687    for idx in indices {
688        if cancelers
689            .get(*idx)
690            .and_then(Option::as_ref)
691            .is_some_and(DecommissionCanceler::is_active)
692        {
693            break;
694        }
695        missing.push(*idx);
696    }
697
698    missing
699}
700
701fn ensure_decommission_start_local_leader(endpoints: &EndpointServerPools, indices: &[usize]) -> Result<()> {
702    if let Some(first) = indices.first() {
703        ensure_local_decommission_pool_leader(endpoints, *first)?;
704    }
705
706    Ok(())
707}
708
709fn build_decommission_start_state(
710    pi: PoolSpaceInfo,
711    queued: bool,
712    now: OffsetDateTime,
713    previous: Option<&PoolDecommissionInfo>,
714) -> PoolDecommissionInfo {
715    let mut info = PoolDecommissionInfo {
716        start_time: if queued { None } else { Some(now) },
717        start_size: pi.free,
718        total_size: pi.total,
719        current_size: pi.free,
720        queued,
721        ..Default::default()
722    };
723
724    if let Some(previous) = previous
725        && (previous.failed || previous.canceled)
726    {
727        info.decommissioned_buckets = previous.decommissioned_buckets.clone();
728        info.items_decommissioned = previous.items_decommissioned;
729        info.bytes_done = previous.bytes_done;
730        info.unresolved_entries = previous.unresolved_entries.clone();
731        if let Some(generation) = info.start_time {
732            for entry in &mut info.unresolved_entries {
733                entry.source_generation = generation;
734            }
735        }
736        info.mark_progress_saved();
737    }
738
739    info
740}
741
742fn spawn_decommission_index_cancelers(
743    store: Arc<ECStore>,
744    rx: CancellationToken,
745    index_cancelers: Vec<(usize, DecommissionCancelerGuard)>,
746    entry_budget: Arc<Semaphore>,
747) -> tokio::task::JoinHandle<()> {
748    tokio::spawn(async move {
749        let mut stop_queue = false;
750
751        for (idx, canceler_guard) in index_cancelers {
752            let canceler = canceler_guard.canceler().clone();
753            if stop_queue || rx.is_cancelled() {
754                canceler.cancel();
755                store.retry_decommission_cancel_for_operation(idx, &canceler).await;
756                continue;
757            }
758
759            let worker = tokio::spawn({
760                let store = store.clone();
761                let canceler = canceler.clone();
762                let entry_budget = entry_budget.clone();
763                async move { store.do_decommission_in_routine(canceler, idx, entry_budget).await }
764            });
765            if let Err(err) = await_decommission_worker(idx, worker).await {
766                if is_decommission_capacity_blocked_error(&err) || is_decommission_target_capacity_error(&err) {
767                    warn!(
768                        event = EVENT_DECOMMISSION_STATE,
769                        component = LOG_COMPONENT_ECSTORE,
770                        subsystem = LOG_SUBSYSTEM_POOLS,
771                        pool_index = idx,
772                        state = "capacity_blocked",
773                        error = %err,
774                        "Decommission routine paused without a terminal transition"
775                    );
776                    store.release_decommission_canceler_slot(idx, &canceler).await;
777                    stop_queue = true;
778                    continue;
779                }
780                if let Err(blocked) = store
781                    .ensure_pool_meta_side_effects_safe("decommission paused because pool metadata requires recovery")
782                    .await
783                {
784                    warn!(
785                        event = EVENT_DECOMMISSION_STATE,
786                        component = LOG_COMPONENT_ECSTORE,
787                        subsystem = LOG_SUBSYSTEM_POOLS,
788                        pool_index = idx,
789                        state = "routine_blocked",
790                        error = %blocked,
791                        "Decommission routine paused without changing terminal state"
792                    );
793                    store.release_decommission_canceler_slot(idx, &canceler).await;
794                    stop_queue = true;
795                    continue;
796                }
797                error!(
798                    event = EVENT_DECOMMISSION_STATE,
799                    component = LOG_COMPONENT_ECSTORE,
800                    subsystem = LOG_SUBSYSTEM_POOLS,
801                    pool_index = idx,
802                    state = "routine_failed",
803                    error = %err,
804                    "Decommission routine failed"
805                );
806                store.quiesce_decommission_worker_after_join_error(&canceler).await;
807                store.retry_decommission_failed_for_operation(idx, &canceler).await;
808                stop_queue = true;
809                continue;
810            }
811
812            stop_queue = {
813                let pool_meta = store.pool_meta.read().await;
814                !should_continue_decommission_queue(&pool_meta, idx)
815            };
816        }
817    })
818}
819
820fn decommission_meta_bucket_options() -> MakeBucketOptions {
821    MakeBucketOptions {
822        force_create: true,
823        ..Default::default()
824    }
825}
826
827fn is_decommission_active(complete: bool, failed: bool, canceled: bool) -> bool {
828    !complete && !failed && !canceled
829}
830
831pub(crate) fn pool_meta_has_active_decommission(meta: &PoolMeta) -> bool {
832    meta.pools.iter().any(|pool| {
833        pool.decommission.as_ref().is_some_and(|info| {
834            info.has_decommission_state() && is_decommission_active(info.complete, info.failed, info.canceled)
835        })
836    })
837}
838
839fn is_decommission_suspended(info: &PoolDecommissionInfo) -> bool {
840    info.has_decommission_state()
841}
842
843fn validate_decommission_terminal_state(complete: bool, failed: bool, canceled: bool) -> Result<()> {
844    let terminal_count = [complete, failed, canceled].into_iter().filter(|terminal| *terminal).count();
845    if terminal_count > 1 {
846        return Err(Error::other(format!(
847            "pool metadata load failed: invalid decommission terminal state complete={complete} failed={failed} canceled={canceled}"
848        )));
849    }
850    Ok(())
851}
852
853fn invalid_decommission_pool_index_error(pool_count: usize, idx: usize) -> Error {
854    Error::other(format!("invalid decommission pool index {idx} for {pool_count} pools"))
855}
856
857#[derive(Debug, Clone, Copy, PartialEq, Eq)]
858enum DecommissionStartPoolState {
859    Missing,
860    Active,
861    Retryable,
862    Decommissioning,
863    Decommissioned,
864    Blocked,
865}
866
867fn decommission_start_pool_state(pool: Option<&PoolStatus>) -> DecommissionStartPoolState {
868    let Some(pool) = pool else {
869        return DecommissionStartPoolState::Missing;
870    };
871    let Some(info) = pool.decommission.as_ref() else {
872        return DecommissionStartPoolState::Active;
873    };
874    if !info.has_decommission_state() {
875        return DecommissionStartPoolState::Active;
876    }
877
878    if info.complete {
879        DecommissionStartPoolState::Decommissioned
880    } else if info.failed || info.canceled {
881        if info.unresolved_entries.is_empty() {
882            DecommissionStartPoolState::Blocked
883        } else {
884            DecommissionStartPoolState::Retryable
885        }
886    } else {
887        DecommissionStartPoolState::Decommissioning
888    }
889}
890
891fn is_decommission_start_active_pool(pool: &PoolStatus) -> bool {
892    decommission_start_pool_state(Some(pool)) == DecommissionStartPoolState::Active
893}
894
895fn invalid_decommission_request(reason: impl Into<String>) -> Error {
896    Error::InvalidArgument("decommission".to_string(), "pool-state".to_string(), reason.into())
897}
898
899fn ensure_decommission_start_allowed(state: DecommissionStartPoolState) -> Result<()> {
900    match state {
901        DecommissionStartPoolState::Missing => {
902            Err(invalid_decommission_request("failed to start decommission: target pool was not found"))
903        }
904        DecommissionStartPoolState::Active | DecommissionStartPoolState::Retryable => Ok(()),
905        DecommissionStartPoolState::Decommissioning => Err(StorageError::DecommissionAlreadyRunning),
906        DecommissionStartPoolState::Decommissioned => Err(invalid_decommission_request(
907            "failed to start decommission: target pool is already decommissioned",
908        )),
909        DecommissionStartPoolState::Blocked => Err(invalid_decommission_request(
910            "failed to start decommission: target pool decommission is blocked; clear failed or canceled metadata before starting again",
911        )),
912    }
913}
914
915fn ensure_decommission_start_keeps_active_pool(meta: &PoolMeta, indices: &[usize]) -> Result<()> {
916    let active_count = meta
917        .pools
918        .iter()
919        .filter(|pool| is_decommission_start_active_pool(pool))
920        .count();
921    let active_target_count = indices
922        .iter()
923        .filter(|idx| meta.pools.get(**idx).is_some_and(is_decommission_start_active_pool))
924        .count();
925    if active_count.saturating_sub(active_target_count) == 0 {
926        return Err(invalid_decommission_request(
927            "failed to start decommission: at least one active pool must remain after decommission start",
928        ));
929    }
930
931    Ok(())
932}
933
934fn ensure_decommission_start_pool_states(meta: &PoolMeta, indices: &[usize]) -> Result<()> {
935    for idx in indices.iter().copied() {
936        ensure_decommission_start_allowed(decommission_start_pool_state(meta.pools.get(idx)))?;
937    }
938    ensure_decommission_start_keeps_active_pool(meta, indices)
939}
940
941fn capacity_mul_div_ceil(value: usize, multiplier: usize, divisor: usize) -> usize {
942    if value == 0 || multiplier == 0 {
943        return 0;
944    }
945    if divisor == 0 {
946        return usize::MAX;
947    }
948    value
949        .checked_mul(multiplier)
950        .map(|product| product.div_ceil(divisor))
951        .unwrap_or(usize::MAX)
952}
953
954fn capacity_source_data_equivalent(physical_bytes: usize, layout: DecommissionErasureLayout) -> Result<usize> {
955    if !layout.is_valid() {
956        return Err(Error::DecommissionCapacity(format!(
957            "failed to model decommission capacity: invalid source layout data={} parity={}",
958            layout.data, layout.parity
959        )));
960    }
961    Ok(capacity_mul_div_ceil(physical_bytes, layout.data, layout.width()))
962}
963
964fn capacity_target_physical_bytes(data_bytes: usize, layout: DecommissionErasureLayout) -> Result<usize> {
965    if !layout.is_valid() {
966        return Err(Error::DecommissionCapacity(format!(
967            "failed to model decommission capacity: invalid target layout data={} parity={}",
968            layout.data, layout.parity
969        )));
970    }
971    Ok(capacity_mul_div_ceil(data_bytes, layout.width(), layout.data))
972}
973
974async fn decommission_owned_physical_usage(sets: &Sets, layout: DecommissionErasureLayout) -> HashMap<i32, usize> {
975    let mut usage = HashMap::with_capacity(sets.disk_set.len());
976    for (set_index, set) in sets.disk_set.iter().enumerate() {
977        let Ok(cache) = load_data_usage_cache(set.as_ref(), DATA_USAGE_CACHE_NAME).await else {
978            continue;
979        };
980        let data_usage = cache.dui(DATA_USAGE_ROOT, &[]);
981        if !data_usage.is_complete_bucket_usage_snapshot() {
982            continue;
983        }
984        let Ok(logical_bytes) = usize::try_from(data_usage.objects_total_size) else {
985            continue;
986        };
987        let Ok(set_index) = i32::try_from(set_index) else {
988            continue;
989        };
990        // The data-usage snapshot excludes unrelated files on a shared mount.
991        // Convert logical object bytes to the source set's physical EC footprint.
992        usage.insert(set_index, capacity_mul_div_ceil(logical_bytes, layout.width(), layout.data));
993    }
994    usage
995}
996
997fn worst_decommission_target_layout(targets: &[DecommissionPoolCapacityInfo]) -> Result<DecommissionErasureLayout> {
998    targets
999        .iter()
1000        .map(|target| target.layout)
1001        .filter(|layout| layout.is_valid())
1002        .max_by(|left, right| {
1003            ((left.width() as u128) * (right.data as u128)).cmp(&((right.width() as u128) * (left.data as u128)))
1004        })
1005        .ok_or_else(|| Error::other("failed to start decommission: no valid target erasure layout is available"))
1006}
1007
1008fn decommission_capacity_writer_supported(meta: &PoolMeta) -> bool {
1009    decommission_capacity_writer_supported_for(meta.version, pool_meta_v2_writer_enabled(), pool_meta_v3_writer_enabled())
1010}
1011
1012fn ensure_decommission_capacity_writer_supported(meta: &PoolMeta) -> Result<()> {
1013    if decommission_capacity_writer_supported(meta) {
1014        return Ok(());
1015    }
1016    Err(Error::DecommissionCapacity(format!(
1017        "failed to start decommission: durable capacity reservations require pool metadata V2 or V3; enable either the {} + {} V2 gate or the {} + {} V3 gate only after every reader and writer supports that format",
1018        rustfs_config::ENV_POOL_META_V2_WRITE,
1019        rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
1020        rustfs_config::ENV_POOL_META_V3_WRITE,
1021        rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED,
1022    )))
1023}
1024
1025fn decommission_capacity_blocked_error(message: impl Display) -> Error {
1026    Error::DecommissionCapacityBlocked {
1027        message: message.to_string(),
1028    }
1029}
1030
1031fn is_decommission_capacity_blocked_error(err: &Error) -> bool {
1032    if matches!(err, Error::DecommissionCapacityBlocked { .. }) {
1033        return true;
1034    }
1035    data_movement::data_movement_stage_source(err).is_some_and(is_decommission_capacity_blocked_error)
1036}
1037
1038fn is_decommission_capacity_intent_conflict(err: &Error) -> bool {
1039    if let Error::DecommissionCapacityBlocked { message } = err {
1040        return message.contains("unresolved target capacity intent")
1041            || message.contains("pending capacity intent belongs to another mutation");
1042    }
1043    data_movement::data_movement_stage_source(err).is_some_and(is_decommission_capacity_intent_conflict)
1044        || err.to_string().contains("unresolved target capacity intent")
1045}
1046
1047pub(crate) fn decommission_capacity_target_gate_busy_index(err: &Error) -> Option<usize> {
1048    if let Error::DecommissionCapacityBlocked { message } = err {
1049        return message
1050            .strip_prefix(DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX)?
1051            .strip_suffix(DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX)?
1052            .parse()
1053            .ok();
1054    }
1055    data_movement::data_movement_stage_source(err).and_then(decommission_capacity_target_gate_busy_index)
1056}
1057
1058pub(crate) fn is_decommission_capacity_target_gate_busy(err: &Error) -> bool {
1059    decommission_capacity_target_gate_busy_index(err).is_some()
1060}
1061
1062/// How the decommission capacity path must treat a failure.
1063///
1064/// Callers decide between retrying, ignoring, and propagating from this single
1065/// classification so the same failure shape cannot be treated differently at
1066/// different call sites.
1067#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1068pub(crate) enum DecommissionCapacityFailure {
1069    /// The target pool's capacity mutation gate is held by another mutation.
1070    /// Transient: the mutation can be retried after a short backoff.
1071    TargetGateBusy { target_pool_index: usize },
1072    /// Another mutation's durable target capacity intent is still unresolved.
1073    /// Transient, but converged on a different retry budget than gate contention.
1074    IntentConflict,
1075    /// The mutation is no longer needed: the object or version disappeared, or
1076    /// an equivalent target already exists. Never counted as a failure.
1077    BenignContention,
1078    /// A real failure that must propagate to the caller.
1079    Fatal,
1080}
1081
1082/// Predicate for the benign-contention class.
1083///
1084/// Only identity/overwrite shapes belong here: a missing object or version, an
1085/// equivalent target that already published the mutation, and an upload id that
1086/// a competing mutation already superseded. Everything else must stay fatal so a
1087/// genuine migration failure cannot be silently dropped.
1088fn is_decommission_benign_contention_error(err: &Error) -> bool {
1089    if is_decommission_copy_cleanup_safe_error(err) || is_err_data_movement_overwrite(err) || is_err_invalid_upload_id(err) {
1090        return true;
1091    }
1092    data_movement::data_movement_stage_source(err).is_some_and(is_decommission_benign_contention_error)
1093}
1094
1095fn classify_decommission_capacity_failure(err: &Error) -> DecommissionCapacityFailure {
1096    if let Some(target_pool_index) = decommission_capacity_target_gate_busy_index(err) {
1097        return DecommissionCapacityFailure::TargetGateBusy { target_pool_index };
1098    }
1099    if is_decommission_capacity_intent_conflict(err) {
1100        return DecommissionCapacityFailure::IntentConflict;
1101    }
1102    if is_decommission_benign_contention_error(err) {
1103        return DecommissionCapacityFailure::BenignContention;
1104    }
1105    DecommissionCapacityFailure::Fatal
1106}
1107
1108#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1109enum DecommissionCapacityRetryKind {
1110    IntentConflict,
1111}
1112
1113fn decommission_capacity_retry_kind(err: &Error, intent_conflict_attempt: usize) -> Option<DecommissionCapacityRetryKind> {
1114    (intent_conflict_attempt < DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS
1115        && matches!(classify_decommission_capacity_failure(err), DecommissionCapacityFailure::IntentConflict))
1116    .then_some(DecommissionCapacityRetryKind::IntentConflict)
1117}
1118
1119#[derive(Debug, Clone, Copy, PartialEq, Eq)]
1120enum DecommissionScannerBacklogRetryKind {
1121    HandoffConflict,
1122}
1123
1124fn is_decommission_scanner_backlog_handoff_conflict(err: &Error) -> bool {
1125    data_movement::data_movement_stage_source_as::<data_movement::scanner_backlog::ScannerPauseBacklogRetirementError>(err)
1126        .is_some_and(data_movement::scanner_backlog::ScannerPauseBacklogRetirementError::is_retryable_handoff_conflict)
1127}
1128
1129/// Classify a retryable scanner pause backlog handoff conflict.
1130///
1131/// The typed planner error is the only signal consulted: string matching would
1132/// turn unrelated retirement failures into retries. `handoff_conflict_attempt`
1133/// is the number of retries already consumed by this entry.
1134fn decommission_scanner_backlog_retry_kind(
1135    err: &Error,
1136    handoff_conflict_attempt: usize,
1137) -> Option<DecommissionScannerBacklogRetryKind> {
1138    (handoff_conflict_attempt < DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS
1139        && is_decommission_scanner_backlog_handoff_conflict(err))
1140    .then_some(DecommissionScannerBacklogRetryKind::HandoffConflict)
1141}
1142
1143fn ensure_decommission_capacity_target_fence(
1144    guard: &rustfs_lock::NamespaceLockGuard,
1145    target_pool_index: usize,
1146    phase: &str,
1147) -> Result<()> {
1148    if guard.is_lock_lost() {
1149        return Err(Error::DecommissionCapacity(format!(
1150            "target pool {target_pool_index} capacity mutation fence was lost during {phase}"
1151        )));
1152    }
1153    Ok(())
1154}
1155
1156fn ensure_decommission_capacity_mutation_intent_current(
1157    meta: &PoolMeta,
1158    owner: DecommissionCapacityOwner,
1159    target_pool_index: usize,
1160    expected_target_physical_bytes: usize,
1161    mutation_id: uuid::Uuid,
1162    temporary_release: bool,
1163    model_version: u16,
1164) -> Result<()> {
1165    let reservation = meta
1166        .pools
1167        .get(owner.source_pool_index)
1168        .and_then(|pool| pool.decommission.as_ref())
1169        .and_then(|info| info.capacity_reservation.as_ref())
1170        .filter(|reservation| {
1171            // Admission validated the owner nonce before persisting this
1172            // intent. The target lock is the mutation fence after that point,
1173            // so a concurrent lease renewal may rotate the nonce without
1174            // invalidating the already-durable intent.
1175            reservation.active()
1176                && reservation.source_pool_index == owner.source_pool_index
1177                && reservation.operation_id == owner.operation_id
1178                && reservation.generation == owner.generation
1179                && reservation.model_version == model_version
1180        })
1181        .ok_or_else(|| {
1182            decommission_capacity_blocked_error("decommission target mutation owner changed before capacity finalize")
1183        })?;
1184    let target = reservation
1185        .targets
1186        .iter()
1187        .find(|target| target.pool_index == target_pool_index)
1188        .ok_or_else(|| decommission_capacity_blocked_error("decommission target allocation changed before capacity finalize"))?;
1189
1190    if temporary_release {
1191        // Cleanup is authorized by the persisted owner and target identity.
1192        // Its exact temporary or pending record may already be absent after a
1193        // prior successful finalize, so attribution is decided from the
1194        // physical cleanup result below instead of rejecting the retry here.
1195        return Ok(());
1196    }
1197
1198    if expected_target_physical_bytes == 0
1199        || (target.pending_mutation_id == Some(mutation_id) && target.pending_physical_bytes >= expected_target_physical_bytes)
1200    {
1201        return Ok(());
1202    }
1203    Err(decommission_capacity_blocked_error(
1204        "pending capacity intent belongs to another mutation before capacity finalize",
1205    ))
1206}
1207
1208fn validate_decommission_capacity_reservation(reservation: Option<&DecommissionCapacityReservation>) -> Result<()> {
1209    let Some(reservation) = reservation else {
1210        return Ok(());
1211    };
1212    if !matches!(
1213        reservation.model_version,
1214        DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
1215    ) {
1216        return Err(Error::DecommissionCapacity(format!(
1217            "pool metadata load failed: unsupported decommission capacity model version {}",
1218            reservation.model_version
1219        )));
1220    }
1221    if reservation.operation_id.is_nil() || reservation.generation == 0 || reservation.owner_nonce.is_nil() {
1222        return Err(Error::other(
1223            "pool metadata load failed: decommission capacity reservation identity is invalid",
1224        ));
1225    }
1226    if !reservation.source_layout.is_valid() || reservation.targets.iter().any(|target| !target.layout.is_valid()) {
1227        return Err(Error::other(
1228            "pool metadata load failed: decommission capacity reservation contains an invalid erasure layout",
1229        ));
1230    }
1231    if reservation.temporary_copies != DECOMMISSION_CAPACITY_TEMPORARY_COPIES {
1232        return Err(Error::other(
1233            "pool metadata load failed: decommission capacity reservation temporary-copy model is invalid",
1234        ));
1235    }
1236    if reservation.source_data_equivalent_bytes
1237        != capacity_source_data_equivalent(reservation.source_physical_bytes, reservation.source_layout)?
1238    {
1239        return Err(Error::other(
1240            "pool metadata load failed: decommission capacity reservation source estimate is invalid",
1241        ));
1242    }
1243    if reservation.peak_physical_bytes
1244        != reservation
1245            .predicted_physical_bytes
1246            .saturating_add(reservation.temporary_physical_bytes)
1247        || reservation.temporary_physical_bytes
1248            != reservation
1249                .predicted_physical_bytes
1250                .saturating_mul(reservation.temporary_copies)
1251    {
1252        return Err(Error::other(
1253            "pool metadata load failed: decommission capacity reservation peak does not match its physical model",
1254        ));
1255    }
1256    if reservation.committed_data_bytes > reservation.source_data_equivalent_bytes {
1257        return Err(Error::other(
1258            "pool metadata load failed: decommission capacity committed bytes exceed the source estimate",
1259        ));
1260    }
1261    if reservation.consumed_target_physical_bytes > reservation.predicted_physical_bytes {
1262        return Err(Error::other(
1263            "pool metadata load failed: decommission capacity consumed target bytes exceed the prediction",
1264        ));
1265    }
1266    let mut target_indices = HashSet::with_capacity(reservation.targets.len());
1267    let mut reserved_physical_bytes = 0usize;
1268    let mut consumed_physical_bytes = 0usize;
1269    let mut observed_physical_bytes = 0usize;
1270    let mut inflight_physical_bytes = 0usize;
1271    let mut pending_physical_bytes = 0usize;
1272    for target in &reservation.targets {
1273        let mut temporary_mutation_ids = HashSet::with_capacity(target.temporary_mutations.len());
1274        let temporary_mutation_bytes = target.temporary_mutations.iter().try_fold(0usize, |total, mutation| {
1275            if mutation.mutation_id.is_nil()
1276                || (mutation.physical_bytes == 0 && reservation.model_version != DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION)
1277                || !temporary_mutation_ids.insert(mutation.mutation_id)
1278            {
1279                return Err(Error::other(
1280                    "pool metadata load failed: decommission capacity temporary mutation is invalid",
1281                ));
1282            }
1283            Ok(total.saturating_add(mutation.physical_bytes))
1284        })?;
1285        if target.pool_index == reservation.source_pool_index
1286            || !target_indices.insert(target.pool_index)
1287            || target.reserved_physical_bytes == 0
1288            || target.reserved_physical_bytes > target.physical_free_at_reservation
1289            || target.physical_free_at_reservation > target.physical_total_at_reservation
1290            || target.consumed_physical_bytes > reservation.consumed_target_physical_bytes
1291            || target.observed_physical_bytes > reservation.observed_target_physical_bytes
1292            || target.inflight_physical_bytes > target.observed_physical_bytes
1293            || temporary_mutation_bytes > target.inflight_physical_bytes
1294        {
1295            return Err(Error::other(
1296                "pool metadata load failed: decommission capacity reservation target allocation is invalid",
1297            ));
1298        }
1299        if reservation.predicted_physical_bytes
1300            < capacity_target_physical_bytes(reservation.source_data_equivalent_bytes, target.layout)?
1301        {
1302            return Err(Error::other(
1303                "pool metadata load failed: decommission capacity reservation target estimate is unsafe",
1304            ));
1305        }
1306        reserved_physical_bytes =
1307            reserved_physical_bytes.saturating_add(target.remaining_reserved_physical_bytes(reservation.temporary_copies));
1308        consumed_physical_bytes = consumed_physical_bytes.saturating_add(target.consumed_physical_bytes);
1309        observed_physical_bytes = observed_physical_bytes.saturating_add(target.observed_physical_bytes);
1310        inflight_physical_bytes = inflight_physical_bytes.saturating_add(target.inflight_physical_bytes);
1311        pending_physical_bytes = pending_physical_bytes.saturating_add(target.pending_physical_bytes);
1312    }
1313    if reserved_physical_bytes != reservation.remaining_peak_physical_bytes() {
1314        return Err(Error::other(
1315            "pool metadata load failed: decommission capacity reservation target allocation is incomplete",
1316        ));
1317    }
1318    if inflight_physical_bytes != reservation.inflight_target_physical_bytes {
1319        return Err(Error::other(
1320            "pool metadata load failed: decommission capacity inflight target accounting is incomplete",
1321        ));
1322    }
1323    if pending_physical_bytes != reservation.pending_target_physical_bytes {
1324        return Err(Error::other(
1325            "pool metadata load failed: decommission capacity pending target accounting is incomplete",
1326        ));
1327    }
1328    if consumed_physical_bytes != reservation.consumed_target_physical_bytes
1329        || observed_physical_bytes != reservation.observed_target_physical_bytes
1330    {
1331        return Err(Error::other(
1332            "pool metadata load failed: decommission capacity target progress accounting is incomplete",
1333        ));
1334    }
1335    if reservation.released_at.is_some() != reservation.release_reason.is_some() {
1336        return Err(Error::other(
1337            "pool metadata load failed: decommission capacity reservation release state is incomplete",
1338        ));
1339    }
1340    Ok(())
1341}
1342
1343fn active_decommission_capacity_model(meta: &PoolMeta) -> Result<Option<u16>> {
1344    let mut active_model = None;
1345    for reservation in meta
1346        .pools
1347        .iter()
1348        .filter_map(|pool| pool.decommission.as_ref()?.capacity_reservation.as_ref())
1349        .filter(|reservation| reservation.active())
1350    {
1351        if !matches!(
1352            reservation.model_version,
1353            DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
1354        ) {
1355            return Err(Error::DecommissionCapacity(format!(
1356                "pool metadata load failed: unsupported active decommission capacity model version {}",
1357                reservation.model_version
1358            )));
1359        }
1360        match active_model {
1361            Some(model_version) if model_version != reservation.model_version => {
1362                return Err(Error::DecommissionCapacity(
1363                    "pool metadata load failed: active decommission capacity reservations use mixed lock models".to_string(),
1364                ));
1365            }
1366            Some(_) => {}
1367            None => active_model = Some(reservation.model_version),
1368        }
1369    }
1370    Ok(active_model)
1371}
1372
1373fn validate_decommission_capacity_model_cohort(meta: &PoolMeta) -> Result<()> {
1374    active_decommission_capacity_model(meta).map(|_| ())
1375}
1376
1377fn select_decommission_capacity_model(meta: &PoolMeta, target_fence_proof_available: bool) -> Result<u16> {
1378    match active_decommission_capacity_model(meta)? {
1379        Some(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION) => Ok(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION),
1380        Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) if target_fence_proof_available => {
1381            Ok(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION)
1382        }
1383        Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) => Err(Error::DecommissionCapacity(
1384            "failed to start decommission: an active per-target capacity cohort requires a live all-v4 fleet proof".to_string(),
1385        )),
1386        Some(version) => Err(Error::DecommissionCapacity(format!(
1387            "failed to start decommission: unsupported active capacity model version {version}"
1388        ))),
1389        None if target_fence_proof_available => Ok(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION),
1390        None => Ok(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION),
1391    }
1392}
1393
1394fn ensure_decommission_target_fence_fleet_proof(
1395    proof: Option<&crate::services::notification_sys::DecommissionTargetFenceFleetProofToken>,
1396    required: bool,
1397) -> Result<()> {
1398    if !required {
1399        return Ok(());
1400    }
1401    if proof.is_some_and(crate::services::notification_sys::decommission_target_fence_fleet_proof_matches) {
1402        return Ok(());
1403    }
1404    Err(Error::other(DECOMMISSION_TARGET_FLEET_PROOF_EXPIRED))
1405}
1406
1407fn release_decommission_capacity_reservation(info: &mut PoolDecommissionInfo, reason: &str, now: OffsetDateTime) -> bool {
1408    let Some(reservation) = info.capacity_reservation.as_mut() else {
1409        return false;
1410    };
1411    if !reservation.active() {
1412        return false;
1413    }
1414    reservation.released_at = Some(now);
1415    reservation.release_reason = Some(reason.to_string());
1416    metrics::gauge!(
1417        METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES,
1418        "pool_index" => reservation.source_pool_index.to_string()
1419    )
1420    .set(0.0);
1421    true
1422}
1423
1424fn renew_decommission_capacity_reservation(
1425    reservation: &mut DecommissionCapacityReservation,
1426    now: OffsetDateTime,
1427    recover_expired: bool,
1428) -> bool {
1429    if !reservation.active() {
1430        return false;
1431    }
1432    let expired = reservation.expires_at <= now;
1433    if expired && recover_expired {
1434        reservation.owner_nonce = uuid::Uuid::new_v4();
1435        reservation.recovered_at = Some(now);
1436    }
1437    reservation.renewed_at = now;
1438    reservation.expires_at = now + DECOMMISSION_CAPACITY_RESERVATION_TTL;
1439    true
1440}
1441
1442fn next_decommission_capacity_generation(meta: &PoolMeta) -> Result<u64> {
1443    meta.pools
1444        .iter()
1445        .filter_map(|pool| pool.decommission.as_ref()?.capacity_reservation.as_ref())
1446        .map(|reservation| reservation.generation)
1447        .max()
1448        .unwrap_or_default()
1449        .checked_add(1)
1450        .ok_or_else(|| Error::other("failed to start decommission: capacity reservation generation overflow"))
1451}
1452
1453fn active_decommission_source_indices(meta: &PoolMeta) -> HashSet<usize> {
1454    meta.pools
1455        .iter()
1456        .enumerate()
1457        .filter_map(|(idx, pool)| {
1458            pool.decommission.as_ref().and_then(|info| {
1459                (info.has_decommission_state() && is_decommission_active(info.complete, info.failed, info.canceled))
1460                    .then_some(idx)
1461            })
1462        })
1463        .collect()
1464}
1465
1466#[cfg(test)]
1467fn build_decommission_capacity_reservation(
1468    source: DecommissionPoolCapacityInfo,
1469    target_layout: DecommissionErasureLayout,
1470    operation_id: uuid::Uuid,
1471    generation: u64,
1472    now: OffsetDateTime,
1473) -> Result<DecommissionCapacityReservation> {
1474    build_decommission_capacity_reservation_with_model(
1475        source,
1476        target_layout,
1477        operation_id,
1478        generation,
1479        now,
1480        DECOMMISSION_CAPACITY_MODEL_VERSION,
1481    )
1482}
1483
1484fn build_decommission_capacity_reservation_with_model(
1485    source: DecommissionPoolCapacityInfo,
1486    target_layout: DecommissionErasureLayout,
1487    operation_id: uuid::Uuid,
1488    generation: u64,
1489    now: OffsetDateTime,
1490    model_version: u16,
1491) -> Result<DecommissionCapacityReservation> {
1492    if !matches!(
1493        model_version,
1494        DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
1495    ) {
1496        return Err(Error::DecommissionCapacity(format!(
1497            "failed to build decommission capacity reservation: unsupported lock model {model_version}"
1498        )));
1499    }
1500    let source_data_equivalent_bytes = capacity_source_data_equivalent(source.physical_used, source.layout)?;
1501    let predicted_physical_bytes = capacity_target_physical_bytes(source_data_equivalent_bytes, target_layout)?;
1502    let temporary_physical_bytes = predicted_physical_bytes.saturating_mul(DECOMMISSION_CAPACITY_TEMPORARY_COPIES);
1503    Ok(DecommissionCapacityReservation {
1504        model_version,
1505        operation_id,
1506        generation,
1507        owner_nonce: uuid::Uuid::new_v4(),
1508        source_pool_index: source.pool_index,
1509        source_layout: source.layout,
1510        source_physical_total_bytes: source.physical_total,
1511        source_physical_bytes: source.physical_used,
1512        source_data_equivalent_bytes,
1513        predicted_physical_bytes,
1514        temporary_copies: DECOMMISSION_CAPACITY_TEMPORARY_COPIES,
1515        temporary_physical_bytes,
1516        peak_physical_bytes: predicted_physical_bytes.saturating_add(temporary_physical_bytes),
1517        committed_data_bytes: 0,
1518        consumed_target_physical_bytes: 0,
1519        observed_target_physical_bytes: 0,
1520        inflight_target_physical_bytes: 0,
1521        pending_target_physical_bytes: 0,
1522        prediction_error_bytes: 0,
1523        targets: Vec::new(),
1524        created_at: now,
1525        renewed_at: now,
1526        expires_at: now + DECOMMISSION_CAPACITY_RESERVATION_TTL,
1527        recovered_at: None,
1528        released_at: None,
1529        release_reason: None,
1530    })
1531}
1532
1533fn reserve_decommission_start_target_capacity(
1534    meta: &mut PoolMeta,
1535    requested_indices: &[usize],
1536    capacity_infos: &[DecommissionPoolCapacityInfo],
1537    operation_id: uuid::Uuid,
1538    generation: u64,
1539    now: OffsetDateTime,
1540    model_version: u16,
1541) -> Result<()> {
1542    validate_decommission_capacity_model_cohort(meta)?;
1543    let active_sources = active_decommission_source_indices(meta);
1544    let targets = capacity_infos
1545        .iter()
1546        .copied()
1547        .filter(|capacity| {
1548            !active_sources.contains(&capacity.pool_index)
1549                && meta
1550                    .pools
1551                    .get(capacity.pool_index)
1552                    .is_some_and(is_decommission_start_active_pool)
1553        })
1554        .collect::<Vec<_>>();
1555    let target_layout = worst_decommission_target_layout(&targets)?;
1556    let requested = requested_indices.iter().copied().collect::<HashSet<_>>();
1557    let mut reservations = Vec::with_capacity(active_sources.len());
1558
1559    let mut source_indices = active_sources.into_iter().collect::<Vec<_>>();
1560    source_indices.sort_unstable();
1561    for source_index in source_indices {
1562        let source = capacity_infos
1563            .iter()
1564            .copied()
1565            .find(|capacity| capacity.pool_index == source_index)
1566            .ok_or_else(|| {
1567                Error::DecommissionCapacity(format!(
1568                    "failed to start decommission: capacity snapshot is missing source pool {source_index}"
1569                ))
1570            })?;
1571        let pool = meta
1572            .pools
1573            .get(source_index)
1574            .ok_or_else(|| invalid_decommission_pool_index_error(meta.pools.len(), source_index))?;
1575        let info = pool
1576            .decommission
1577            .as_ref()
1578            .ok_or_else(|| decommission_metadata_not_initialized_error("reserve decommission capacity"))?;
1579        let new_reservation = requested.contains(&source_index);
1580        let mut reservation = if new_reservation {
1581            build_decommission_capacity_reservation_with_model(
1582                source,
1583                target_layout,
1584                operation_id,
1585                generation,
1586                now,
1587                model_version,
1588            )?
1589        } else {
1590            info.capacity_reservation.clone().ok_or_else(|| {
1591                Error::DecommissionCapacity(format!(
1592                    "failed to start decommission: active source pool {source_index} has no durable capacity reservation"
1593                ))
1594            })?
1595        };
1596        if !reservation.active() {
1597            return Err(Error::DecommissionCapacity(format!(
1598                "failed to start decommission: active source pool {source_index} has a released capacity reservation"
1599            )));
1600        }
1601        renew_decommission_capacity_reservation(&mut reservation, now, true);
1602        reservations.push((source_index, reservation, new_reservation));
1603    }
1604
1605    let mut remaining_target_capacity = targets
1606        .iter()
1607        .map(|target| (target.pool_index, target.physical_free))
1608        .collect::<HashMap<_, _>>();
1609    for (_, reservation, new_reservation) in &reservations {
1610        if *new_reservation {
1611            continue;
1612        }
1613        for target in &reservation.targets {
1614            let available = remaining_target_capacity.entry(target.pool_index).or_default();
1615            *available = available
1616                .saturating_add(target.inflight_physical_bytes)
1617                .saturating_add(target.pending_physical_bytes);
1618            let required = target.remaining_reserved_physical_bytes(reservation.temporary_copies);
1619            if *available < required {
1620                return Err(decommission_capacity_blocked_error(format!(
1621                    "failed to start decommission: existing reservation for source pool {} requires {required} bytes on target pool {}, but only {} bytes remain",
1622                    reservation.source_pool_index, target.pool_index, *available
1623                )));
1624            }
1625            *available -= required;
1626        }
1627    }
1628    let required = reservations
1629        .iter()
1630        .filter(|(_, _, new_reservation)| *new_reservation)
1631        .fold(0usize, |total, (_, reservation, _)| {
1632            total.saturating_add(reservation.remaining_peak_physical_bytes())
1633        });
1634    let available = remaining_target_capacity
1635        .values()
1636        .copied()
1637        .fold(0usize, usize::saturating_add);
1638    if available < required {
1639        metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => "activation").increment(1);
1640        return Err(decommission_capacity_blocked_error(format!(
1641            "failed to start decommission: insufficient reserved physical target capacity: operation {operation_id} generation {generation} requires {required} bytes, but {available} bytes are available after source/target parity and temporary-copy accounting"
1642        )));
1643    }
1644
1645    for (_, reservation, new_reservation) in &mut reservations {
1646        if !*new_reservation {
1647            continue;
1648        }
1649        let mut remaining = reservation.remaining_peak_physical_bytes();
1650        for target in &targets {
1651            if remaining == 0 {
1652                break;
1653            }
1654            let target_remaining = remaining_target_capacity.entry(target.pool_index).or_default();
1655            let allocated = remaining.min(*target_remaining);
1656            if allocated == 0 {
1657                continue;
1658            }
1659            *target_remaining -= allocated;
1660            remaining -= allocated;
1661            reservation.targets.push(DecommissionCapacityTarget {
1662                pool_index: target.pool_index,
1663                layout: target.layout,
1664                physical_total_at_reservation: target.physical_total,
1665                physical_free_at_reservation: target.physical_free,
1666                reserved_physical_bytes: allocated,
1667                consumed_physical_bytes: 0,
1668                observed_physical_bytes: 0,
1669                inflight_physical_bytes: 0,
1670                pending_physical_bytes: 0,
1671                pending_mutation_id: None,
1672                temporary_mutations: Vec::new(),
1673            });
1674        }
1675        debug_assert_eq!(remaining, 0);
1676    }
1677
1678    for (source_index, reservation, _) in reservations {
1679        metrics::gauge!(
1680            METRIC_DECOMMISSION_CAPACITY_PREDICTED_BYTES,
1681            "pool_index" => source_index.to_string()
1682        )
1683        .set(reservation.predicted_physical_bytes as f64);
1684        metrics::gauge!(
1685            METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES,
1686            "pool_index" => source_index.to_string()
1687        )
1688        .set(reservation.remaining_peak_physical_bytes() as f64);
1689        let pool_count = meta.pools.len();
1690        let info = meta
1691            .pools
1692            .get_mut(source_index)
1693            .and_then(|pool| pool.decommission.as_mut())
1694            .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_index))?;
1695        info.capacity_blocked_reason = None;
1696        info.capacity_reservation = Some(reservation);
1697    }
1698    if meta.version != POOL_META_GENERATION_VERSION {
1699        meta.version = POOL_META_VERSION;
1700    }
1701    validate_decommission_capacity_model_cohort(meta)?;
1702    Ok(())
1703}
1704
1705fn ensure_decommission_start_target_capacity(
1706    meta: &PoolMeta,
1707    indices: &[usize],
1708    capacity_infos: &[DecommissionPoolCapacityInfo],
1709    target_fence_proof_available: bool,
1710) -> Result<()> {
1711    let generation = next_decommission_capacity_generation(meta)?;
1712    let mut projected = meta.clone();
1713    let first_idx = indices.first().copied();
1714    for idx in indices.iter().copied() {
1715        let capacity = capacity_infos
1716            .iter()
1717            .find(|capacity| capacity.pool_index == idx)
1718            .ok_or_else(|| Error::DecommissionCapacity(format!("decommission capacity snapshot is missing pool {idx}")))?;
1719        if Some(idx) == first_idx {
1720            projected.decommission(idx, capacity.space)?;
1721        } else {
1722            projected.queue_decommission(idx, capacity.space)?;
1723        }
1724    }
1725    let model_version = select_decommission_capacity_model(&projected, target_fence_proof_available)?;
1726    reserve_decommission_start_target_capacity(
1727        &mut projected,
1728        indices,
1729        capacity_infos,
1730        uuid::Uuid::new_v4(),
1731        generation,
1732        OffsetDateTime::now_utc(),
1733        model_version,
1734    )
1735}
1736
1737fn recover_decommission_capacity_reservations(
1738    meta: &mut PoolMeta,
1739    capacity_infos: &[DecommissionPoolCapacityInfo],
1740    now: OffsetDateTime,
1741    target_fence_proof_available: bool,
1742) -> Result<Vec<usize>> {
1743    ensure_decommission_capacity_writer_supported(meta)?;
1744    let mut active_indices = active_decommission_source_indices(meta).into_iter().collect::<Vec<_>>();
1745    active_indices.sort_unstable();
1746    let missing_indices = active_indices
1747        .iter()
1748        .copied()
1749        .filter(|idx| {
1750            meta.pools
1751                .get(*idx)
1752                .and_then(|pool| pool.decommission.as_ref())
1753                .and_then(|info| info.capacity_reservation.as_ref())
1754                .is_none_or(|reservation| !reservation.active())
1755        })
1756        .collect::<Vec<_>>();
1757    if missing_indices.is_empty() {
1758        validate_decommission_capacity_model_cohort(meta)?;
1759        return Ok(active_indices);
1760    }
1761    let generation = next_decommission_capacity_generation(meta)?;
1762    let model_version = select_decommission_capacity_model(meta, target_fence_proof_available)?;
1763    reserve_decommission_start_target_capacity(
1764        meta,
1765        &missing_indices,
1766        capacity_infos,
1767        uuid::Uuid::new_v4(),
1768        generation,
1769        now,
1770        model_version,
1771    )?;
1772    for idx in missing_indices {
1773        if let Some(reservation) = meta
1774            .pools
1775            .get_mut(idx)
1776            .and_then(|pool| pool.decommission.as_mut())
1777            .and_then(|info| info.capacity_reservation.as_mut())
1778        {
1779            reservation.recovered_at = Some(now);
1780        }
1781    }
1782    Ok(active_indices)
1783}
1784
1785fn signed_capacity_difference(observed: usize, predicted: usize) -> i64 {
1786    if observed >= predicted {
1787        i64::try_from(observed.saturating_sub(predicted)).unwrap_or(i64::MAX)
1788    } else {
1789        -i64::try_from(predicted.saturating_sub(observed)).unwrap_or(i64::MAX)
1790    }
1791}
1792
1793fn active_decommission_target_reservations(meta: &PoolMeta) -> HashMap<usize, usize> {
1794    let mut target_reservations = HashMap::new();
1795    for reservation in meta
1796        .pools
1797        .iter()
1798        .filter_map(|pool| pool.decommission.as_ref()?.capacity_reservation.as_ref())
1799        .filter(|reservation| reservation.active())
1800    {
1801        for target in &reservation.targets {
1802            let reserved = target_reservations.entry(target.pool_index).or_insert(0usize);
1803            *reserved = reserved.saturating_add(target.remaining_reserved_physical_bytes(reservation.temporary_copies));
1804        }
1805    }
1806    target_reservations
1807}
1808
1809fn observe_decommission_capacity_reservation(
1810    info: &mut PoolDecommissionInfo,
1811    _capacity_infos: &[DecommissionPoolCapacityInfo],
1812    _active_target_reservations: &HashMap<usize, usize>,
1813) {
1814    let Some(reservation) = info.capacity_reservation.as_mut() else {
1815        return;
1816    };
1817    reservation.prediction_error_bytes =
1818        signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes);
1819    metrics::gauge!(
1820        METRIC_DECOMMISSION_CAPACITY_PREDICTION_ERROR_BYTES,
1821        "pool_index" => reservation.source_pool_index.to_string()
1822    )
1823    .set(reservation.prediction_error_bytes as f64);
1824    metrics::histogram!(
1825        METRIC_DECOMMISSION_CAPACITY_PREDICTION_ABSOLUTE_ERROR_BYTES,
1826        "pool_index" => reservation.source_pool_index.to_string()
1827    )
1828    .record(reservation.prediction_error_bytes.unsigned_abs() as f64);
1829}
1830
1831fn ensure_decommission_capacity_reservations_available(
1832    meta: &PoolMeta,
1833    capacity_infos: &[DecommissionPoolCapacityInfo],
1834    phase: &'static str,
1835) -> Result<()> {
1836    let mut required_by_target = HashMap::<usize, usize>::new();
1837    let mut inflight_by_target = HashMap::<usize, usize>::new();
1838    let mut pending_by_target = HashMap::<usize, usize>::new();
1839    for source_index in active_decommission_source_indices(meta) {
1840        let info = meta.pools[source_index]
1841            .decommission
1842            .as_ref()
1843            .ok_or_else(|| decommission_metadata_not_initialized_error("check decommission capacity reservation"))?;
1844        let reservation = info.capacity_reservation.as_ref().ok_or_else(|| {
1845            decommission_capacity_blocked_error(format!(
1846                "active decommission source pool {source_index} has no durable capacity reservation"
1847            ))
1848        })?;
1849        if !reservation.active() {
1850            return Err(decommission_capacity_blocked_error(format!(
1851                "active decommission source pool {source_index} has a released capacity reservation"
1852            )));
1853        }
1854        metrics::gauge!(
1855            METRIC_DECOMMISSION_CAPACITY_RESERVED_BYTES,
1856            "pool_index" => source_index.to_string()
1857        )
1858        .set(reservation.remaining_peak_physical_bytes() as f64);
1859        for target in &reservation.targets {
1860            let temporary_budget = target.remaining_reserved_physical_bytes(reservation.temporary_copies)
1861                / 1usize.saturating_add(reservation.temporary_copies);
1862            if target.inflight_physical_bytes > temporary_budget {
1863                return Err(decommission_capacity_blocked_error(format!(
1864                    "source pool {source_index} target pool {} has {} inflight physical bytes, exceeding its {temporary_budget}-byte temporary-copy reservation during {phase}",
1865                    target.pool_index, target.inflight_physical_bytes
1866                )));
1867            }
1868            if target.pending_physical_bytes > temporary_budget {
1869                return Err(decommission_capacity_blocked_error(format!(
1870                    "source pool {source_index} target pool {} has {} pending physical bytes, exceeding its {temporary_budget}-byte committed-copy reservation during {phase}",
1871                    target.pool_index, target.pending_physical_bytes
1872                )));
1873            }
1874            let required = required_by_target.entry(target.pool_index).or_default();
1875            *required = required.saturating_add(target.remaining_reserved_physical_bytes(reservation.temporary_copies));
1876            let inflight = inflight_by_target.entry(target.pool_index).or_default();
1877            *inflight = inflight.saturating_add(target.inflight_physical_bytes);
1878            let pending = pending_by_target.entry(target.pool_index).or_default();
1879            *pending = pending.saturating_add(target.pending_physical_bytes);
1880        }
1881    }
1882    for (target_pool_index, required) in required_by_target {
1883        let available = capacity_infos
1884            .iter()
1885            .find(|capacity| capacity.pool_index == target_pool_index)
1886            .map(|capacity| capacity.physical_free)
1887            .ok_or_else(|| {
1888                decommission_capacity_blocked_error(format!(
1889                    "capacity snapshot is missing target pool {target_pool_index} during {phase}"
1890                ))
1891            })?
1892            .saturating_add(inflight_by_target.get(&target_pool_index).copied().unwrap_or_default())
1893            .saturating_add(pending_by_target.get(&target_pool_index).copied().unwrap_or_default());
1894        if available >= required {
1895            continue;
1896        }
1897        metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => phase).increment(1);
1898        return Err(decommission_capacity_blocked_error(format!(
1899            "target pool {target_pool_index} reservation requires {required} physical bytes, but only {available} bytes remain during {phase}"
1900        )));
1901    }
1902    Ok(())
1903}
1904
1905#[derive(Clone, Copy)]
1906pub(crate) enum DecommissionCapacityAdmission {
1907    Mutation,
1908    ExistingMultipart,
1909    ScannerBacklog,
1910    BatchDelete,
1911    Heal,
1912}
1913
1914impl DecommissionCapacityAdmission {
1915    fn phase(self) -> &'static str {
1916        match self {
1917            Self::Mutation => "mutation",
1918            Self::ExistingMultipart => "existing_multipart",
1919            Self::ScannerBacklog => "scanner_backlog",
1920            Self::BatchDelete => "batch_delete",
1921            Self::Heal => "heal",
1922        }
1923    }
1924}
1925
1926fn ensure_external_decommission_target_admission(
1927    meta: &PoolMeta,
1928    target_pool_index: usize,
1929    admission: DecommissionCapacityAdmission,
1930) -> Result<()> {
1931    let phase = admission.phase();
1932    // Pool selection may predate retirement or use a stale node-local snapshot.
1933    // Recheck publication against the fenced durable state. Repair and pure
1934    // capacity release retain their separate admission contracts.
1935    if matches!(admission, DecommissionCapacityAdmission::ScannerBacklog)
1936        && !meta.scanner_pause_backlog_pool_writable(target_pool_index)
1937    {
1938        return Err(Error::SlowDown);
1939    }
1940    let active_sources = active_decommission_source_indices(meta);
1941    if meta.is_suspended(target_pool_index) {
1942        let active_source = active_sources.contains(&target_pool_index);
1943        if !matches!(
1944            admission,
1945            DecommissionCapacityAdmission::Heal | DecommissionCapacityAdmission::ScannerBacklog
1946        ) && !(matches!(admission, DecommissionCapacityAdmission::ExistingMultipart) && active_source)
1947        {
1948            return Err(Error::SlowDown);
1949        }
1950    }
1951    if active_sources.into_iter().any(|source_pool_index| {
1952        meta.pools
1953            .get(source_pool_index)
1954            .and_then(|pool| pool.decommission.as_ref())
1955            .and_then(|info| info.capacity_reservation.as_ref())
1956            .is_none_or(|reservation| !reservation.active())
1957    }) {
1958        metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => phase).increment(1);
1959        return Err(Error::SlowDown);
1960    }
1961    // Migration reservations budget the mover, not exclusive ownership of a
1962    // healthy pool. Foreground publication shares its actual disk capacity;
1963    // migration must retain the source if its capacity or target write fails.
1964    // Repair keeps its separate, conservative reservation admission contract.
1965    if !matches!(admission, DecommissionCapacityAdmission::Heal) {
1966        return Ok(());
1967    }
1968    let reserved = active_decommission_target_reservations(meta)
1969        .get(&target_pool_index)
1970        .copied()
1971        .unwrap_or_default();
1972    if reserved == 0 {
1973        return Ok(());
1974    }
1975    metrics::counter!(METRIC_DECOMMISSION_CAPACITY_CONFLICTS_TOTAL, "phase" => phase).increment(1);
1976    Err(Error::SlowDown)
1977}
1978
1979fn grow_decommission_target_reservation(
1980    meta: &mut PoolMeta,
1981    capacity_infos: &[DecommissionPoolCapacityInfo],
1982    source_pool_index: usize,
1983    target_pool_index: usize,
1984    expected_data_bytes: usize,
1985    mutation_id: uuid::Uuid,
1986    now: OffsetDateTime,
1987) -> Result<bool> {
1988    let original = meta.clone();
1989    let result = (|| {
1990        let target_capacity = capacity_infos
1991            .iter()
1992            .find(|capacity| capacity.pool_index == target_pool_index)
1993            .ok_or_else(|| {
1994                decommission_capacity_blocked_error("target capacity snapshot is missing during reservation growth")
1995            })?;
1996        let pool_count = meta.pools.len();
1997        let pool = meta
1998            .pools
1999            .get_mut(source_pool_index)
2000            .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2001        let reservation = pool
2002            .decommission
2003            .as_mut()
2004            .and_then(|info| info.capacity_reservation.as_mut())
2005            .filter(|reservation| reservation.active())
2006            .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during reservation growth"))?;
2007        let target_layout = reservation
2008            .targets
2009            .iter()
2010            .find(|target| target.pool_index == target_pool_index)
2011            .map(|target| target.layout)
2012            .unwrap_or(target_capacity.layout);
2013        let data_bytes = expected_data_bytes.max(1);
2014        let required_target_bytes = capacity_target_physical_bytes(data_bytes, target_layout)?;
2015        let required_peak = required_target_bytes.saturating_mul(1usize.saturating_add(reservation.temporary_copies));
2016        let existing_target = reservation
2017            .targets
2018            .iter()
2019            .find(|target| target.pool_index == target_pool_index);
2020        if let Some(target) = existing_target
2021            && target.pending_physical_bytes > 0
2022            && target.pending_mutation_id != Some(mutation_id)
2023        {
2024            return Err(decommission_capacity_blocked_error(format!(
2025                "source pool {source_pool_index} target pool {target_pool_index} has an unresolved target capacity intent"
2026            )));
2027        }
2028        let existing_remaining = existing_target
2029            .map(|target| target.remaining_reserved_physical_bytes(reservation.temporary_copies))
2030            .unwrap_or_default();
2031        if required_peak <= existing_remaining {
2032            return Ok(false);
2033        }
2034        if reservation.source_data_equivalent_bytes > 0 {
2035            return Err(decommission_capacity_blocked_error(format!(
2036                "source pool {source_pool_index} reservation estimate is exhausted"
2037            )));
2038        }
2039        if let Some(target) = existing_target
2040            && target_capacity.physical_free <= target.physical_free_at_reservation
2041        {
2042            return Err(decommission_capacity_blocked_error(format!(
2043                "source pool {source_pool_index} target pool {target_pool_index} reservation is exhausted"
2044            )));
2045        }
2046        let current_peak = reservation.remaining_peak_physical_bytes();
2047        let additional_peak = required_peak.saturating_sub(existing_remaining);
2048        let additional_predicted = additional_peak.div_ceil(1usize.saturating_add(reservation.temporary_copies));
2049        let additional_source_data = capacity_source_data_equivalent(additional_predicted, target_layout)?;
2050        reservation.source_data_equivalent_bytes = reservation
2051            .source_data_equivalent_bytes
2052            .saturating_add(additional_source_data);
2053        reservation.source_physical_bytes = reservation
2054            .source_physical_bytes
2055            .saturating_add(capacity_target_physical_bytes(additional_source_data, reservation.source_layout)?);
2056        reservation.predicted_physical_bytes = reservation
2057            .predicted_physical_bytes
2058            .saturating_add(additional_predicted)
2059            .max(capacity_target_physical_bytes(reservation.source_data_equivalent_bytes, target_layout)?);
2060        reservation.temporary_physical_bytes = reservation
2061            .predicted_physical_bytes
2062            .saturating_mul(reservation.temporary_copies);
2063        reservation.peak_physical_bytes = reservation
2064            .predicted_physical_bytes
2065            .saturating_add(reservation.temporary_physical_bytes);
2066        let allocation_growth = reservation.remaining_peak_physical_bytes().saturating_sub(current_peak);
2067        if let Some(target) = reservation
2068            .targets
2069            .iter_mut()
2070            .find(|target| target.pool_index == target_pool_index)
2071        {
2072            target.reserved_physical_bytes = target.reserved_physical_bytes.saturating_add(allocation_growth);
2073            target.physical_total_at_reservation = target.physical_total_at_reservation.max(target_capacity.physical_total);
2074            target.physical_free_at_reservation = target.physical_free_at_reservation.max(target_capacity.physical_free);
2075        } else {
2076            reservation.targets.push(DecommissionCapacityTarget {
2077                pool_index: target_pool_index,
2078                layout: target_layout,
2079                physical_total_at_reservation: target_capacity.physical_total,
2080                physical_free_at_reservation: target_capacity.physical_free,
2081                reserved_physical_bytes: allocation_growth,
2082                consumed_physical_bytes: 0,
2083                observed_physical_bytes: 0,
2084                inflight_physical_bytes: 0,
2085                pending_physical_bytes: 0,
2086                pending_mutation_id: None,
2087                temporary_mutations: Vec::new(),
2088            });
2089        }
2090        renew_decommission_capacity_reservation(reservation, now, true);
2091        pool.last_update = pool.last_update.max(now);
2092        Ok(true)
2093    })();
2094    if result.is_ok() {
2095        if let Err(err) = ensure_decommission_capacity_reservations_available(meta, capacity_infos, "reservation growth") {
2096            *meta = original;
2097            return Err(err);
2098        }
2099    } else {
2100        *meta = original;
2101    }
2102    result
2103}
2104
2105fn ensure_decommission_target_owner_admission(
2106    meta: &PoolMeta,
2107    owner: DecommissionCapacityOwner,
2108    target_pool_index: usize,
2109    target_physical_bytes: usize,
2110    now: OffsetDateTime,
2111) -> Result<()> {
2112    let reservation = meta
2113        .pools
2114        .get(owner.source_pool_index)
2115        .and_then(|pool| pool.decommission.as_ref())
2116        .and_then(|info| info.capacity_reservation.as_ref())
2117        .filter(|reservation| reservation.admits_owner(owner, now))
2118        .ok_or_else(|| Error::SlowDown)?;
2119    let target = reservation
2120        .targets
2121        .iter()
2122        .find(|target| target.pool_index == target_pool_index)
2123        .ok_or_else(|| Error::SlowDown)?;
2124    let required_peak = target_physical_bytes.saturating_mul(1usize.saturating_add(reservation.temporary_copies));
2125    let remaining = target.remaining_reserved_physical_bytes(reservation.temporary_copies);
2126    if required_peak <= remaining {
2127        return Ok(());
2128    }
2129    Err(decommission_capacity_blocked_error(format!(
2130        "source pool {} target pool {target_pool_index} operation requires {required_peak} physical bytes, but only {remaining} reserved bytes remain",
2131        owner.source_pool_index
2132    )))
2133}
2134
2135fn reserve_decommission_target_pending(
2136    meta: &mut PoolMeta,
2137    source_pool_index: usize,
2138    target_pool_index: usize,
2139    requested_physical_bytes: usize,
2140    mutation_id: uuid::Uuid,
2141    now: OffsetDateTime,
2142) -> Result<usize> {
2143    if requested_physical_bytes == 0 {
2144        return Ok(0);
2145    }
2146    let pool_count = meta.pools.len();
2147    let pool = meta
2148        .pools
2149        .get_mut(source_pool_index)
2150        .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2151    let info = pool
2152        .decommission
2153        .as_mut()
2154        .ok_or_else(|| decommission_metadata_not_initialized_error("reserve target capacity mutation"))?;
2155    let reservation = info
2156        .capacity_reservation
2157        .as_mut()
2158        .filter(|reservation| reservation.active())
2159        .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared before target write"))?;
2160    let target = reservation
2161        .targets
2162        .iter_mut()
2163        .find(|target| target.pool_index == target_pool_index)
2164        .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared before target write"))?;
2165    if target.pending_physical_bytes == 0 {
2166        target.pending_mutation_id = None;
2167    }
2168    if target.pending_physical_bytes > 0 {
2169        match target.pending_mutation_id {
2170            Some(pending_mutation_id) if pending_mutation_id == mutation_id => {
2171                if requested_physical_bytes <= target.pending_physical_bytes {
2172                    return Ok(0);
2173                }
2174                let budget = target.remaining_reserved_physical_bytes(reservation.temporary_copies)
2175                    / 1usize.saturating_add(reservation.temporary_copies);
2176                if requested_physical_bytes > budget {
2177                    return Err(decommission_capacity_blocked_error(format!(
2178                        "source pool {source_pool_index} target pool {target_pool_index} mutation intent size changed during retry"
2179                    )));
2180                }
2181                let additional = requested_physical_bytes.saturating_sub(target.pending_physical_bytes);
2182                target.pending_physical_bytes = requested_physical_bytes;
2183                reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_add(additional);
2184                renew_decommission_capacity_reservation(reservation, now, true);
2185                pool.last_update = pool.last_update.max(now);
2186                return Ok(additional);
2187            }
2188            pending_mutation_id => {
2189                return Err(decommission_capacity_blocked_error(format!(
2190                    "source pool {source_pool_index} target pool {target_pool_index} has an unresolved target capacity intent {pending_mutation_id:?} while mutation {mutation_id} is waiting"
2191                )));
2192            }
2193        }
2194    }
2195    let budget = target.remaining_reserved_physical_bytes(reservation.temporary_copies)
2196        / 1usize.saturating_add(reservation.temporary_copies);
2197    if requested_physical_bytes > budget {
2198        return Err(decommission_capacity_blocked_error(format!(
2199            "source pool {source_pool_index} target pool {target_pool_index} mutation requires {requested_physical_bytes} physical bytes, but only {budget} committed-copy bytes remain"
2200        )));
2201    }
2202    let additional = requested_physical_bytes;
2203    target.pending_mutation_id = Some(mutation_id);
2204    target.pending_physical_bytes = target.pending_physical_bytes.saturating_add(additional);
2205    reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_add(additional);
2206    renew_decommission_capacity_reservation(reservation, now, true);
2207    pool.last_update = pool.last_update.max(now);
2208    Ok(additional)
2209}
2210
2211fn resolve_decommission_target_pending(
2212    meta: &mut PoolMeta,
2213    source_pool_index: usize,
2214    target_pool_index: usize,
2215    resolved_physical_bytes: usize,
2216    mutation_id: uuid::Uuid,
2217) -> Result<()> {
2218    if resolved_physical_bytes == 0 {
2219        return Ok(());
2220    }
2221    let pool_count = meta.pools.len();
2222    let pool = meta
2223        .pools
2224        .get_mut(source_pool_index)
2225        .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2226    let reservation = pool
2227        .decommission
2228        .as_mut()
2229        .and_then(|info| info.capacity_reservation.as_mut())
2230        .filter(|reservation| reservation.active())
2231        .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared after target write"))?;
2232    let target = reservation
2233        .targets
2234        .iter_mut()
2235        .find(|target| target.pool_index == target_pool_index)
2236        .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared after target write"))?;
2237    if target.pending_physical_bytes > 0 && target.pending_mutation_id != Some(mutation_id) {
2238        return Err(decommission_capacity_blocked_error(format!(
2239            "source pool {source_pool_index} target pool {target_pool_index} pending capacity intent belongs to another mutation"
2240        )));
2241    }
2242    let resolved = target.pending_physical_bytes.min(resolved_physical_bytes);
2243    target.pending_physical_bytes = target.pending_physical_bytes.saturating_sub(resolved);
2244    reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_sub(resolved);
2245    if target.pending_physical_bytes == 0 {
2246        target.pending_mutation_id = None;
2247    }
2248    Ok(())
2249}
2250
2251fn release_decommission_target_temporary_mutation(
2252    target: &mut DecommissionCapacityTarget,
2253    mutation_id: uuid::Uuid,
2254    maximum_physical_bytes: usize,
2255) -> (usize, bool) {
2256    let Some(index) = target
2257        .temporary_mutations
2258        .iter()
2259        .position(|mutation| mutation.mutation_id == mutation_id)
2260    else {
2261        return (0, false);
2262    };
2263    let released = target.temporary_mutations[index].physical_bytes.min(maximum_physical_bytes);
2264    target.temporary_mutations[index].physical_bytes = target.temporary_mutations[index].physical_bytes.saturating_sub(released);
2265    let removed = target.temporary_mutations[index].physical_bytes == 0;
2266    if removed {
2267        target.temporary_mutations.remove(index);
2268    }
2269    target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released);
2270    (released, released > 0 || removed)
2271}
2272
2273fn settle_decommission_target_non_growing_replacement(
2274    meta: &mut PoolMeta,
2275    source_pool_index: usize,
2276    target_pool_index: usize,
2277    mutation_id: uuid::Uuid,
2278    now: OffsetDateTime,
2279) -> Result<bool> {
2280    let pool_count = meta.pools.len();
2281    let pool = meta
2282        .pools
2283        .get_mut(source_pool_index)
2284        .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2285    let info = pool
2286        .decommission
2287        .as_mut()
2288        .ok_or_else(|| decommission_metadata_not_initialized_error("settle non-growing target replacement"))?;
2289    let reservation = info
2290        .capacity_reservation
2291        .as_mut()
2292        .filter(|reservation| reservation.active())
2293        .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target replacement"))?;
2294    let target = reservation
2295        .targets
2296        .iter_mut()
2297        .find(|target| target.pool_index == target_pool_index)
2298        .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target replacement"))?;
2299    let (released, changed) = release_decommission_target_temporary_mutation(target, mutation_id, usize::MAX);
2300    reservation.inflight_target_physical_bytes = reservation.inflight_target_physical_bytes.saturating_sub(released);
2301    if changed {
2302        renew_decommission_capacity_reservation(reservation, now, true);
2303        pool.last_update = pool.last_update.max(now);
2304    }
2305    Ok(changed)
2306}
2307
2308struct DecommissionTargetConsumption {
2309    committed_data_bytes: usize,
2310    target_physical_bytes: usize,
2311    observed_physical_bytes: usize,
2312}
2313
2314fn record_decommission_target_consumption(
2315    meta: &mut PoolMeta,
2316    source_pool_index: usize,
2317    target_pool_index: usize,
2318    consumption: DecommissionTargetConsumption,
2319    mutation_id: uuid::Uuid,
2320    now: OffsetDateTime,
2321) -> Result<()> {
2322    let DecommissionTargetConsumption {
2323        committed_data_bytes,
2324        target_physical_bytes,
2325        observed_physical_bytes,
2326    } = consumption;
2327    let pool_count = meta.pools.len();
2328    let pool = meta
2329        .pools
2330        .get_mut(source_pool_index)
2331        .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2332    let info = pool
2333        .decommission
2334        .as_mut()
2335        .ok_or_else(|| decommission_metadata_not_initialized_error("record target capacity consumption"))?;
2336    let reservation = info
2337        .capacity_reservation
2338        .as_mut()
2339        .filter(|reservation| reservation.active())
2340        .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target write"))?;
2341    let target = reservation
2342        .targets
2343        .iter_mut()
2344        .find(|target| target.pool_index == target_pool_index)
2345        .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target write"))?;
2346    let remaining_target_bytes = target.remaining_reserved_physical_bytes(reservation.temporary_copies)
2347        / 1usize.saturating_add(reservation.temporary_copies);
2348    let remaining_total_bytes = reservation
2349        .predicted_physical_bytes
2350        .saturating_sub(reservation.consumed_target_physical_bytes);
2351    let consumed = target_physical_bytes.min(remaining_target_bytes).min(remaining_total_bytes);
2352    target.consumed_physical_bytes = target.consumed_physical_bytes.saturating_add(consumed);
2353    target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes);
2354    let has_scoped_temporary_mutations = !target.temporary_mutations.is_empty();
2355    let (released_inflight, _) = release_decommission_target_temporary_mutation(target, mutation_id, usize::MAX);
2356    let released_inflight = if released_inflight == 0 && !has_scoped_temporary_mutations {
2357        let released = target.inflight_physical_bytes.min(consumed);
2358        target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released);
2359        released
2360    } else {
2361        released_inflight
2362    };
2363    reservation.consumed_target_physical_bytes = reservation.consumed_target_physical_bytes.saturating_add(consumed);
2364    let committed = committed_data_bytes.min(
2365        reservation
2366            .source_data_equivalent_bytes
2367            .saturating_sub(reservation.committed_data_bytes),
2368    );
2369    reservation.committed_data_bytes = reservation.committed_data_bytes.saturating_add(committed);
2370    reservation.observed_target_physical_bytes = reservation
2371        .observed_target_physical_bytes
2372        .saturating_add(observed_physical_bytes);
2373    reservation.inflight_target_physical_bytes = reservation.inflight_target_physical_bytes.saturating_sub(released_inflight);
2374    reservation.prediction_error_bytes =
2375        signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes);
2376    renew_decommission_capacity_reservation(reservation, now, true);
2377    info.capacity_blocked_reason = None;
2378    pool.last_update = pool.last_update.max(now);
2379    Ok(())
2380}
2381
2382fn record_decommission_target_inflight(
2383    meta: &mut PoolMeta,
2384    source_pool_index: usize,
2385    target_pool_index: usize,
2386    observed_physical_bytes: usize,
2387    mutation_id: uuid::Uuid,
2388    now: OffsetDateTime,
2389) -> Result<bool> {
2390    let pool_count = meta.pools.len();
2391    let pool = meta
2392        .pools
2393        .get_mut(source_pool_index)
2394        .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2395    let info = pool
2396        .decommission
2397        .as_mut()
2398        .ok_or_else(|| decommission_metadata_not_initialized_error("record target capacity inflight bytes"))?;
2399    let reservation = info
2400        .capacity_reservation
2401        .as_mut()
2402        .filter(|reservation| reservation.active())
2403        .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target write"))?;
2404    let target = reservation
2405        .targets
2406        .iter_mut()
2407        .find(|target| target.pool_index == target_pool_index)
2408        .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target write"))?;
2409    let ledger_changed = if let Some(mutation) = target
2410        .temporary_mutations
2411        .iter_mut()
2412        .find(|mutation| mutation.mutation_id == mutation_id)
2413    {
2414        mutation.physical_bytes = mutation.physical_bytes.saturating_add(observed_physical_bytes);
2415        observed_physical_bytes > 0
2416    } else if observed_physical_bytes > 0 || reservation.model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION {
2417        // A v2 zero-byte record is a durable discovery marker. It keeps
2418        // restart cleanup scoped to mutations that actually staged an MPU
2419        // without charging capacity when statfs observed no physical delta.
2420        target.temporary_mutations.push(DecommissionCapacityTemporaryMutation {
2421            mutation_id,
2422            physical_bytes: observed_physical_bytes,
2423        });
2424        true
2425    } else {
2426        false
2427    };
2428    target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes);
2429    target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_add(observed_physical_bytes);
2430    reservation.observed_target_physical_bytes = reservation
2431        .observed_target_physical_bytes
2432        .saturating_add(observed_physical_bytes);
2433    reservation.inflight_target_physical_bytes = reservation
2434        .inflight_target_physical_bytes
2435        .saturating_add(observed_physical_bytes);
2436    reservation.prediction_error_bytes =
2437        signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes);
2438    renew_decommission_capacity_reservation(reservation, now, true);
2439    pool.last_update = pool.last_update.max(now);
2440    Ok(ledger_changed)
2441}
2442
2443fn record_decommission_target_observation(
2444    meta: &mut PoolMeta,
2445    source_pool_index: usize,
2446    target_pool_index: usize,
2447    observed_physical_bytes: usize,
2448    now: OffsetDateTime,
2449) -> Result<()> {
2450    let pool_count = meta.pools.len();
2451    let pool = meta
2452        .pools
2453        .get_mut(source_pool_index)
2454        .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2455    let info = pool
2456        .decommission
2457        .as_mut()
2458        .ok_or_else(|| decommission_metadata_not_initialized_error("record target capacity observation"))?;
2459    let reservation = info
2460        .capacity_reservation
2461        .as_mut()
2462        .filter(|reservation| reservation.active())
2463        .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target write"))?;
2464    let target = reservation
2465        .targets
2466        .iter_mut()
2467        .find(|target| target.pool_index == target_pool_index)
2468        .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target write"))?;
2469    target.observed_physical_bytes = target.observed_physical_bytes.saturating_add(observed_physical_bytes);
2470    reservation.observed_target_physical_bytes = reservation
2471        .observed_target_physical_bytes
2472        .saturating_add(observed_physical_bytes);
2473    reservation.prediction_error_bytes =
2474        signed_capacity_difference(reservation.observed_target_physical_bytes, reservation.consumed_target_physical_bytes);
2475    renew_decommission_capacity_reservation(reservation, now, true);
2476    pool.last_update = pool.last_update.max(now);
2477    Ok(())
2478}
2479
2480fn release_decommission_target_inflight(
2481    meta: &mut PoolMeta,
2482    source_pool_index: usize,
2483    target_pool_index: usize,
2484    released_physical_bytes: usize,
2485    mutation_id: uuid::Uuid,
2486    proof: DecommissionCapacityReleaseProof,
2487    now: OffsetDateTime,
2488) -> Result<bool> {
2489    let DecommissionCapacityReleaseProof {
2490        confirmed_absent,
2491        clear_pending,
2492    } = proof;
2493    let pool_count = meta.pools.len();
2494    let pool = meta
2495        .pools
2496        .get_mut(source_pool_index)
2497        .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, source_pool_index))?;
2498    let info = pool
2499        .decommission
2500        .as_mut()
2501        .ok_or_else(|| decommission_metadata_not_initialized_error("release target capacity inflight bytes"))?;
2502    let reservation = info
2503        .capacity_reservation
2504        .as_mut()
2505        .filter(|reservation| reservation.active())
2506        .ok_or_else(|| decommission_capacity_blocked_error("active reservation disappeared during target cleanup"))?;
2507    let model_version = reservation.model_version;
2508    let target = reservation
2509        .targets
2510        .iter_mut()
2511        .find(|target| target.pool_index == target_pool_index)
2512        .ok_or_else(|| decommission_capacity_blocked_error("target allocation disappeared during target cleanup"))?;
2513    let has_temporary_mutations = !target.temporary_mutations.is_empty();
2514    let pending_belongs_to_mutation = target.pending_mutation_id == Some(mutation_id);
2515    let zero_ledger_cleanup = confirmed_absent
2516        && model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
2517        && !has_temporary_mutations
2518        && target.pending_mutation_id.is_none()
2519        && target.pending_physical_bytes == 0;
2520    let (released, temporary_mutation_changed) = release_decommission_target_temporary_mutation(
2521        target,
2522        mutation_id,
2523        if confirmed_absent {
2524            usize::MAX
2525        } else {
2526            released_physical_bytes
2527        },
2528    );
2529    let released = if released == 0 && !has_temporary_mutations && model_version == DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION {
2530        let released = target.inflight_physical_bytes.min(released_physical_bytes);
2531        target.inflight_physical_bytes = target.inflight_physical_bytes.saturating_sub(released);
2532        released
2533    } else {
2534        released
2535    };
2536    let pending_matches = clear_pending && confirmed_absent && pending_belongs_to_mutation;
2537    let published_pending_delayed_release = confirmed_absent
2538        && model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
2539        && !clear_pending
2540        && !has_temporary_mutations
2541        && pending_belongs_to_mutation;
2542    if released == 0
2543        && released_physical_bytes > 0
2544        && !zero_ledger_cleanup
2545        && !published_pending_delayed_release
2546        && (has_temporary_mutations || !pending_matches)
2547    {
2548        return Err(decommission_capacity_blocked_error(
2549            "temporary target cleanup released bytes that cannot be attributed to its mutation",
2550        ));
2551    }
2552    reservation.inflight_target_physical_bytes = reservation.inflight_target_physical_bytes.saturating_sub(released);
2553    let cleared_pending = if pending_matches {
2554        let cleared = target.pending_physical_bytes;
2555        target.pending_physical_bytes = 0;
2556        target.pending_mutation_id = None;
2557        reservation.pending_target_physical_bytes = reservation.pending_target_physical_bytes.saturating_sub(cleared);
2558        cleared
2559    } else {
2560        0
2561    };
2562    let changed = released > 0 || temporary_mutation_changed || cleared_pending > 0;
2563    if changed {
2564        renew_decommission_capacity_reservation(reservation, now, true);
2565        pool.last_update = pool.last_update.max(now);
2566    }
2567    Ok(changed)
2568}
2569
2570fn ensure_valid_decommission_pool_index(pool_count: usize, idx: usize) -> Result<()> {
2571    if idx >= pool_count {
2572        return Err(invalid_decommission_pool_index_error(pool_count, idx));
2573    }
2574
2575    Ok(())
2576}
2577
2578fn get_by_index<'a, T>(items: &'a [T], idx: usize, operation: &'static str) -> Result<&'a T> {
2579    items.get(idx).ok_or_else(|| {
2580        Error::other(format!(
2581            "failed to {operation}: invalid decommission pool index {idx} for {pool_count} pools",
2582            pool_count = items.len()
2583        ))
2584    })
2585}
2586
2587fn decommission_metadata_not_initialized_error(operation: &str) -> Error {
2588    Error::other(format!("failed to {operation}: decommission metadata not initialized"))
2589}
2590
2591fn resolve_decommission_bucket_state(meta: &PoolMeta, idx: usize, bucket: &DecomBucketInfo) -> Result<bool> {
2592    let pool_count = meta.pools.len();
2593    ensure_valid_decommission_pool_index(pool_count, idx)?;
2594
2595    let Some(pool) = meta.pools.get(idx) else {
2596        return Err(invalid_decommission_pool_index_error(pool_count, idx));
2597    };
2598    let Some(info) = pool.decommission.as_ref() else {
2599        return Err(decommission_metadata_not_initialized_error("resolve decommission bucket state"));
2600    };
2601
2602    Ok(info.is_bucket_decommissioned(&bucket.to_string()))
2603}
2604
2605fn mark_decommission_bucket_done(meta: &mut PoolMeta, idx: usize, bucket: &DecomBucketInfo) -> Result<bool> {
2606    let pool_count = meta.pools.len();
2607    ensure_valid_decommission_pool_index(pool_count, idx)?;
2608
2609    let Some(pool) = meta.pools.get_mut(idx) else {
2610        return Err(invalid_decommission_pool_index_error(pool_count, idx));
2611    };
2612    let Some(info) = pool.decommission.as_mut() else {
2613        return Err(decommission_metadata_not_initialized_error("mark decommission bucket done"));
2614    };
2615
2616    Ok(info.bucket_pop(&bucket.to_string()))
2617}
2618
2619fn count_decommission_item(meta: &mut PoolMeta, idx: usize, size: usize, failed: bool) -> Result<()> {
2620    let pool_count = meta.pools.len();
2621    ensure_valid_decommission_pool_index(pool_count, idx)?;
2622
2623    let Some(pool) = meta.pools.get_mut(idx) else {
2624        return Err(invalid_decommission_pool_index_error(pool_count, idx));
2625    };
2626    let Some(info) = pool.decommission.as_mut() else {
2627        return Err(decommission_metadata_not_initialized_error("count decommission item"));
2628    };
2629
2630    if failed {
2631        info.items_decommission_failed += 1;
2632        info.bytes_failed += size;
2633    } else {
2634        info.items_decommissioned += 1;
2635        info.bytes_done += size;
2636    }
2637
2638    Ok(())
2639}
2640
2641fn ensure_decommission_generation(meta: &PoolMeta, idx: usize, generation: OffsetDateTime) -> Result<()> {
2642    let Some(pool) = meta.pools.get(idx) else {
2643        return Err(invalid_decommission_pool_index_error(meta.pools.len(), idx));
2644    };
2645    let Some(info) = pool.decommission.as_ref() else {
2646        return Err(decommission_metadata_not_initialized_error("check decommission generation"));
2647    };
2648
2649    if info.start_time == Some(generation) && !info.queued && is_decommission_active(info.complete, info.failed, info.canceled) {
2650        Ok(())
2651    } else {
2652        Err(Error::OperationCanceled)
2653    }
2654}
2655
2656fn record_decommission_unresolved_entry(
2657    meta: &mut PoolMeta,
2658    idx: usize,
2659    generation: OffsetDateTime,
2660    entry: DecommissionUnresolvedEntry,
2661    now: OffsetDateTime,
2662    rebalance_meta: Option<&RebalanceMeta>,
2663) -> Result<bool> {
2664    ensure_decommission_generation(meta, idx, generation)?;
2665    if entry.pool_index != idx || entry.source_generation != generation {
2666        return Err(Error::other("decommission unresolved entry does not match the active pool generation"));
2667    }
2668
2669    let pool_count = meta.pools.len();
2670    let Some(pool) = meta.pools.get(idx) else {
2671        return Err(invalid_decommission_pool_index_error(pool_count, idx));
2672    };
2673    let Some(info) = pool.decommission.as_ref() else {
2674        return Err(decommission_metadata_not_initialized_error("record decommission unresolved entry"));
2675    };
2676    let existing_index = info.unresolved_entries.iter().position(|existing| {
2677        existing.bucket == entry.bucket
2678            && existing.object == entry.object
2679            && existing.pool_index == entry.pool_index
2680            && existing.set_index == entry.set_index
2681            && existing.source_generation == entry.source_generation
2682    });
2683    if existing_index.is_some_and(|index| info.unresolved_entries[index] == entry) {
2684        return Ok(false);
2685    }
2686
2687    let last_update = meta.next_scanner_data_movement_update(now, rebalance_meta);
2688    let Some(pool) = meta.pools.get_mut(idx) else {
2689        return Err(invalid_decommission_pool_index_error(pool_count, idx));
2690    };
2691    let Some(info) = pool.decommission.as_mut() else {
2692        return Err(decommission_metadata_not_initialized_error("record decommission unresolved entry"));
2693    };
2694    if let Some(index) = existing_index {
2695        info.unresolved_entries[index] = entry;
2696    } else {
2697        info.unresolved_entries.push(entry);
2698    }
2699    pool.last_update = last_update;
2700    Ok(true)
2701}
2702
2703type DecommissionUnresolvedEntryIdentity = (usize, String, String);
2704
2705fn decommission_unresolved_entry_identity(entry: &DecommissionUnresolvedEntry) -> DecommissionUnresolvedEntryIdentity {
2706    (entry.set_index, entry.bucket.clone(), entry.object.clone())
2707}
2708
2709fn reconcile_decommission_unresolved_entries_for_completion(
2710    meta: &mut PoolMeta,
2711    idx: usize,
2712    verified_generation: Option<OffsetDateTime>,
2713    verified_entries: Option<&[DecommissionUnresolvedEntry]>,
2714) -> Result<()> {
2715    let pool_count = meta.pools.len();
2716    let Some(pool) = meta.pools.get(idx) else {
2717        return Err(invalid_decommission_pool_index_error(pool_count, idx));
2718    };
2719    let Some(info) = pool.decommission.as_ref() else {
2720        return Err(decommission_metadata_not_initialized_error("reconcile decommission unresolved entries"));
2721    };
2722    if info.unresolved_entries.is_empty() {
2723        return Ok(());
2724    }
2725
2726    let Some(generation) = verified_generation else {
2727        return Err(Error::other(format!(
2728            "failed to complete decommission for pool {idx}: {} unresolved listing entries remain",
2729            info.unresolved_entries.len()
2730        )));
2731    };
2732    ensure_decommission_generation(meta, idx, generation)?;
2733    if info
2734        .unresolved_entries
2735        .iter()
2736        .any(|entry| entry.source_generation != generation)
2737    {
2738        return Err(Error::other(format!(
2739            "failed to complete decommission for pool {idx}: unresolved listing ledger contains a different generation"
2740        )));
2741    }
2742
2743    let verified_entries = verified_entries.unwrap_or_default();
2744    let unverified_count = info
2745        .unresolved_entries
2746        .iter()
2747        .filter(|entry| !verified_entries.contains(entry))
2748        .count();
2749    if unverified_count > 0 {
2750        return Err(Error::other(format!(
2751            "failed to complete decommission for pool {idx}: {unverified_count} unresolved listing entries were not individually verified"
2752        )));
2753    }
2754
2755    let Some(info) = meta.pools.get_mut(idx).and_then(|pool| pool.decommission.as_mut()) else {
2756        return Err(decommission_metadata_not_initialized_error("reconcile decommission unresolved entries"));
2757    };
2758    info.unresolved_entries.clear();
2759    Ok(())
2760}
2761
2762#[cfg(test)]
2763async fn run_decommission_side_effect<T, E, F, Fut>(
2764    rx: &CancellationToken,
2765    operation_gate: &Arc<tokio::sync::RwLock<()>>,
2766    operation: F,
2767) -> std::result::Result<T, E>
2768where
2769    F: FnOnce() -> Fut,
2770    Fut: std::future::Future<Output = std::result::Result<T, E>>,
2771    E: From<Error>,
2772{
2773    let _operation_guard = tokio::select! {
2774        biased;
2775        _ = rx.cancelled() => return Err(Error::OperationCanceled.into()),
2776        guard = operation_gate.read() => guard,
2777    };
2778
2779    if rx.is_cancelled() {
2780        return Err(Error::OperationCanceled.into());
2781    }
2782
2783    let result = operation().await;
2784    if rx.is_cancelled() {
2785        return Err(Error::OperationCanceled.into());
2786    }
2787    result
2788}
2789
2790fn track_decommission_current_object_stage(
2791    meta: &mut PoolMeta,
2792    idx: usize,
2793    bucket: &str,
2794    object: &str,
2795    stage: &str,
2796) -> Result<()> {
2797    let pool_count = meta.pools.len();
2798    ensure_valid_decommission_pool_index(pool_count, idx)?;
2799
2800    let Some(pool) = meta.pools.get_mut(idx) else {
2801        return Err(invalid_decommission_pool_index_error(pool_count, idx));
2802    };
2803    let Some(info) = pool.decommission.as_mut() else {
2804        return Err(decommission_metadata_not_initialized_error("track decommission current object"));
2805    };
2806
2807    info.object = object.to_string();
2808    info.bucket = bucket.to_string();
2809    info.stage = stage.to_string();
2810    Ok(())
2811}
2812
2813fn track_decommission_current_object(meta: &mut PoolMeta, idx: usize, bucket: &str, object: &str) -> Result<()> {
2814    track_decommission_current_object_stage(meta, idx, bucket, object, "")
2815}
2816
2817fn resolve_decommission_update_after_result(result: Result<bool>) -> Result<bool> {
2818    result.map_err(|err| Error::other(format!("decommission metadata update failed: {err}")))
2819}
2820
2821fn resolve_decommission_progress_save_result(result: Result<()>) -> Option<Error> {
2822    result
2823        .err()
2824        .map(|err| Error::other(format!("decommission progress save failed: {err}")))
2825}
2826
2827fn resolve_decommission_preflight_heal_result<T>(bucket: &str, result: Result<T>) -> Result<T> {
2828    result.map_err(|err| Error::other(format!("decommission preflight heal failed for bucket {bucket}: {err}")))
2829}
2830
2831fn resolve_decommission_optional_bucket_config_result<T>(bucket: &str, stage: &str, result: Result<T>) -> Result<Option<T>> {
2832    match result {
2833        Ok(config) => Ok(Some(config)),
2834        Err(Error::ConfigNotFound) => Ok(None),
2835        Err(err) => Err(Error::other(format!(
2836            "decommission {stage} config load failed for bucket {bucket}: {err}"
2837        ))),
2838    }
2839}
2840
2841fn resolve_decommission_entry_cleanup_delete_result<T>(result: Result<T>, bucket: &str, object_name: &str) -> Result<()> {
2842    match result {
2843        Ok(_) => Ok(()),
2844        Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(()),
2845        Err(err) => Err(Error::other(format!(
2846            "decommission cleanup_delete_object failed for {bucket}/{object_name}: {err}"
2847        ))),
2848    }
2849}
2850
2851fn resolve_decommission_entry_reload_result(result: Result<()>, bucket: &str, object_name: &str) -> Result<()> {
2852    result.map_err(|err| Error::other(format!("decommission reload_pool_meta failed for {bucket}/{object_name}: {err}")))
2853}
2854
2855fn resolve_decommission_terminal_mark_result(result: Result<()>, stage: &str, pool_label: &str) -> Result<()> {
2856    result.map_err(|err| Error::other(format!("decommission terminal mark {stage} failed for pool {pool_label}: {err}")))
2857}
2858
2859fn resolve_decommission_terminal_mark_after_error_result(result: Result<()>, idx: usize, primary_err: &Error) -> Result<()> {
2860    result.map_err(|err| {
2861        Error::other(format!(
2862            "decommission terminal mark failed after background error on pool {idx}: {primary_err}; mark error: {err}"
2863        ))
2864    })
2865}
2866
2867fn observe_decommission_terminal_reload_result(result: Result<()>, stage: &str) -> Option<Error> {
2868    result
2869        .err()
2870        .map(|err| Error::other(format!("decommission terminal pool meta reload failed during {stage}: {err}")))
2871}
2872
2873fn decommission_item_size<T>(size: T) -> usize
2874where
2875    usize: TryFrom<T>,
2876{
2877    usize::try_from(size).unwrap_or_default()
2878}
2879
2880fn is_decommission_set_local_usage_cache(bucket: &str, object: &str) -> bool {
2881    if bucket != RUSTFS_META_BUCKET {
2882        return false;
2883    }
2884    let Some(path) = object
2885        .strip_prefix(BUCKET_META_PREFIX)
2886        .and_then(|path| path.strip_prefix('/'))
2887    else {
2888        return false;
2889    };
2890    let name = match path.rsplit_once('/') {
2891        Some((bucket, name)) if !bucket.is_empty() && !bucket.contains('/') && bucket != "." && bucket != ".." => name,
2892        Some(_) => return false,
2893        None => path,
2894    };
2895    name.strip_suffix(".bkp").unwrap_or(name) == DATA_USAGE_CACHE_NAME
2896}
2897
2898fn with_decommission_entry_context<E: Display>(stage: &str, bucket: &str, object: &str, err: E) -> Error {
2899    Error::other(format!("decommission entry {stage} failed for bucket {bucket} object {object}: {err}"))
2900}
2901
2902#[cfg(test)]
2903fn load_decommission_entry_versions(entry: &MetaCacheEntry, bucket: &str, stage: &str) -> Result<FileInfoVersions> {
2904    entry
2905        .file_info_versions(bucket)
2906        .map_err(|err| with_decommission_entry_context(stage, bucket, &entry.name, err))
2907}
2908
2909fn empty_decommission_entry_versions(bucket: &str, object: &str) -> FileInfoVersions {
2910    FileInfoVersions {
2911        volume: bucket.to_string(),
2912        name: object.to_string(),
2913        versions: Vec::new(),
2914        ..Default::default()
2915    }
2916}
2917
2918fn resolve_decommission_entry_exact_versions(
2919    result: Result<Option<FileInfoVersions>>,
2920    entry: &MetaCacheEntry,
2921    bucket: &str,
2922    stage: &str,
2923) -> Result<FileInfoVersions> {
2924    match result {
2925        Ok(Some(fivs)) => Ok(fivs),
2926        Ok(None) => Ok(empty_decommission_entry_versions(bucket, &entry.name)),
2927        Err(err) => Err(with_decommission_entry_context(stage, bucket, &entry.name, err)),
2928    }
2929}
2930
2931async fn load_decommission_entry_exact_versions(
2932    set: &SetDisks,
2933    entry: &MetaCacheEntry,
2934    bucket: &str,
2935    stage: &str,
2936) -> Result<FileInfoVersions> {
2937    resolve_decommission_entry_exact_versions(set.load_file_info_versions_exact(bucket, &entry.name).await, entry, bucket, stage)
2938}
2939
2940fn resolve_decommission_check_after_list_result(list_result: Result<()>, entry_error: Option<Error>) -> Result<()> {
2941    match list_result {
2942        Ok(()) => entry_error.map_or(Ok(()), Err),
2943        Err(list_err) => resolve_decommission_listing_error(Some(list_err), entry_error).map_or(Ok(()), Err),
2944    }
2945}
2946
2947fn resolve_decommission_listing_error(listing_error: Option<Error>, entry_error: Option<Error>) -> Option<Error> {
2948    match (listing_error, entry_error) {
2949        (Some(listing_error), Some(entry_error)) if is_err_operation_canceled(&listing_error) => Some(entry_error),
2950        (Some(listing_error), Some(entry_error)) if is_err_operation_canceled(&entry_error) => Some(listing_error),
2951        (Some(listing_error), _) => Some(listing_error),
2952        (None, entry_error) => entry_error,
2953    }
2954}
2955
2956fn decommission_unresolved_listing_error(entry: &DecommissionUnresolvedEntry) -> Error {
2957    Error::other(format!(
2958        "decommission listing could not resolve metadata for {bucket}/{object} on pool {pool_index} set {set_index} ({candidate_count} candidate(s), {disk_error_count} disk error(s))",
2959        bucket = entry.bucket,
2960        object = entry.object,
2961        pool_index = entry.pool_index,
2962        set_index = entry.set_index,
2963        candidate_count = entry.candidate_count,
2964        disk_error_count = entry.disk_error_count,
2965    ))
2966}
2967
2968// The unresolved-entry payload carries listing context by design; boxing it
2969// would churn every caller without changing behavior.
2970#[allow(clippy::result_large_err)]
2971#[allow(clippy::too_many_arguments)]
2972fn resolve_decommission_partial_listing_entry(
2973    entries: MetaCacheEntries,
2974    resolver: MetadataResolutionParams,
2975    bucket: &str,
2976    prefix: &str,
2977    disk_error_count: usize,
2978    pool_index: usize,
2979    set_index: usize,
2980    source_generation: OffsetDateTime,
2981) -> std::result::Result<MetaCacheEntry, DecommissionUnresolvedEntry> {
2982    let candidate_count = entries.as_ref().iter().flatten().count();
2983    if let Some(entry) = entries.resolve(resolver) {
2984        return Ok(entry);
2985    }
2986
2987    let object = entries
2988        .as_ref()
2989        .iter()
2990        .flatten()
2991        .map(|entry| entry.name.as_str())
2992        .next()
2993        .unwrap_or(prefix)
2994        .to_string();
2995    Err(DecommissionUnresolvedEntry {
2996        bucket: bucket.to_string(),
2997        object,
2998        candidate_count,
2999        disk_error_count,
3000        pool_index,
3001        set_index,
3002        source_generation,
3003        observed_at: OffsetDateTime::now_utc(),
3004        reason: "metadata_resolution_failed".to_string(),
3005    })
3006}
3007
3008fn validate_decommission_durable_ilm_copy(
3009    path: &str,
3010    source_record: &ValidatedDurableIlmRecord,
3011    target: &[u8],
3012) -> Result<ValidatedDurableIlmRecord> {
3013    let target_record = validate_durable_ilm_record(path, target).map_err(|err| {
3014        Error::other(format!(
3015            "target durable ILM record is invalid at path `{path}` {}: {err}",
3016            source_record.context()
3017        ))
3018    })?;
3019    source_record
3020        .checkpoint
3021        .validate_successor(&target_record.checkpoint)
3022        .map_err(|err| {
3023            Error::other(format!(
3024                "target durable ILM record generation mismatch at path `{path}` {}: {err}",
3025                source_record.context()
3026            ))
3027        })?;
3028    Ok(target_record)
3029}
3030
3031#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3032#[serde(deny_unknown_fields)]
3033struct DecommissionDurableIlmReceipt {
3034    source_path: String,
3035    namespace: String,
3036    id_kind: String,
3037    id: String,
3038    checkpoint: DurableIlmRecordCheckpoint,
3039    terminal_checkpoint: Option<DurableIlmRecordCheckpoint>,
3040    #[serde(default, skip_serializing_if = "Option::is_none")]
3041    fleet_topology_generation: Option<String>,
3042}
3043
3044impl DecommissionDurableIlmReceipt {
3045    fn new(path: &str, record: &ValidatedDurableIlmRecord, fleet_topology_generation: Option<String>) -> Self {
3046        Self {
3047            source_path: path.to_string(),
3048            namespace: record.namespace.to_string(),
3049            id_kind: record.id_kind.to_string(),
3050            id: record.id.clone(),
3051            checkpoint: record.checkpoint.clone(),
3052            terminal_checkpoint: None,
3053            fleet_topology_generation,
3054        }
3055    }
3056
3057    fn context(&self) -> String {
3058        format!("namespace `{}` {} `{}`", self.namespace, self.id_kind, self.id)
3059    }
3060
3061    fn validate(&self) -> Result<()> {
3062        let namespace = classify_durable_ilm_record(&self.source_path)?
3063            .ok_or_else(|| Error::other(format!("receipt source path `{}` is not a durable ILM record", self.source_path)))?;
3064        if namespace.name != self.namespace {
3065            return Err(Error::other(format!(
3066                "receipt namespace `{}` does not match source path `{}`",
3067                self.namespace, self.source_path
3068            )));
3069        }
3070        if self.id_kind.is_empty() || self.id.is_empty() {
3071            return Err(Error::other(format!(
3072                "receipt identity is missing for source path `{}`",
3073                self.source_path
3074            )));
3075        }
3076        if !is_sha256_checksum(self.checkpoint.content_sha256()) {
3077            return Err(Error::other(format!(
3078                "receipt target checksum is invalid for source path `{}` {}",
3079                self.source_path,
3080                self.context()
3081            )));
3082        }
3083        if let Some(terminal_checkpoint) = &self.terminal_checkpoint {
3084            self.checkpoint.validate_successor(terminal_checkpoint).map_err(|err| {
3085                Error::other(format!(
3086                    "receipt terminal checkpoint is invalid for source path `{}` {}: {err}",
3087                    self.source_path,
3088                    self.context()
3089                ))
3090            })?;
3091        }
3092        if self
3093            .fleet_topology_generation
3094            .as_deref()
3095            .is_some_and(|generation| !is_sha256_checksum(generation))
3096        {
3097            return Err(Error::other_with_context(
3098                "receipt fleet topology generation is invalid",
3099                format!("source path `{}` {}", self.source_path, self.context()),
3100            ));
3101        }
3102        Ok(())
3103    }
3104
3105    fn encode(&self) -> Result<Vec<u8>> {
3106        let mut receipt = self.clone();
3107        receipt.checkpoint = receipt.checkpoint.compacted()?;
3108        receipt.terminal_checkpoint = receipt
3109            .terminal_checkpoint
3110            .as_ref()
3111            .map(DurableIlmRecordCheckpoint::compacted)
3112            .transpose()?;
3113        receipt.validate()?;
3114        let receipt_bytes = serde_json::to_vec(&receipt)?;
3115        let persisted = PersistedDecommissionDurableIlmReceipt {
3116            schema: DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA.to_string(),
3117            content_sha256: hex_sha256(&receipt_bytes, ToOwned::to_owned),
3118            receipt,
3119        };
3120        let encoded = serde_json::to_vec(&persisted)?;
3121        if encoded.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE {
3122            return Err(Error::other(format!(
3123                "durable ILM receipt exceeds maximum size for source path `{}` {}",
3124                self.source_path,
3125                self.context()
3126            )));
3127        }
3128        Ok(encoded)
3129    }
3130
3131    fn decode(data: &[u8]) -> Result<Self> {
3132        if data.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE {
3133            return Err(Error::other("durable ILM receipt exceeds maximum size"));
3134        }
3135        let persisted: PersistedDecommissionDurableIlmReceipt = serde_json::from_slice(data)?;
3136        if persisted.schema != DECOMMISSION_DURABLE_ILM_RECEIPT_SCHEMA {
3137            return Err(Error::other(format!("unsupported durable ILM receipt schema `{}`", persisted.schema)));
3138        }
3139        if !is_sha256_checksum(&persisted.content_sha256) {
3140            return Err(Error::other("durable ILM receipt checksum is invalid"));
3141        }
3142        let receipt_bytes = serde_json::to_vec(&persisted.receipt)?;
3143        let actual_checksum = hex_sha256(&receipt_bytes, ToOwned::to_owned);
3144        if persisted.content_sha256 != actual_checksum {
3145            return Err(Error::other("durable ILM receipt checksum mismatch"));
3146        }
3147        persisted.receipt.validate()?;
3148        Ok(persisted.receipt)
3149    }
3150}
3151
3152fn merge_decommission_durable_ilm_receipts(
3153    existing: &DecommissionDurableIlmReceipt,
3154    incoming: &DecommissionDurableIlmReceipt,
3155) -> Result<DecommissionDurableIlmReceipt> {
3156    if existing.source_path != incoming.source_path
3157        || existing.namespace != incoming.namespace
3158        || existing.id_kind != incoming.id_kind
3159        || existing.id != incoming.id
3160    {
3161        return Err(Error::other(format!(
3162            "durable ILM receipt identity conflict for source path `{}` {}; incoming {}",
3163            existing.source_path,
3164            existing.context(),
3165            incoming.context()
3166        )));
3167    }
3168
3169    let checkpoint =
3170        if existing.checkpoint == incoming.checkpoint || incoming.checkpoint.validate_successor(&existing.checkpoint).is_ok() {
3171            existing.checkpoint.clone()
3172        } else {
3173            existing.checkpoint.validate_successor(&incoming.checkpoint).map_err(|err| {
3174                Error::other(format!(
3175                    "durable ILM receipt checkpoint conflict for source path `{}` {}: {err}",
3176                    existing.source_path,
3177                    existing.context()
3178                ))
3179            })?;
3180            incoming.checkpoint.clone()
3181        };
3182    let terminal_checkpoint = match (&existing.terminal_checkpoint, &incoming.terminal_checkpoint) {
3183        (Some(existing_terminal), Some(incoming_terminal)) if existing_terminal == incoming_terminal => {
3184            Some(existing_terminal.clone())
3185        }
3186        (Some(existing_terminal), Some(incoming_terminal)) if incoming_terminal.validate_successor(existing_terminal).is_ok() => {
3187            Some(existing_terminal.clone())
3188        }
3189        (Some(existing_terminal), Some(incoming_terminal)) => {
3190            existing_terminal.validate_successor(incoming_terminal).map_err(|err| {
3191                Error::other(format!(
3192                    "durable ILM receipt terminal checkpoint conflict for source path `{}` {}: {err}",
3193                    existing.source_path,
3194                    existing.context()
3195                ))
3196            })?;
3197            Some(incoming_terminal.clone())
3198        }
3199        (Some(existing_terminal), None) => Some(existing_terminal.clone()),
3200        (None, Some(incoming_terminal)) => Some(incoming_terminal.clone()),
3201        (None, None) => None,
3202    };
3203    let merged = DecommissionDurableIlmReceipt {
3204        source_path: existing.source_path.clone(),
3205        namespace: existing.namespace.clone(),
3206        id_kind: existing.id_kind.clone(),
3207        id: existing.id.clone(),
3208        checkpoint,
3209        terminal_checkpoint,
3210        fleet_topology_generation: match (
3211            existing.fleet_topology_generation.as_ref(),
3212            incoming.fleet_topology_generation.as_ref(),
3213        ) {
3214            (Some(existing_generation), Some(incoming_generation)) if existing_generation == incoming_generation => {
3215                Some(existing_generation.clone())
3216            }
3217            (None, None) => None,
3218            _ => {
3219                return Err(Error::other_with_context(
3220                    "durable ILM receipt fleet topology conflict",
3221                    format!("source path `{}` {}", existing.source_path, existing.context()),
3222                ));
3223            }
3224        },
3225    };
3226    merged.validate()?;
3227    Ok(merged)
3228}
3229
3230#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3231#[serde(deny_unknown_fields)]
3232struct PersistedDecommissionDurableIlmReceipt {
3233    schema: String,
3234    content_sha256: String,
3235    receipt: DecommissionDurableIlmReceipt,
3236}
3237
3238#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
3239#[serde(deny_unknown_fields)]
3240struct DecommissionDurableIlmManifest {
3241    schema: String,
3242    run_token: String,
3243    receipt_count: u64,
3244    receipt_paths_sha256: String,
3245}
3246
3247impl DecommissionDurableIlmManifest {
3248    fn new(run_token: &str, receipt_paths: &[String]) -> Result<Self> {
3249        let manifest = Self {
3250            schema: DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA.to_string(),
3251            run_token: run_token.to_string(),
3252            receipt_count: u64::try_from(receipt_paths.len())
3253                .map_err(|_| Error::other("durable ILM expected manifest receipt count exceeds u64"))?,
3254            receipt_paths_sha256: decommission_durable_ilm_manifest_paths_sha256(receipt_paths)?,
3255        };
3256        manifest.validate(run_token, receipt_paths)?;
3257        Ok(manifest)
3258    }
3259
3260    fn validate(&self, run_token: &str, receipt_paths: &[String]) -> Result<()> {
3261        if self.schema != DECOMMISSION_DURABLE_ILM_MANIFEST_SCHEMA {
3262            return Err(Error::other(format!(
3263                "unsupported durable ILM expected manifest schema `{}`",
3264                self.schema
3265            )));
3266        }
3267        if self.run_token != run_token || !is_sha256_checksum(&self.run_token) {
3268            return Err(Error::other("durable ILM expected manifest run token is invalid"));
3269        }
3270        let receipt_count = u64::try_from(receipt_paths.len())
3271            .map_err(|_| Error::other("durable ILM expected manifest receipt count exceeds u64"))?;
3272        if self.receipt_count != receipt_count {
3273            return Err(Error::other(format!(
3274                "durable ILM expected manifest receipt count mismatch: expected {}, found {receipt_count}",
3275                self.receipt_count
3276            )));
3277        }
3278        if !is_sha256_checksum(&self.receipt_paths_sha256)
3279            || self.receipt_paths_sha256 != decommission_durable_ilm_manifest_paths_sha256(receipt_paths)?
3280        {
3281            return Err(Error::other("durable ILM expected manifest receipt paths checksum mismatch"));
3282        }
3283        Ok(())
3284    }
3285
3286    fn encode(&self) -> Result<Vec<u8>> {
3287        let encoded = serde_json::to_vec(self)?;
3288        if encoded.len() > DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE {
3289            return Err(Error::other("durable ILM expected manifest exceeds maximum size"));
3290        }
3291        Ok(encoded)
3292    }
3293
3294    fn decode(data: &[u8], run_token: &str, receipt_paths: &[String]) -> Result<Self> {
3295        if data.len() > DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE {
3296            return Err(Error::other("durable ILM expected manifest exceeds maximum size"));
3297        }
3298        let manifest: Self = serde_json::from_slice(data)?;
3299        manifest.validate(run_token, receipt_paths)?;
3300        Ok(manifest)
3301    }
3302}
3303
3304#[derive(Debug, Clone, PartialEq, Eq)]
3305struct DecommissionDurableIlmReceiptLocator {
3306    run_token: String,
3307    source_path: String,
3308    id_kind: String,
3309    id: String,
3310}
3311
3312impl DecommissionDurableIlmReceiptLocator {
3313    fn context(&self) -> String {
3314        format!("source path `{}` {} `{}`", self.source_path, self.id_kind, self.id)
3315    }
3316}
3317
3318fn decommission_durable_ilm_receipt_run_token(cmd_line: &str, start_time: OffsetDateTime) -> String {
3319    let identity = format!("{cmd_line}\0{}", start_time.unix_timestamp_nanos());
3320    hex_sha256(identity.as_bytes(), ToOwned::to_owned)
3321}
3322
3323fn decommission_durable_ilm_receipt_run_prefix(run_token: &str) -> String {
3324    format!("{DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT}/{run_token}/")
3325}
3326
3327fn decommission_durable_ilm_receipt_path(run_token: &str, source_path: &str, id_kind: &str, id: &str) -> String {
3328    format!(
3329        "{}{}/{}/{}.json",
3330        decommission_durable_ilm_receipt_run_prefix(run_token),
3331        source_path,
3332        id_kind,
3333        id
3334    )
3335}
3336
3337fn decommission_durable_ilm_manifest_path(run_token: &str) -> String {
3338    format!("{DECOMMISSION_DURABLE_ILM_MANIFEST_ROOT}/{run_token}.json")
3339}
3340
3341fn decommission_durable_ilm_manifest_paths_sha256(receipt_paths: &[String]) -> Result<String> {
3342    let mut sorted_paths = receipt_paths.iter().map(String::as_str).collect::<Vec<_>>();
3343    sorted_paths.sort_unstable();
3344    let encoded = serde_json::to_vec(&sorted_paths)?;
3345    Ok(hex_sha256(&encoded, ToOwned::to_owned))
3346}
3347
3348fn parse_decommission_durable_ilm_receipt_path(path: &str) -> Result<DecommissionDurableIlmReceiptLocator> {
3349    let prefix = format!("{DECOMMISSION_DURABLE_ILM_RECEIPT_ROOT}/");
3350    let suffix = path
3351        .strip_prefix(&prefix)
3352        .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has the wrong root")))?;
3353    let (run_token, record_path) = suffix
3354        .split_once('/')
3355        .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record path")))?;
3356    let mut parts = record_path.rsplitn(3, '/');
3357    let id = parts
3358        .next()
3359        .and_then(|file| file.strip_suffix(".json"))
3360        .filter(|id| !id.is_empty())
3361        .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its record id")))?;
3362    let id_kind = parts
3363        .next()
3364        .filter(|id_kind| matches!(*id_kind, "operation_id" | "transaction_id" | "job_id" | "control_id"))
3365        .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` has an invalid id kind")))?;
3366    let source_path = parts
3367        .next()
3368        .filter(|source_path| !source_path.is_empty())
3369        .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` is missing its source path")))?;
3370    if !is_sha256_checksum(run_token) {
3371        return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid run token")));
3372    }
3373    match id_kind {
3374        "operation_id" | "control_id" if !is_sha256_checksum(id) => {
3375            let id_label = id_kind.trim_end_matches("_id");
3376            return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid {id_label} id")));
3377        }
3378        "transaction_id" | "job_id" if uuid::Uuid::parse_str(id).is_err() => {
3379            return Err(Error::other(format!("durable ILM receipt path `{path}` has an invalid UUID")));
3380        }
3381        _ => {}
3382    }
3383    classify_durable_ilm_record(source_path)?
3384        .ok_or_else(|| Error::other(format!("durable ILM receipt path `{path}` does not identify a durable ILM source path")))?;
3385    Ok(DecommissionDurableIlmReceiptLocator {
3386        run_token: run_token.to_string(),
3387        source_path: source_path.to_string(),
3388        id_kind: id_kind.to_string(),
3389        id: id.to_string(),
3390    })
3391}
3392
3393fn resolve_decommission_pool_meta_reload_result(result: Result<()>, stage: &str) -> Result<()> {
3394    result.map_err(|err| Error::other(format!("decommission pool meta reload failed during {stage}: {err}")))
3395}
3396
3397fn apply_decommission_status_space_info(mut pool_info: PoolStatus, space_info: PoolSpaceInfo) -> PoolStatus {
3398    match pool_info.decommission.as_mut() {
3399        Some(d) => {
3400            d.total_size = space_info.total;
3401            d.current_size = space_info.free;
3402        }
3403        None => {
3404            pool_info.decommission = Some(PoolDecommissionInfo {
3405                total_size: space_info.total,
3406                current_size: space_info.free,
3407                ..Default::default()
3408            });
3409        }
3410    }
3411
3412    pool_info
3413}
3414
3415fn should_replace_pool_status_for_status_refresh(
3416    current: Option<&PoolStatus>,
3417    persisted: &PoolStatus,
3418    has_active_worker: bool,
3419) -> bool {
3420    let Some(current) = current else {
3421        return true;
3422    };
3423
3424    !has_active_worker && persisted.last_update > current.last_update
3425}
3426
3427fn pool_decommission_movement_snapshot(
3428    info: Option<&PoolDecommissionInfo>,
3429) -> (bool, bool, bool, bool, bool, Option<OffsetDateTime>) {
3430    info.map(|info| {
3431        (
3432            info.has_decommission_state(),
3433            info.complete,
3434            info.failed,
3435            info.canceled,
3436            info.queued,
3437            info.start_time,
3438        )
3439    })
3440    .unwrap_or_default()
3441}
3442
3443pub(crate) fn pool_meta_movement_snapshot_changed(before: &PoolMeta, after: &PoolMeta) -> bool {
3444    before.pools.len() != after.pools.len()
3445        || before.pools.iter().zip(after.pools.iter()).any(|(before, after)| {
3446            pool_decommission_movement_snapshot(before.decommission.as_ref())
3447                != pool_decommission_movement_snapshot(after.decommission.as_ref())
3448        })
3449}
3450
3451/// Merges a persisted pool metadata snapshot into `current` monotonically:
3452/// a pool entry is replaced only when no active worker covers it and the
3453/// snapshot is strictly newer, so delayed snapshots never roll back local
3454/// queued/terminal progressions. Returns whether any entry was replaced or
3455/// appended.
3456pub(crate) fn merge_pool_status_refresh(current: &mut PoolMeta, persisted: PoolMeta, active_workers: &[bool]) -> bool {
3457    let observed_version = current.version.max(persisted.version);
3458    if persisted.pools.is_empty() {
3459        current.version = observed_version;
3460        return false;
3461    }
3462
3463    if current.pools.is_empty() {
3464        *current = persisted;
3465        current.version = observed_version;
3466        return true;
3467    }
3468
3469    current.version = observed_version;
3470
3471    let mut merged_newer = false;
3472    for (idx, persisted_pool) in persisted.pools.into_iter().enumerate() {
3473        if persisted_pool.id != idx {
3474            continue;
3475        }
3476
3477        let has_active_worker = active_workers.get(idx).copied().unwrap_or(false);
3478        if idx < current.pools.len() {
3479            if should_replace_pool_status_for_status_refresh(current.pools.get(idx), &persisted_pool, has_active_worker) {
3480                current.pools[idx] = persisted_pool;
3481                merged_newer = true;
3482            }
3483        } else if idx == current.pools.len() && !has_active_worker {
3484            current.pools.push(persisted_pool);
3485            merged_newer = true;
3486        }
3487    }
3488    merged_newer
3489}
3490
3491fn merge_pool_meta_updates_for_save(
3492    persisted: &mut PoolMeta,
3493    current: &PoolMeta,
3494    indices: &[usize],
3495    operation: &str,
3496) -> Result<()> {
3497    if persisted.pools.is_empty() {
3498        *persisted = current.clone();
3499        return Ok(());
3500    }
3501    if persisted.version != current.version {
3502        return Err(Error::other(format!(
3503            "{operation}: pool metadata version changed from {} to {}",
3504            current.version, persisted.version
3505        )));
3506    }
3507
3508    for &idx in indices {
3509        let current_pool = current
3510            .pools
3511            .get(idx)
3512            .ok_or_else(|| invalid_decommission_pool_index_error(current.pools.len(), idx))?;
3513        let persisted_count = persisted.pools.len();
3514        let persisted_pool = persisted
3515            .pools
3516            .get_mut(idx)
3517            .ok_or_else(|| invalid_decommission_pool_index_error(persisted_count, idx))?;
3518        if current_pool.id != idx || persisted_pool.id != idx || current_pool.cmd_line != persisted_pool.cmd_line {
3519            return Err(Error::other(format!("{operation}: pool metadata layout changed for pool {idx}")));
3520        }
3521        let current_clears_decommission = current_pool
3522            .decommission
3523            .as_ref()
3524            .is_none_or(|info| !info.has_decommission_state());
3525        if current_clears_decommission
3526            && persisted_pool.decommission.as_ref().is_some_and(|info| {
3527                info.has_decommission_state() && is_decommission_active(info.complete, info.failed, info.canceled)
3528            })
3529        {
3530            record_pool_meta_stale_write_rejection("nonterminal_decommission_clear");
3531            return Err(Error::other(format!(
3532                "{operation}: stale pool metadata update rejected for pool {idx}; persisted active or queued decommission cannot be cleared"
3533            )));
3534        }
3535        if current_clears_decommission
3536            && persisted_pool
3537                .decommission
3538                .as_ref()
3539                .is_some_and(|info| info.complete || !info.unresolved_entries.is_empty())
3540        {
3541            record_pool_meta_stale_write_rejection("unsafe_terminal_decommission_clear");
3542            return Err(Error::StalePoolMetadataUpdate {
3543                operation: operation.to_string(),
3544                pool_index: idx,
3545                reason: "completed or unresolved decommission state cannot be cleared",
3546            });
3547        }
3548        if current_pool.last_update < persisted_pool.last_update {
3549            record_pool_meta_stale_write_rejection("older_pool_revision");
3550            return Err(Error::other(format!(
3551                "{operation}: stale pool metadata update rejected for pool {idx}; persisted update is newer"
3552            )));
3553        }
3554        if let (Some(persisted_info), Some(current_info)) =
3555            (persisted_pool.decommission.as_ref(), current_pool.decommission.as_ref())
3556            && current_info.has_decommission_state()
3557        {
3558            let persisted_terminal = (persisted_info.complete, persisted_info.failed, persisted_info.canceled);
3559            let current_terminal = (current_info.complete, current_info.failed, current_info.canceled);
3560            if persisted_terminal != (false, false, false) && persisted_terminal != current_terminal {
3561                record_pool_meta_stale_write_rejection("terminal_state_regression");
3562                return Err(Error::other(format!(
3563                    "{operation}: stale pool metadata update rejected for pool {idx}; terminal state cannot be replaced"
3564                )));
3565            }
3566            if current_info.items_decommissioned < persisted_info.items_decommissioned
3567                || current_info.items_decommission_failed < persisted_info.items_decommission_failed
3568                || current_info.bytes_done < persisted_info.bytes_done
3569                || current_info.bytes_failed < persisted_info.bytes_failed
3570            {
3571                record_pool_meta_stale_write_rejection("progress_regression");
3572                return Err(Error::other(format!(
3573                    "{operation}: stale pool metadata update rejected for pool {idx}; durable progress cannot decrease"
3574                )));
3575            }
3576        }
3577        *persisted_pool = current_pool.clone();
3578    }
3579
3580    Ok(())
3581}
3582
3583fn publish_pool_meta_updates(current: &mut PoolMeta, saved: &PoolMeta, indices: &[usize]) {
3584    current.version = current.version.max(saved.version);
3585    for &idx in indices {
3586        let Some(saved_pool) = saved.pools.get(idx) else {
3587            continue;
3588        };
3589        let Some(current_pool) = current.pools.get_mut(idx) else {
3590            continue;
3591        };
3592        if current_pool.id == idx && saved_pool.id == idx && current_pool.cmd_line == saved_pool.cmd_line {
3593            *current_pool = saved_pool.clone();
3594        }
3595    }
3596}
3597
3598/// Callers retain the save gate and namespace fence from the durable read/save
3599/// through publication, so capacity snapshots cannot publish out of order.
3600fn publish_decommission_capacity_update(current: &mut PoolMeta, saved: &PoolMeta, idx: usize) -> Result<()> {
3601    let stale = |reason| Error::StalePoolMetadataUpdate {
3602        operation: "publish decommission capacity".to_string(),
3603        pool_index: idx,
3604        reason,
3605    };
3606    let saved_pool = saved
3607        .pools
3608        .get(idx)
3609        .ok_or_else(|| invalid_decommission_pool_index_error(saved.pools.len(), idx))?;
3610    let current_count = current.pools.len();
3611    let pool = current
3612        .pools
3613        .get_mut(idx)
3614        .ok_or_else(|| invalid_decommission_pool_index_error(current_count, idx))?;
3615    if pool.id != idx || saved_pool.id != idx || pool.cmd_line != saved_pool.cmd_line || saved.version < current.version {
3616        return Err(stale("pool identity changed before capacity publication"));
3617    }
3618    let info = pool
3619        .decommission
3620        .as_mut()
3621        .ok_or_else(|| stale("decommission cleared before capacity publication"))?;
3622    let saved_info = saved_pool
3623        .decommission
3624        .as_ref()
3625        .ok_or_else(|| stale("persisted decommission is missing during capacity publication"))?;
3626    if (info.start_time, info.queued, info.complete, info.failed, info.canceled)
3627        != (
3628            saved_info.start_time,
3629            saved_info.queued,
3630            saved_info.complete,
3631            saved_info.failed,
3632            saved_info.canceled,
3633        )
3634    {
3635        return Err(stale("decommission generation or terminal state changed before capacity publication"));
3636    }
3637    let identity = |reservation: &DecommissionCapacityReservation| {
3638        (
3639            reservation.source_pool_index,
3640            reservation.operation_id,
3641            reservation.generation,
3642            reservation.owner_nonce,
3643            reservation.model_version,
3644        )
3645    };
3646    if info.capacity_reservation.as_ref().map(identity) != saved_info.capacity_reservation.as_ref().map(identity) {
3647        return Err(stale("decommission capacity owner changed before publication"));
3648    }
3649
3650    // Capacity transactions do not checkpoint the worker's counters or queue.
3651    // Keep their clock separate before advancing the shared pool revision.
3652    info.progress_save_last_at.get_or_insert(pool.last_update);
3653    info.capacity_reservation = saved_info.capacity_reservation.clone();
3654    info.capacity_blocked_reason = saved_info.capacity_blocked_reason.clone();
3655    pool.last_update = pool.last_update.max(saved_pool.last_update);
3656    current.version = current.version.max(saved.version);
3657    Ok(())
3658}
3659
3660fn resolve_start_decommission_pool_meta_reload_result(result: Result<()>) -> Result<()> {
3661    resolve_decommission_pool_meta_reload_result(result, "start_decommission")
3662}
3663
3664fn activation_rebalance_meta_lock_error(err: rustfs_lock::LockError) -> Error {
3665    match err {
3666        rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
3667            mode: "write",
3668            bucket: RUSTFS_META_BUCKET.to_string(),
3669            object: REBAL_META_NAME.to_string(),
3670            required,
3671            achieved,
3672        },
3673        other => Error::other(format!(
3674            "failed to acquire rebalance activation lock on {RUSTFS_META_BUCKET}/{REBAL_META_NAME}: {other}"
3675        )),
3676    }
3677}
3678
3679fn activation_pool_meta_lock_error(err: rustfs_lock::LockError) -> Error {
3680    match err {
3681        rustfs_lock::LockError::QuorumNotReached { required, achieved } => Error::NamespaceLockQuorumUnavailable {
3682            mode: "write",
3683            bucket: RUSTFS_META_BUCKET.to_string(),
3684            object: POOL_META_NAME.to_string(),
3685            required,
3686            achieved,
3687        },
3688        other => Error::other(format!(
3689            "failed to acquire pool activation lock on {RUSTFS_META_BUCKET}/{POOL_META_NAME}: {other}"
3690        )),
3691    }
3692}
3693
3694pub(crate) struct PoolRebalanceActivationFence {
3695    pool_meta_guard: rustfs_lock::NamespaceLockGuard,
3696    rebalance_meta_guard: rustfs_lock::NamespaceLockGuard,
3697    fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
3698    decommission_target_fence_proof: Option<crate::services::notification_sys::DecommissionTargetFenceFleetProofToken>,
3699    #[cfg(test)]
3700    forced_lost: Arc<AtomicBool>,
3701}
3702
3703impl PoolRebalanceActivationFence {
3704    pub(crate) fn set_fleet_proof(
3705        &mut self,
3706        fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
3707    ) {
3708        self.fleet_proof = fleet_proof;
3709    }
3710
3711    fn set_decommission_target_fence_proof(
3712        &mut self,
3713        fleet_proof: Option<crate::services::notification_sys::DecommissionTargetFenceFleetProofToken>,
3714    ) {
3715        self.decommission_target_fence_proof = fleet_proof;
3716    }
3717
3718    pub(crate) fn ensure_held(&self) -> Result<()> {
3719        #[cfg(test)]
3720        let forced_lost = self.forced_lost.load(Ordering::Acquire);
3721        #[cfg(not(test))]
3722        let forced_lost = false;
3723        if forced_lost || self.pool_meta_guard.is_lock_lost() || self.rebalance_meta_guard.is_lock_lost() {
3724            return Err(Error::other("activation lock lost before metadata commit or worker admission"));
3725        }
3726        if self
3727            .fleet_proof
3728            .as_ref()
3729            .is_some_and(|proof| !crate::services::notification_sys::cross_pool_fence_fleet_proof_matches(proof))
3730        {
3731            return Err(Error::other(POOL_ACTIVATION_FLEET_PROOF_EXPIRED));
3732        }
3733        if self
3734            .decommission_target_fence_proof
3735            .as_ref()
3736            .is_some_and(|proof| !crate::services::notification_sys::decommission_target_fence_fleet_proof_matches(proof))
3737        {
3738            return Err(Error::other(DECOMMISSION_TARGET_FLEET_PROOF_EXPIRED));
3739        }
3740
3741        Ok(())
3742    }
3743
3744    pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
3745        opts.add_namespace_lock_guard(&self.pool_meta_guard);
3746        opts.add_namespace_lock_guard(&self.rebalance_meta_guard);
3747    }
3748
3749    #[cfg(test)]
3750    fn force_lost_for_test(&self) {
3751        self.forced_lost.store(true, Ordering::Release);
3752    }
3753}
3754
3755#[cfg(test)]
3756tokio::task_local! {
3757    pub(crate) static REBALANCE_ACTIVATION_LOCK_ATTEMPT: Arc<tokio::sync::Notify>;
3758}
3759
3760pub(crate) async fn acquire_pool_rebalance_activation_locks<S>(
3761    pool: Arc<S>,
3762    fleet_proof: Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>,
3763) -> Result<PoolRebalanceActivationFence>
3764where
3765    S: crate::storage_api_contracts::namespace::NamespaceLocking<
3766            Error = Error,
3767            NamespaceLock = rustfs_lock::NamespaceLockWrapper,
3768        >,
3769{
3770    // Match entry admission: rebalance.bin -> pool.bin. An entry retains its
3771    // run read fence while target mutations acquire the pool metadata fence;
3772    // activation must not hold pool.bin while waiting for that entry to drain.
3773    let rebalance_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, REBAL_META_NAME).await?;
3774    #[cfg(test)]
3775    let _ = REBALANCE_ACTIVATION_LOCK_ATTEMPT.try_with(|attempted| attempted.notify_one());
3776    let rebalance_meta_guard = rebalance_meta_lock
3777        .get_write_lock(get_lock_acquire_timeout())
3778        .await
3779        .map_err(activation_rebalance_meta_lock_error)?;
3780    let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
3781    let pool_meta_guard = pool_meta_lock
3782        .get_write_lock(get_lock_acquire_timeout())
3783        .await
3784        .map_err(activation_pool_meta_lock_error)?;
3785
3786    Ok(PoolRebalanceActivationFence {
3787        pool_meta_guard,
3788        rebalance_meta_guard,
3789        fleet_proof,
3790        decommission_target_fence_proof: None,
3791        #[cfg(test)]
3792        forced_lost: Arc::new(AtomicBool::new(false)),
3793    })
3794}
3795
3796pub(crate) async fn acquire_pool_activation_fleet_proof(
3797    ctx: &crate::runtime::instance::InstanceContext,
3798) -> Result<Option<crate::services::notification_sys::CrossPoolFenceFleetProofToken>> {
3799    if !ctx.is_dist_erasure().await {
3800        return Ok(None);
3801    }
3802    crate::services::notification_sys::acquire_cross_pool_fence_fleet_proof()
3803        .map(Some)
3804        .ok_or_else(|| Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED))
3805}
3806
3807pub fn is_pool_activation_fleet_proof_error(err: &Error) -> bool {
3808    // Save-stage helpers add context by formatting the original error, so the
3809    // marker may be nested in the display string. Restrict matching to the
3810    // `Error::other` I/O shape used by this activation path.
3811    matches!(err, Error::Io(io_error) if io_error.kind() == std::io::ErrorKind::Other && {
3812        let message = io_error.to_string();
3813        message.contains(POOL_ACTIVATION_FLEET_PROOF_REQUIRED) || message.contains(POOL_ACTIVATION_FLEET_PROOF_EXPIRED)
3814    })
3815}
3816
3817#[cfg(test)]
3818#[derive(Clone, Copy, Debug, Eq, PartialEq)]
3819pub(crate) enum PoolActivationStartKind {
3820    Rebalance,
3821    Decommission,
3822}
3823
3824#[cfg(test)]
3825struct PoolActivationDurableSaveBarrierState {
3826    pool_key: usize,
3827    arrived: tokio::sync::Notify,
3828    release: tokio::sync::Notify,
3829    force_fence_loss: AtomicBool,
3830}
3831
3832#[cfg(test)]
3833static POOL_ACTIVATION_DURABLE_SAVE_BARRIER: std::sync::OnceLock<
3834    std::sync::Mutex<Option<Arc<PoolActivationDurableSaveBarrierState>>>,
3835> = std::sync::OnceLock::new();
3836
3837#[cfg(test)]
3838pub(crate) struct PoolActivationDurableSaveBarrier {
3839    state: Arc<PoolActivationDurableSaveBarrierState>,
3840}
3841
3842#[cfg(test)]
3843fn pool_activation_test_pool_key<S>(pool: &Arc<S>) -> usize {
3844    Arc::as_ptr(pool).cast::<()>() as usize
3845}
3846
3847#[cfg(test)]
3848impl PoolActivationDurableSaveBarrier {
3849    pub(crate) fn install<S>(pool: &Arc<S>) -> Self {
3850        let state = Arc::new(PoolActivationDurableSaveBarrierState {
3851            pool_key: pool_activation_test_pool_key(pool),
3852            arrived: tokio::sync::Notify::new(),
3853            release: tokio::sync::Notify::new(),
3854            force_fence_loss: AtomicBool::new(false),
3855        });
3856        let mut barrier = POOL_ACTIVATION_DURABLE_SAVE_BARRIER
3857            .get_or_init(|| std::sync::Mutex::new(None))
3858            .lock()
3859            .expect("pool activation durable save barrier should not be poisoned");
3860        assert!(barrier.is_none(), "pool activation durable save barrier must be unique");
3861        *barrier = Some(Arc::clone(&state));
3862        Self { state }
3863    }
3864
3865    pub(crate) async fn wait_until_paused(&self) {
3866        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.arrived.notified())
3867            .await
3868            .expect("activation should reach the post-durable-save barrier");
3869    }
3870
3871    pub(crate) fn release_after_fence_loss(&self) {
3872        self.state.force_fence_loss.store(true, Ordering::Release);
3873        self.state.release.notify_one();
3874    }
3875
3876    pub(crate) fn release_without_fence_loss(&self) {
3877        self.state.release.notify_one();
3878    }
3879}
3880
3881#[cfg(test)]
3882impl Drop for PoolActivationDurableSaveBarrier {
3883    fn drop(&mut self) {
3884        self.state.release.notify_one();
3885        let mut barrier = POOL_ACTIVATION_DURABLE_SAVE_BARRIER
3886            .get_or_init(|| std::sync::Mutex::new(None))
3887            .lock()
3888            .expect("pool activation durable save barrier should not be poisoned");
3889        if barrier.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
3890            *barrier = None;
3891        }
3892    }
3893}
3894
3895#[cfg(test)]
3896pub(crate) async fn pause_pool_activation_after_durable_save<S>(pool: &Arc<S>, fence: &PoolRebalanceActivationFence) {
3897    let pool_key = pool_activation_test_pool_key(pool);
3898    let barrier = {
3899        let mut barrier = POOL_ACTIVATION_DURABLE_SAVE_BARRIER
3900            .get_or_init(|| std::sync::Mutex::new(None))
3901            .lock()
3902            .expect("pool activation durable save barrier should not be poisoned");
3903        if barrier.as_ref().is_some_and(|state| state.pool_key == pool_key) {
3904            barrier.take()
3905        } else {
3906            None
3907        }
3908    };
3909    if let Some(barrier) = barrier {
3910        barrier.arrived.notify_one();
3911        barrier.release.notified().await;
3912        if barrier.force_fence_loss.load(Ordering::Acquire) {
3913            fence.force_lost_for_test();
3914        }
3915    }
3916}
3917
3918#[cfg(test)]
3919struct PoolActivationStartProbeState {
3920    kind: PoolActivationStartKind,
3921    preflight_side_effect_attempted: std::sync::atomic::AtomicBool,
3922    attempted: std::sync::atomic::AtomicBool,
3923    notify: tokio::sync::Notify,
3924}
3925
3926#[cfg(test)]
3927static POOL_ACTIVATION_START_PROBES: std::sync::OnceLock<std::sync::Mutex<Vec<Arc<PoolActivationStartProbeState>>>> =
3928    std::sync::OnceLock::new();
3929
3930#[cfg(test)]
3931pub(crate) struct PoolActivationStartProbe {
3932    state: Arc<PoolActivationStartProbeState>,
3933}
3934
3935#[cfg(test)]
3936impl PoolActivationStartProbe {
3937    pub(crate) fn install(kind: PoolActivationStartKind) -> Self {
3938        let state = Arc::new(PoolActivationStartProbeState {
3939            kind,
3940            preflight_side_effect_attempted: std::sync::atomic::AtomicBool::new(false),
3941            attempted: std::sync::atomic::AtomicBool::new(false),
3942            notify: tokio::sync::Notify::new(),
3943        });
3944        POOL_ACTIVATION_START_PROBES
3945            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
3946            .lock()
3947            .expect("pool activation start probe should not be poisoned")
3948            .push(Arc::clone(&state));
3949        Self { state }
3950    }
3951
3952    pub(crate) async fn wait_until_attempted(&self) {
3953        while !self.state.attempted.load(Ordering::Acquire) {
3954            self.state.notify.notified().await;
3955        }
3956    }
3957
3958    pub(crate) fn preflight_side_effect_was_attempted(&self) -> bool {
3959        self.state.preflight_side_effect_attempted.load(Ordering::Acquire)
3960    }
3961
3962    pub(crate) fn activation_was_attempted(&self) -> bool {
3963        self.state.attempted.load(Ordering::Acquire)
3964    }
3965}
3966
3967#[cfg(test)]
3968impl Drop for PoolActivationStartProbe {
3969    fn drop(&mut self) {
3970        let mut probes = POOL_ACTIVATION_START_PROBES
3971            .get_or_init(|| std::sync::Mutex::new(Vec::new()))
3972            .lock()
3973            .expect("pool activation start probe should not be poisoned");
3974        probes.retain(|state| !Arc::ptr_eq(state, &self.state));
3975    }
3976}
3977
3978#[cfg(test)]
3979pub(crate) fn observe_pool_activation_start_attempt(kind: PoolActivationStartKind) {
3980    let probes = POOL_ACTIVATION_START_PROBES
3981        .get_or_init(|| std::sync::Mutex::new(Vec::new()))
3982        .lock()
3983        .expect("pool activation start probe should not be poisoned")
3984        .iter()
3985        .filter(|state| state.kind == kind)
3986        .cloned()
3987        .collect::<Vec<_>>();
3988    for state in probes {
3989        state.attempted.store(true, Ordering::Release);
3990        state.notify.notify_one();
3991    }
3992}
3993
3994#[cfg(test)]
3995fn observe_pool_activation_preflight_side_effect_attempt(kind: PoolActivationStartKind) {
3996    let probes = POOL_ACTIVATION_START_PROBES
3997        .get_or_init(|| std::sync::Mutex::new(Vec::new()))
3998        .lock()
3999        .expect("pool activation start probe should not be poisoned")
4000        .iter()
4001        .filter(|state| state.kind == kind)
4002        .cloned()
4003        .collect::<Vec<_>>();
4004    for state in probes {
4005        state.preflight_side_effect_attempted.store(true, Ordering::Release);
4006    }
4007}
4008
4009fn rollback_decommission_pool_meta(pool_meta: &mut PoolMeta, previous_pool_meta: &PoolMeta, indices: &[usize]) {
4010    publish_pool_meta_updates(pool_meta, previous_pool_meta, indices);
4011}
4012
4013#[derive(Debug)]
4014struct DecommissionCancelCommit {
4015    previous_start_time: Option<OffsetDateTime>,
4016    previous_queued: bool,
4017    previous_last_update: OffsetDateTime,
4018    canceled_pool: PoolStatus,
4019}
4020
4021fn commit_decommission_cancel(pool_meta: &mut PoolMeta, idx: usize, commit: DecommissionCancelCommit) -> Result<()> {
4022    let pool_count = pool_meta.pools.len();
4023    let Some(current) = pool_meta.pools.get(idx) else {
4024        return Err(invalid_decommission_pool_index_error(pool_count, idx));
4025    };
4026    // A peer reload can install the saved cancel while runtime-only fields are reconstructed.
4027    let cancel_already_published = commit
4028        .canceled_pool
4029        .decommission
4030        .as_ref()
4031        .is_some_and(|info| info.canceled && !info.complete && !info.failed && info.start_time.is_none())
4032        && PersistedPoolStatus::from(current) == PersistedPoolStatus::from(&commit.canceled_pool);
4033    if cancel_already_published {
4034        return Ok(());
4035    }
4036
4037    let matches_generation = current.id == commit.canceled_pool.id
4038        && current.cmd_line == commit.canceled_pool.cmd_line
4039        && current.decommission.as_ref().is_some_and(|info| {
4040            info.start_time == commit.previous_start_time
4041                && info.queued == commit.previous_queued
4042                && is_decommission_active(info.complete, info.failed, info.canceled)
4043                && (commit.previous_start_time.is_some() || current.last_update == commit.previous_last_update)
4044        });
4045    if !matches_generation {
4046        return Err(Error::other(format!(
4047            "failed to publish decommission cancel for pool {idx}: operation generation changed"
4048        )));
4049    }
4050
4051    pool_meta.pools[idx] = commit.canceled_pool;
4052    Ok(())
4053}
4054
4055fn rollback_start_decommission_pool_meta(pool_meta: &mut PoolMeta, previous_pool_meta: &PoolMeta, indices: &[usize]) {
4056    let active_updates = indices
4057        .iter()
4058        .filter_map(|&idx| pool_meta.pools.get(idx).map(|pool| (idx, pool.last_update)))
4059        .collect::<Vec<_>>();
4060    rollback_decommission_pool_meta(pool_meta, previous_pool_meta, indices);
4061    let rollback_at = OffsetDateTime::now_utc();
4062    for (idx, active_update) in active_updates {
4063        if let Some(pool) = pool_meta.pools.get_mut(idx) {
4064            pool.last_update = std::cmp::max(rollback_at, active_update + Duration::nanoseconds(1));
4065        }
4066    }
4067}
4068
4069fn ensure_pool_meta_write_fence(guard: &rustfs_lock::NamespaceLockGuard, operation: &str) -> Result<()> {
4070    if guard.is_lock_lost() {
4071        return Err(Error::other(format!("{operation}: pool metadata distributed fence was lost")));
4072    }
4073    Ok(())
4074}
4075
4076fn ensure_pool_not_left_in_cmdline_after_decommission(position: usize, cmd_line: &str, completed: bool) -> Result<()> {
4077    if completed {
4078        return Err(Error::other(format!(
4079            "pool({}) = {} is decommissioned, please remove from server command line",
4080            position + 1,
4081            cmd_line
4082        )));
4083    }
4084
4085    Ok(())
4086}
4087
4088fn resolve_decommission_listing_worker_result(
4089    set_idx: usize,
4090    worker_result: std::result::Result<Result<()>, tokio::task::JoinError>,
4091) -> Result<()> {
4092    worker_result.map_err(|err| Error::other(format!("decommission listing worker {set_idx} task join error: {err}")))?
4093}
4094
4095/// A bucket set may only finish once every deferred entry has been resolved.
4096///
4097/// The deferred round is the last chance for a contended entry; leaving one
4098/// unresolved would mark the bucket done and drop the object. This is reported
4099/// as a capacity failure so the pool takes the existing durable-pause path
4100/// instead of a terminal failure.
4101fn ensure_decommission_deferred_drained(bucket: &str, set_idx: usize, deferred_remaining: usize) -> Result<()> {
4102    if deferred_remaining == 0 {
4103        return Ok(());
4104    }
4105    Err(decommission_capacity_blocked_error(format!(
4106        "decommission bucket {bucket} set {set_idx} finished with {deferred_remaining} deferred entries remaining"
4107    )))
4108}
4109
4110fn decommission_entry_budget_acquire_error(err: impl Display) -> Error {
4111    Error::other(format!("decommission entry budget permit acquire failed: {err}"))
4112}
4113
4114/// Collapse a single-entry test invocation into a plain result.
4115///
4116/// A deferred outcome means the entry still needs another round, which the test
4117/// callers do not run; surface it as the contention failure that caused it.
4118#[cfg(test)]
4119fn resolve_decommission_entry_test_outcome(outcome: Result<DecommissionEntryOutcome>) -> Result<()> {
4120    match outcome? {
4121        DecommissionEntryOutcome::Complete => Ok(()),
4122        DecommissionEntryOutcome::Deferred(err) => Err(err),
4123    }
4124}
4125
4126fn should_retry_decommission_listing(err: &Error, attempt: usize, max_attempts: usize) -> bool {
4127    !is_err_bucket_not_found(err) && attempt + 1 < max_attempts
4128}
4129
4130async fn wait_decommission_retry_backoff(rx: &CancellationToken, delay: std::time::Duration) -> bool {
4131    tokio::select! {
4132        _ = rx.cancelled() => true,
4133        _ = tokio::time::sleep(delay) => false,
4134    }
4135}
4136
4137/// Exponential backoff ceiling for a 1-based `attempt`, capped so a long
4138/// contention streak cannot turn into a multi-minute stall.
4139fn decommission_retry_backoff_ceiling(base: std::time::Duration, attempt: usize) -> std::time::Duration {
4140    let exponent = u32::try_from(attempt.saturating_sub(1)).unwrap_or(u32::MAX).min(31);
4141    base.saturating_mul(1u32 << exponent).min(DECOMMISSION_RETRY_BACKOFF_CAP)
4142}
4143
4144/// Full-jitter exponential decommission retry backoff.
4145///
4146/// Racing workers keep observing the same target capacity gate. A shared fixed
4147/// delay makes them collide again after every round, so each retry draws a
4148/// uniform delay from `[0, ceiling]` instead.
4149fn decommission_retry_backoff_delay(base: std::time::Duration, attempt: usize) -> std::time::Duration {
4150    let ceiling = decommission_retry_backoff_ceiling(base, attempt);
4151    if ceiling.is_zero() {
4152        return ceiling;
4153    }
4154    let ceiling_millis = u64::try_from(ceiling.as_millis()).unwrap_or(u64::MAX);
4155    std::time::Duration::from_millis(rand::rng().random_range(0..=ceiling_millis))
4156}
4157
4158/// Backoff for retries that wait on another mutation's target capacity gate.
4159pub(crate) fn decommission_capacity_gate_retry_delay(attempt: usize) -> std::time::Duration {
4160    let delay = decommission_retry_backoff_delay(DECOMMISSION_CAPACITY_GATE_RETRY_BASE, attempt);
4161    record_decommission_capacity_gate_wait(delay);
4162    delay
4163}
4164
4165#[cfg(test)]
4166async fn run_decommission_listing_with_retry<List, ListFuture>(
4167    rx: CancellationToken,
4168    bucket: String,
4169    cb: ListCallback,
4170    pool_idx: usize,
4171    set_idx: usize,
4172    max_attempts: usize,
4173    list: List,
4174) -> Result<()>
4175where
4176    List: FnMut(ListCallback) -> ListFuture,
4177    ListFuture: std::future::Future<Output = Result<()>>,
4178{
4179    run_decommission_listing_with_retry_and_drain(rx, bucket, cb, pool_idx, set_idx, max_attempts, list, || async { false }).await
4180}
4181
4182#[allow(clippy::too_many_arguments)]
4183async fn run_decommission_listing_with_retry_and_drain<List, ListFuture, Drain, DrainFuture>(
4184    rx: CancellationToken,
4185    bucket: String,
4186    cb: ListCallback,
4187    pool_idx: usize,
4188    set_idx: usize,
4189    max_attempts: usize,
4190    mut list: List,
4191    mut drain: Drain,
4192) -> Result<()>
4193where
4194    List: FnMut(ListCallback) -> ListFuture,
4195    ListFuture: std::future::Future<Output = Result<()>>,
4196    Drain: FnMut() -> DrainFuture,
4197    DrainFuture: std::future::Future<Output = bool>,
4198{
4199    let max_attempts = max_attempts.max(1);
4200
4201    for attempt in 0..max_attempts {
4202        if rx.is_cancelled() {
4203            debug!(
4204                event = EVENT_DECOMMISSION_BUCKET,
4205                component = LOG_COMPONENT_ECSTORE,
4206                subsystem = LOG_SUBSYSTEM_POOLS,
4207                pool_index = pool_idx,
4208                set_index = set_idx,
4209                bucket = %bucket,
4210                state = "listing_worker_cancelled",
4211                "Decommission listing worker cancelled"
4212            );
4213            return Ok(());
4214        }
4215
4216        debug!(
4217            event = EVENT_DECOMMISSION_BUCKET,
4218            component = LOG_COMPONENT_ECSTORE,
4219            subsystem = LOG_SUBSYSTEM_POOLS,
4220            pool_index = pool_idx,
4221            set_index = set_idx,
4222            bucket = %bucket,
4223            attempt = attempt + 1,
4224            max_attempts,
4225            state = "listing_started",
4226            "Decommission listing started"
4227        );
4228
4229        let list_result = list(cb.clone()).await;
4230        if drain().await {
4231            return Ok(());
4232        }
4233
4234        match list_result {
4235            Ok(()) => {
4236                debug!(
4237                    event = EVENT_DECOMMISSION_BUCKET,
4238                    component = LOG_COMPONENT_ECSTORE,
4239                    subsystem = LOG_SUBSYSTEM_POOLS,
4240                    pool_index = pool_idx,
4241                    set_index = set_idx,
4242                    bucket = %bucket,
4243                    attempt = attempt + 1,
4244                    max_attempts,
4245                    state = "listing_completed",
4246                    "Decommission listing completed"
4247                );
4248                return Ok(());
4249            }
4250            Err(err) if is_err_bucket_not_found(&err) => {
4251                warn!(
4252                    event = EVENT_DECOMMISSION_BUCKET,
4253                    component = LOG_COMPONENT_ECSTORE,
4254                    subsystem = LOG_SUBSYSTEM_POOLS,
4255                    pool_index = pool_idx,
4256                    set_index = set_idx,
4257                    bucket = %bucket,
4258                    attempt = attempt + 1,
4259                    max_attempts,
4260                    state = "listing_bucket_missing",
4261                    "Decommission listing bucket missing"
4262                );
4263                return Ok(());
4264            }
4265            Err(err) if should_retry_decommission_listing(&err, attempt, max_attempts) => {
4266                error!(
4267                    event = EVENT_DECOMMISSION_BUCKET,
4268                    component = LOG_COMPONENT_ECSTORE,
4269                    subsystem = LOG_SUBSYSTEM_POOLS,
4270                    pool_index = pool_idx,
4271                    set_index = set_idx,
4272                    bucket = %bucket,
4273                    attempt = attempt + 1,
4274                    max_attempts,
4275                    retry_delay_ms = DECOMMISSION_LISTING_RETRY_DELAY.as_millis(),
4276                    state = "listing_failed_retrying",
4277                    error = ?err,
4278                    "Decommission listing failed; retrying"
4279                );
4280                if wait_decommission_retry_backoff(&rx, DECOMMISSION_LISTING_RETRY_DELAY).await {
4281                    debug!(
4282                        event = EVENT_DECOMMISSION_BUCKET,
4283                        component = LOG_COMPONENT_ECSTORE,
4284                        subsystem = LOG_SUBSYSTEM_POOLS,
4285                        pool_index = pool_idx,
4286                        set_index = set_idx,
4287                        bucket = %bucket,
4288                        state = "listing_worker_cancelled",
4289                        "Decommission listing worker cancelled during retry wait"
4290                    );
4291                    return Ok(());
4292                }
4293            }
4294            Err(err) => {
4295                error!(
4296                    event = EVENT_DECOMMISSION_BUCKET,
4297                    component = LOG_COMPONENT_ECSTORE,
4298                    subsystem = LOG_SUBSYSTEM_POOLS,
4299                    pool_index = pool_idx,
4300                    set_index = set_idx,
4301                    bucket = %bucket,
4302                    attempt = attempt + 1,
4303                    max_attempts,
4304                    state = "listing_failed",
4305                    error = ?err,
4306                    "Decommission listing failed"
4307                );
4308                return Err(Error::other(format!(
4309                    "decommission listing failed for bucket {bucket} pool {pool_idx} set {set_idx} attempt {}/{}: {err}",
4310                    attempt + 1,
4311                    max_attempts
4312                )));
4313            }
4314        }
4315    }
4316
4317    Ok(())
4318}
4319
4320fn should_count_decommission_version_complete(ignore: bool, cleanup_ignored: bool, failure: bool) -> bool {
4321    cleanup_ignored || (!ignore && !failure)
4322}
4323
4324fn is_decommission_copy_cleanup_safe_error(err: &Error) -> bool {
4325    // DataMovementOverwriteErr only means source and destination pool resolved to
4326    // the same pool. Without a target equivalence check it is not cleanup-safe.
4327    if is_err_object_not_found(err) || is_err_version_not_found(err) {
4328        return true;
4329    }
4330
4331    // A not-found surfacing from inside a data-movement stage is the same
4332    // condition once the wrapper is unwrapped (backlog#1827 T2).
4333    data_movement::data_movement_stage_source(err).is_some_and(is_decommission_copy_cleanup_safe_error)
4334}
4335
4336fn is_decommission_target_capacity_error(err: &Error) -> bool {
4337    if matches!(err, Error::DiskFull | Error::StorageFull) {
4338        return true;
4339    }
4340
4341    // A stage failure keeps the error it wrapped, so classify by type rather
4342    // than by the rendered message (backlog#1827 T2). The substring fallback
4343    // stays for errors that reached here through some other wrapper.
4344    if let Some(source) = data_movement::data_movement_stage_source(err) {
4345        return is_decommission_target_capacity_error(source);
4346    }
4347
4348    let message = err.to_string();
4349    let disk_full = Error::DiskFull.to_string();
4350    let storage_full = Error::StorageFull.to_string();
4351    message.contains(&disk_full) || message.contains(&storage_full)
4352}
4353
4354fn should_cleanup_decommission_source_entry(decommissioned: usize, total_versions: usize, expired: usize) -> bool {
4355    decommissioned.saturating_add(expired) == total_versions
4356}
4357
4358fn should_fail_decommission_pool_after_exhausted_source_changed(exhausted_entries: usize) -> bool {
4359    exhausted_entries > DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT
4360}
4361
4362#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4363enum DecommissionEntryAttemptOutcome {
4364    Complete,
4365    SourceChanged,
4366}
4367
4368/// Compact retry state retained until the listing has drained.
4369#[derive(Debug)]
4370struct DeferredDecommissionEntry {
4371    name: String,
4372    attempts: usize,
4373}
4374
4375/// Terminal result of a decommission entry pass.
4376#[derive(Debug)]
4377enum DecommissionEntryOutcome {
4378    Complete,
4379    /// The entry is still contended on a transient capacity condition after its
4380    /// attempt budget. It must be retried at the end of the current round
4381    /// instead of failing the whole bucket. The carried error is the last
4382    /// contention failure, which becomes terminal only if the deferred round
4383    /// cannot make progress either.
4384    Deferred(Error),
4385}
4386
4387/// Decide what a terminal entry-attempt failure means for the current round.
4388///
4389/// Target-gate and capacity-intent contention is a property of the target pool,
4390/// so the entry is deferred to the end of the round instead of failing the
4391/// bucket. Benign contention means the mutation is no longer needed (the object
4392/// or version disappeared, or a competing mutation already superseded it): the
4393/// entry is done and must not be counted as a failure or escalated. Every other
4394/// failure stays terminal.
4395fn resolve_decommission_entry_failure(err: Error) -> Result<DecommissionEntryOutcome> {
4396    match classify_decommission_capacity_failure(&err) {
4397        DecommissionCapacityFailure::TargetGateBusy { .. } | DecommissionCapacityFailure::IntentConflict => {
4398            Ok(DecommissionEntryOutcome::Deferred(err))
4399        }
4400        DecommissionCapacityFailure::BenignContention => Ok(DecommissionEntryOutcome::Complete),
4401        DecommissionCapacityFailure::Fatal => Err(err),
4402    }
4403}
4404
4405#[cfg(test)]
4406pub(crate) type DecommissionTestFaultDecision = Arc<dyn Fn(&'static str, &str, &str, usize, bool) -> bool + Send + Sync>;
4407
4408#[cfg(test)]
4409static DECOMMISSION_TEST_FAULT_HOOK: std::sync::OnceLock<std::sync::Mutex<Option<DecommissionTestFaultDecision>>> =
4410    std::sync::OnceLock::new();
4411
4412#[cfg(test)]
4413pub(crate) struct DecommissionTestFaultGuard(DecommissionTestFaultDecision);
4414
4415#[cfg(test)]
4416impl DecommissionTestFaultGuard {
4417    pub(crate) fn install(decision: DecommissionTestFaultDecision) -> Self {
4418        let mut slot = DECOMMISSION_TEST_FAULT_HOOK
4419            .get_or_init(|| std::sync::Mutex::new(None))
4420            .lock()
4421            .expect("decommission test fault hook mutex should not poison");
4422        assert!(slot.is_none(), "decommission test fault hook must be unique");
4423        let stored = Arc::clone(&decision);
4424        *slot = Some(decision);
4425        Self(stored)
4426    }
4427}
4428
4429#[cfg(test)]
4430impl Drop for DecommissionTestFaultGuard {
4431    fn drop(&mut self) {
4432        let mut slot = DECOMMISSION_TEST_FAULT_HOOK
4433            .get_or_init(|| std::sync::Mutex::new(None))
4434            .lock()
4435            .expect("decommission test fault hook mutex should not poison");
4436        if slot.as_ref().is_some_and(|decision| Arc::ptr_eq(decision, &self.0)) {
4437            *slot = None;
4438        }
4439    }
4440}
4441
4442#[cfg(test)]
4443fn decommission_test_wrap_result<T>(
4444    stage: &'static str,
4445    bucket: &str,
4446    object: &str,
4447    attempt: usize,
4448    result: Result<T>,
4449) -> Result<T> {
4450    let decision = DECOMMISSION_TEST_FAULT_HOOK
4451        .get_or_init(|| std::sync::Mutex::new(None))
4452        .lock()
4453        .expect("decommission test fault hook mutex should not poison")
4454        .clone();
4455    let inject = decision.is_some_and(|decision| decision(stage, bucket, object, attempt, result.is_ok()));
4456    if result.is_ok() && inject {
4457        return Err(Error::other(format!(
4458            "injected decommission test fault at {stage} attempt {attempt} for {bucket}/{object}"
4459        )));
4460    }
4461    result
4462}
4463
4464#[cfg(test)]
4465pub(crate) type DecommissionCleanupMutationHook = Arc<dyn Fn(&str, &str, usize) -> BoxFuture<'static, ()> + Send + Sync>;
4466
4467#[cfg(test)]
4468static DECOMMISSION_CLEANUP_MUTATION_HOOK: std::sync::OnceLock<std::sync::Mutex<Option<DecommissionCleanupMutationHook>>> =
4469    std::sync::OnceLock::new();
4470
4471#[cfg(test)]
4472pub(crate) struct DecommissionCleanupMutationGuard(DecommissionCleanupMutationHook);
4473
4474#[cfg(test)]
4475impl DecommissionCleanupMutationGuard {
4476    pub(crate) fn install(hook: DecommissionCleanupMutationHook) -> Self {
4477        let mut slot = DECOMMISSION_CLEANUP_MUTATION_HOOK
4478            .get_or_init(|| std::sync::Mutex::new(None))
4479            .lock()
4480            .expect("decommission cleanup mutation hook mutex should not poison");
4481        assert!(slot.is_none(), "decommission cleanup mutation hook must be unique");
4482        let stored = Arc::clone(&hook);
4483        *slot = Some(hook);
4484        Self(stored)
4485    }
4486}
4487
4488#[cfg(test)]
4489impl Drop for DecommissionCleanupMutationGuard {
4490    fn drop(&mut self) {
4491        let mut slot = DECOMMISSION_CLEANUP_MUTATION_HOOK
4492            .get_or_init(|| std::sync::Mutex::new(None))
4493            .lock()
4494            .expect("decommission cleanup mutation hook mutex should not poison");
4495        if slot.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.0)) {
4496            *slot = None;
4497        }
4498    }
4499}
4500
4501#[cfg(test)]
4502async fn run_decommission_cleanup_mutation_hook(bucket: &str, object: &str, attempt: usize) {
4503    let hook = DECOMMISSION_CLEANUP_MUTATION_HOOK
4504        .get_or_init(|| std::sync::Mutex::new(None))
4505        .lock()
4506        .expect("decommission cleanup mutation hook mutex should not poison")
4507        .clone();
4508    if let Some(hook) = hook {
4509        hook(bucket, object, attempt).await;
4510    }
4511}
4512
4513const DECOMMISSION_FREE_VERSION_MIGRATED_REASON: &str = "tier_free_version_migrated";
4514const DECOMMISSION_FREE_VERSION_CONSUMED_REASON: &str = "tier_free_version_already_consumed";
4515const DECOMMISSION_FREE_VERSION_RETAINED_REASON: &str = "tier_free_version_migration_failed";
4516const DECOMMISSION_FREE_VERSION_SWEEP_REASON: &str = "tier_free_version_unresolved_after_decommission";
4517const DECOMMISSION_FREE_VERSION_DISPOSITION_REASON: &str = "tier_free_version_disposition_recorded";
4518
4519#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
4520struct DecommissionFreeVersionDisposition {
4521    migrated: usize,
4522    consumed: usize,
4523    retained: usize,
4524}
4525
4526impl DecommissionFreeVersionDisposition {
4527    fn record_migrated(&mut self) {
4528        self.migrated += 1;
4529    }
4530
4531    fn record_consumed(&mut self) {
4532        self.consumed += 1;
4533    }
4534
4535    fn record_retained(&mut self) {
4536        self.retained += 1;
4537    }
4538
4539    fn total(self) -> usize {
4540        self.migrated.saturating_add(self.consumed).saturating_add(self.retained)
4541    }
4542}
4543
4544enum DecommissionFreeVersionAttempt {
4545    Migrated,
4546    Consumed,
4547    CapacityFailure(Error),
4548    Retry(Error),
4549}
4550
4551fn classify_decommission_free_version_attempt(result: Result<()>) -> DecommissionFreeVersionAttempt {
4552    match result {
4553        Ok(()) => DecommissionFreeVersionAttempt::Migrated,
4554        Err(err) if is_decommission_copy_cleanup_safe_error(&err) => DecommissionFreeVersionAttempt::Consumed,
4555        Err(err) if is_decommission_target_capacity_error(&err) => DecommissionFreeVersionAttempt::CapacityFailure(err),
4556        Err(err) => DecommissionFreeVersionAttempt::Retry(err),
4557    }
4558}
4559
4560#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4561#[allow(
4562    dead_code,
4563    reason = "terminal-state classification asserted by this file's tests (backlog#1823)"
4564)]
4565enum DecommissionTerminalState {
4566    Completed,
4567    Failed,
4568}
4569
4570#[allow(
4571    dead_code,
4572    reason = "terminal-state classification asserted by this file's tests (backlog#1823)"
4573)]
4574fn classify_decommission_terminal_state(failed_items_present: bool) -> DecommissionTerminalState {
4575    if failed_items_present {
4576        DecommissionTerminalState::Failed
4577    } else {
4578        DecommissionTerminalState::Completed
4579    }
4580}
4581
4582fn should_preserve_decommission_canceled_state(meta_canceled: bool, _cancel_signal: bool) -> bool {
4583    meta_canceled
4584}
4585
4586fn should_continue_decommission_queue(meta: &PoolMeta, idx: usize) -> bool {
4587    meta.pools
4588        .get(idx)
4589        .and_then(|pool| pool.decommission.as_ref())
4590        .is_some_and(|info| info.complete && !info.failed && !info.canceled)
4591}
4592
4593fn decommission_cancel_signal_result(cancel_signal: bool) -> Result<()> {
4594    if cancel_signal {
4595        Err(StorageError::OperationCanceled)
4596    } else {
4597        Ok(())
4598    }
4599}
4600
4601fn is_decommission_cancel_requested(cancel_signal: bool, pool: Option<&PoolStatus>) -> bool {
4602    cancel_signal
4603        || pool
4604            .and_then(|pool| pool.decommission.as_ref())
4605            .is_some_and(|info| info.canceled)
4606}
4607
4608fn should_skip_canceled_decommission_routine(cancel_signal: bool, pool: Option<&PoolStatus>) -> bool {
4609    cancel_signal
4610        && pool
4611            .and_then(|pool| pool.decommission.as_ref())
4612            .is_some_and(|info| info.canceled)
4613}
4614
4615async fn run_decommission_buckets_bounded<F>(
4616    rx: CancellationToken,
4617    buckets: Vec<DecomBucketInfo>,
4618    limit: usize,
4619    mut start_bucket: F,
4620) -> Result<()>
4621where
4622    F: FnMut(DecomBucketInfo, CancellationToken) -> BoxFuture<'static, Result<()>>,
4623{
4624    let mut pending = buckets.into_iter();
4625    let mut active: FuturesUnordered<BoxFuture<'static, Result<()>>> = FuturesUnordered::new();
4626    let mut first_err = None;
4627    let limit = limit.max(1);
4628
4629    for _ in 0..limit {
4630        let Some(bucket) = pending.next() else {
4631            break;
4632        };
4633
4634        active.push(start_bucket(bucket, rx.clone()));
4635    }
4636
4637    while let Some(result) = active.next().await {
4638        if let Err(err) = result {
4639            rx.cancel();
4640            if first_err.is_none() {
4641                first_err = Some(err);
4642            }
4643            continue;
4644        }
4645
4646        if first_err.is_some() || rx.is_cancelled() {
4647            continue;
4648        }
4649
4650        let Some(bucket) = pending.next() else {
4651            continue;
4652        };
4653
4654        active.push(start_bucket(bucket, rx.clone()));
4655    }
4656
4657    if first_err.is_none() && rx.is_cancelled() && pending.len() > 0 {
4658        return decommission_cancel_signal_result(true);
4659    }
4660
4661    if let Some(err) = first_err {
4662        return Err(err);
4663    }
4664
4665    Ok(())
4666}
4667
4668async fn run_decommission_phases<F>(
4669    rx: CancellationToken,
4670    regular_buckets: Vec<DecomBucketInfo>,
4671    meta_buckets: Vec<DecomBucketInfo>,
4672    bucket_concurrency: usize,
4673    mut start_bucket: F,
4674) -> Result<()>
4675where
4676    F: FnMut(DecomBucketInfo, CancellationToken) -> BoxFuture<'static, Result<()>>,
4677{
4678    decommission_cancel_signal_result(rx.is_cancelled())?;
4679
4680    for bucket in meta_buckets {
4681        decommission_cancel_signal_result(rx.is_cancelled())?;
4682        start_bucket(bucket, rx.clone()).await?;
4683    }
4684
4685    decommission_cancel_signal_result(rx.is_cancelled())?;
4686
4687    if bucket_concurrency <= 1 {
4688        for bucket in regular_buckets {
4689            decommission_cancel_signal_result(rx.is_cancelled())?;
4690            start_bucket(bucket, rx.clone()).await?;
4691        }
4692        return Ok(());
4693    }
4694
4695    run_decommission_buckets_bounded(rx, regular_buckets, bucket_concurrency, start_bucket).await
4696}
4697
4698#[cfg(test)]
4699async fn wait_decommission_worker_drain(workers: &Semaphore, limit: usize) -> Result<()> {
4700    let permits = u32::try_from(limit)
4701        .map_err(|_| Error::other(format!("decommission worker limit {limit} exceeds semaphore drain capacity")))?;
4702    let _drain = workers
4703        .acquire_many(permits)
4704        .await
4705        .map_err(|err| Error::other(format!("decommission worker drain failed: {err}")))?;
4706    Ok(())
4707}
4708
4709fn should_reject_decommission_cancel_as_terminal(complete: bool, failed: bool) -> bool {
4710    complete || failed
4711}
4712
4713fn should_retry_decommission_cancel_reload(changed: bool, already_canceled: bool) -> bool {
4714    changed || already_canceled
4715}
4716
4717fn ensure_decommission_cancel_allowed(pool_present: bool, decommission_present: bool, terminal: bool) -> Result<()> {
4718    if !pool_present {
4719        return Err(invalid_decommission_request("failed to cancel decommission: target pool was not found"));
4720    }
4721
4722    if !decommission_present || terminal {
4723        return Err(StorageError::DecommissionNotStarted);
4724    }
4725
4726    Ok(())
4727}
4728
4729fn ensure_decommission_clear_allowed(
4730    pool_present: bool,
4731    decommission_present: bool,
4732    complete: bool,
4733    failed: bool,
4734    canceled: bool,
4735    unresolved_entries: usize,
4736) -> Result<()> {
4737    if !pool_present {
4738        return Err(invalid_decommission_request("failed to clear decommission: target pool was not found"));
4739    }
4740
4741    if !decommission_present {
4742        return Err(StorageError::DecommissionNotStarted);
4743    }
4744
4745    if complete {
4746        return Err(StorageError::DecommissionNotStarted);
4747    }
4748
4749    if !failed && !canceled {
4750        return Err(StorageError::DecommissionAlreadyRunning);
4751    }
4752
4753    if unresolved_entries > 0 {
4754        return Err(invalid_decommission_request(format!(
4755            "failed to clear decommission: {unresolved_entries} unresolved listing entries must be reconciled by retrying decommission"
4756        )));
4757    }
4758
4759    Ok(())
4760}
4761
4762fn ensure_decommission_terminal_operation_supported(single_pool: bool, operation: &str) -> Result<()> {
4763    if single_pool {
4764        return Err(invalid_decommission_request(format!(
4765            "failed to {operation}: single pool deployments do not support decommission"
4766        )));
4767    }
4768
4769    Ok(())
4770}
4771
4772fn validate_start_decommission_request(indices: &[usize], single_pool: bool) -> Result<()> {
4773    if indices.is_empty() {
4774        return Err(invalid_decommission_request(
4775            "failed to start decommission: no target pools were provided",
4776        ));
4777    }
4778
4779    ensure_decommission_terminal_operation_supported(single_pool, "start decommission")
4780}
4781
4782fn require_decommission_store<T>(store: Option<T>, operation: &str) -> Result<T> {
4783    store.ok_or_else(|| Error::other(format!("failed to {operation}: store not initialized")))
4784}
4785
4786fn ensure_decommission_listing_disks_available(has_disks: bool, bucket: &str) -> Result<()> {
4787    if !has_disks {
4788        return Err(Error::other(format!(
4789            "failed to list objects to decommission for bucket {bucket}: no disks available"
4790        )));
4791    }
4792
4793    Ok(())
4794}
4795
4796#[derive(Debug, Clone, Serialize, Deserialize)]
4797pub struct PoolStatus {
4798    #[serde(rename = "id")]
4799    pub id: usize,
4800    #[serde(rename = "cmdline")]
4801    pub cmd_line: String,
4802    #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
4803    pub last_update: OffsetDateTime,
4804    #[serde(rename = "decommissionInfo")]
4805    pub decommission: Option<PoolDecommissionInfo>,
4806}
4807
4808#[derive(Debug, Clone, Default, Serialize, Deserialize)]
4809pub struct PoolMeta {
4810    pub version: u16,
4811    pub pools: Vec<PoolStatus>,
4812    pub dont_save: bool,
4813}
4814
4815#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4816struct PoolMetaRevision {
4817    version: u16,
4818    cluster_id: Option<uuid::Uuid>,
4819    epoch: u64,
4820    generation: u64,
4821    transaction_id: Option<uuid::Uuid>,
4822}
4823
4824impl PoolMetaRevision {
4825    fn legacy(version: u16) -> Self {
4826        Self {
4827            version,
4828            cluster_id: None,
4829            epoch: 0,
4830            generation: 0,
4831            transaction_id: None,
4832        }
4833    }
4834
4835    fn is_generation_protocol(self) -> bool {
4836        self.version == POOL_META_GENERATION_VERSION
4837    }
4838}
4839
4840#[derive(Debug, Clone)]
4841struct PoolMetaCommittedCandidate {
4842    canonical: Vec<u8>,
4843    meta: PoolMeta,
4844    revision: PoolMetaRevision,
4845}
4846
4847#[derive(Debug)]
4848enum PoolMetaReplica {
4849    Missing,
4850    Valid {
4851        raw: Vec<u8>,
4852        canonical: Vec<u8>,
4853        meta: PoolMeta,
4854        revision: PoolMetaRevision,
4855        committed: bool,
4856        previous: Option<Box<PoolMetaCommittedCandidate>>,
4857    },
4858    Corrupt(String),
4859    Incompatible(String),
4860    Unreadable(Arc<Error>),
4861}
4862
4863#[derive(Debug, Clone)]
4864enum PoolMetaCasToken {
4865    Missing,
4866    Existing(String),
4867    Unsafe,
4868}
4869
4870#[derive(Debug)]
4871struct PoolMetaReplicaRead {
4872    replica: PoolMetaReplica,
4873    cas: PoolMetaCasToken,
4874}
4875
4876impl From<PoolMetaReplica> for PoolMetaReplicaRead {
4877    fn from(replica: PoolMetaReplica) -> Self {
4878        Self {
4879            replica,
4880            cas: PoolMetaCasToken::Unsafe,
4881        }
4882    }
4883}
4884
4885#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4886pub(crate) struct PoolMetaReplicaState {
4887    pub(crate) needs_repair: bool,
4888    pub(crate) repair_write_safe: bool,
4889}
4890
4891impl PoolMetaReplicaState {
4892    pub(crate) fn ensure_write_safe(self, operation: &str) -> Result<()> {
4893        if self.repair_write_safe {
4894            return Ok(());
4895        }
4896        Err(Error::other(format!(
4897            "{operation}: pool metadata update cannot overwrite an unreadable replica"
4898        )))
4899    }
4900}
4901
4902#[derive(Debug, Clone, Default)]
4903pub(crate) struct PoolMetaWriteState {
4904    write_blocked: bool,
4905    aborted_transaction: Arc<AtomicBool>,
4906    block_context: Option<crate::error::PoolMetadataError>,
4907    transaction_failure: Arc<std::sync::Mutex<Option<crate::error::PoolMetadataError>>>,
4908    active_transactions: Arc<AtomicUsize>,
4909    recovery_failure: Option<Arc<Error>>,
4910    block_started_at: Arc<std::sync::Mutex<Option<OffsetDateTime>>>,
4911    expected_cluster_id: Option<uuid::Uuid>,
4912    cluster_epoch: Option<u64>,
4913    pool_meta_absent: bool,
4914    bootstrap_authority: PoolMetaBootstrapAuthority,
4915    /// First-hand bootstrap authority this process holds for each pool
4916    /// (index = pool index): `Fresh` only for pools it formatted itself,
4917    /// `LegacyAdoption` only for pools whose migration it verified. Empty when
4918    /// the caller tracks deployment-wide authority only.
4919    pool_bootstrap_authorities: Vec<PoolMetaBootstrapAuthority>,
4920    /// Whether this process hosts the first endpoint of the first pool and is
4921    /// therefore the only writer allowed to publish the initial `pool.bin`.
4922    /// `None` when the caller did not say; unknown writers are treated as
4923    /// elected so every fail-closed rule still applies to them.
4924    elected_bootstrap_writer: Option<bool>,
4925    identity_initialized: Option<bool>,
4926    identity_fresh_bootstrap_nonce: Option<uuid::Uuid>,
4927    identity_needs_repair: bool,
4928    /// At least one pool has no identity replica at all.
4929    identity_replicas_missing: bool,
4930    /// At least one pool has a replica that is present but not a valid identity.
4931    identity_replicas_invalid: bool,
4932}
4933
4934/// Why an all-missing `pool.bin` set may not be initialized right now.
4935enum MissingMetadataRejection {
4936    /// Another node still has to act (mint, attest, or publish); retrying the
4937    /// startup loop is the remedy, so the write gate stays open.
4938    BootstrapPending(Error),
4939    /// The durable state contradicts a fresh bootstrap; writes stay blocked
4940    /// until an operator recovers the metadata.
4941    RecoveryRequired(Error),
4942}
4943
4944impl MissingMetadataRejection {
4945    fn into_error(self) -> Error {
4946        match self {
4947            Self::BootstrapPending(err) | Self::RecoveryRequired(err) => err,
4948        }
4949    }
4950}
4951
4952#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
4953pub(crate) enum PoolMetaBootstrapAuthority {
4954    #[default]
4955    None,
4956    Fresh,
4957    LegacyAdoption,
4958}
4959
4960impl PoolMetaBootstrapAuthority {
4961    pub(crate) fn combine_across_pools(self, other: Self) -> Self {
4962        if self == other { self } else { Self::None }
4963    }
4964
4965    fn is_proven(self) -> bool {
4966        !matches!(self, Self::None)
4967    }
4968}
4969
4970impl PoolMetaWriteState {
4971    #[cfg(test)]
4972    pub(crate) fn for_startup(cluster_id: uuid::Uuid, fresh_bootstrap_proven: bool) -> Self {
4973        let bootstrap_authority = if fresh_bootstrap_proven {
4974            PoolMetaBootstrapAuthority::Fresh
4975        } else {
4976            PoolMetaBootstrapAuthority::None
4977        };
4978        Self::for_startup_with_bootstrap_authority(cluster_id, bootstrap_authority)
4979    }
4980
4981    #[cfg(test)]
4982    pub(crate) fn for_startup_with_bootstrap_authority(
4983        cluster_id: uuid::Uuid,
4984        bootstrap_authority: PoolMetaBootstrapAuthority,
4985    ) -> Self {
4986        Self {
4987            expected_cluster_id: Some(cluster_id),
4988            bootstrap_authority,
4989            ..Default::default()
4990        }
4991    }
4992
4993    /// Startup state for a process that loaded every pool format itself and
4994    /// remembers, per pool, whether it created (or adopted) that pool
4995    /// first-hand. Deployment-wide authority is the conjunction across pools:
4996    /// any pool this process merely read yields `None`, exactly as before.
4997    pub(crate) fn for_startup_with_pool_bootstrap_authorities(
4998        cluster_id: uuid::Uuid,
4999        pool_bootstrap_authorities: Vec<PoolMetaBootstrapAuthority>,
5000        elected_bootstrap_writer: bool,
5001    ) -> Self {
5002        let bootstrap_authority = pool_bootstrap_authorities
5003            .iter()
5004            .copied()
5005            .reduce(PoolMetaBootstrapAuthority::combine_across_pools)
5006            .unwrap_or_default();
5007        Self {
5008            expected_cluster_id: Some(cluster_id),
5009            bootstrap_authority,
5010            pool_bootstrap_authorities,
5011            elected_bootstrap_writer: Some(elected_bootstrap_writer),
5012            ..Default::default()
5013        }
5014    }
5015
5016    pub(crate) fn bootstrap_identity_proven(&self) -> bool {
5017        self.bootstrap_authority.is_proven()
5018    }
5019
5020    fn pool_bootstrap_authority_proven(&self, pool_idx: usize) -> bool {
5021        self.pool_bootstrap_authorities
5022            .get(pool_idx)
5023            .is_some_and(|authority| authority.is_proven())
5024    }
5025
5026    /// Pools this process formatted or adopted first-hand during this startup.
5027    pub(crate) fn attested_pool_indices(&self) -> Vec<usize> {
5028        self.pool_bootstrap_authorities
5029            .iter()
5030            .enumerate()
5031            .filter(|(_, authority)| authority.is_proven())
5032            .map(|(pool_idx, _)| pool_idx)
5033            .collect()
5034    }
5035
5036    /// Deployment-level proof assembled from per-pool creators: this process
5037    /// created the first pool itself, and every pool replica carries the same
5038    /// pending identity. A pending replica is only ever written by the process
5039    /// that formatted that pool with first-hand proof (see
5040    /// [`PoolMetaIdentityWriteScope::Pools`]), so a complete, agreeing pending
5041    /// set proves that every pool joined this bootstrap fresh. A missing,
5042    /// corrupt, or disagreeing replica keeps the writer fail-closed, and a
5043    /// restart without first-hand proof never reopens bootstrap on its own.
5044    fn pending_identity_attested_by_every_pool(&self) -> bool {
5045        self.elected_bootstrap_writer == Some(true)
5046            && self.pool_bootstrap_authority_proven(0)
5047            && self.identity_initialized == Some(false)
5048            && !self.identity_needs_repair
5049            && self.identity_fresh_bootstrap_nonce.is_some()
5050    }
5051
5052    /// The elected writer minted (or holds) the nonce and the only thing
5053    /// standing between it and a complete attestation is a pool whose creator
5054    /// has not written its replica yet. Corrupt replicas are never transient.
5055    fn awaiting_creator_attestation(&self) -> bool {
5056        self.elected_bootstrap_writer == Some(true)
5057            && self.pool_bootstrap_authority_proven(0)
5058            && self.identity_initialized == Some(false)
5059            && self.identity_fresh_bootstrap_nonce.is_some()
5060            && self.identity_needs_repair
5061            && self.identity_replicas_missing
5062            && !self.identity_replicas_invalid
5063    }
5064
5065    fn is_non_elected_bootstrap_observer(&self) -> bool {
5066        self.elected_bootstrap_writer == Some(false)
5067    }
5068
5069    pub(crate) fn identity_is_pending(&self) -> bool {
5070        self.identity_initialized == Some(false)
5071    }
5072
5073    #[cfg(test)]
5074    pub(crate) fn aborted_transaction_latch_for_test(&self) -> Arc<AtomicBool> {
5075        Arc::clone(&self.aborted_transaction)
5076    }
5077
5078    #[cfg(test)]
5079    pub(crate) fn independent_clone_for_test(&self) -> Self {
5080        let mut cloned = self.clone();
5081        cloned.aborted_transaction = Arc::new(AtomicBool::new(self.aborted_transaction.load(Ordering::Acquire)));
5082        cloned.transaction_failure = Arc::new(std::sync::Mutex::new(
5083            self.transaction_failure
5084                .lock()
5085                .unwrap_or_else(std::sync::PoisonError::into_inner)
5086                .clone(),
5087        ));
5088        cloned.active_transactions = Arc::new(AtomicUsize::new(0));
5089        cloned.block_started_at = Arc::new(std::sync::Mutex::new(
5090            *self
5091                .block_started_at
5092                .lock()
5093                .unwrap_or_else(std::sync::PoisonError::into_inner),
5094        ));
5095        cloned
5096    }
5097
5098    #[cfg(any(test, feature = "test-util"))]
5099    fn for_test_bootstrap() -> Self {
5100        Self {
5101            bootstrap_authority: PoolMetaBootstrapAuthority::Fresh,
5102            identity_initialized: Some(false),
5103            identity_fresh_bootstrap_nonce: Some(uuid::Uuid::new_v4()),
5104            ..Default::default()
5105        }
5106    }
5107
5108    pub(crate) fn observe_replicas(&mut self, replica_state: PoolMetaReplicaState) {
5109        if !replica_state.repair_write_safe {
5110            self.block_writes();
5111        }
5112    }
5113
5114    fn block_writes(&mut self) {
5115        self.block_with_context(pool_metadata_error(
5116            crate::error::PoolMetadataFailure::RecoveryRequired,
5117            "validation",
5118            None,
5119        ));
5120    }
5121
5122    fn block_with_context(&mut self, mut context: crate::error::PoolMetadataError) {
5123        if !self.write_blocked {
5124            record_pool_meta_block_once(&self.block_started_at, &mut context);
5125            self.block_context = Some(context);
5126        } else if let Some(previous) = &self.block_context
5127            && previous.source.is_none()
5128            && context.source.is_some()
5129        {
5130            context.since = previous.since;
5131            self.block_context = Some(context);
5132        }
5133        self.write_blocked = true;
5134    }
5135
5136    pub(crate) fn block_writes_after_fence_loss(&mut self) {
5137        self.block_with_context(pool_metadata_error(crate::error::PoolMetadataFailure::FenceLost, "format_heal", None));
5138    }
5139
5140    fn arm_transaction(&self) -> PoolMetaTransactionArm {
5141        self.active_transactions.fetch_add(1, Ordering::SeqCst);
5142        PoolMetaTransactionArm {
5143            aborted_transaction: Arc::clone(&self.aborted_transaction),
5144            failure: Arc::clone(&self.transaction_failure),
5145            active_transactions: Arc::clone(&self.active_transactions),
5146            block_started_at: Arc::clone(&self.block_started_at),
5147            phase: None,
5148            source: None,
5149        }
5150    }
5151
5152    fn observe_selection(&mut self, selection: &PoolMetaSelection) -> Result<()> {
5153        self.pool_meta_absent = selection.absent;
5154        self.validate_selection(selection)?;
5155        if self.cluster_epoch.is_none()
5156            && let Some((_, metadata_epoch)) = selection.generation_identity
5157        {
5158            self.cluster_epoch = Some(metadata_epoch);
5159        }
5160        Ok(())
5161    }
5162
5163    fn validate_selection(&self, selection: &PoolMetaSelection) -> Result<()> {
5164        if let Some(expected_cluster_id) = self.expected_cluster_id
5165            && let Some((cluster_id, _)) = selection.generation_identity
5166            && cluster_id != expected_cluster_id
5167        {
5168            return Err(Error::other(format!(
5169                "pool metadata incompatible: cluster identity {cluster_id} does not match deployment {expected_cluster_id}"
5170            )));
5171        }
5172        if let Some(identity_epoch) = self.cluster_epoch
5173            && let Some((_, metadata_epoch)) = selection.generation_identity
5174            && metadata_epoch != identity_epoch
5175        {
5176            return Err(Error::other(format!(
5177                "pool metadata recovery required: committed epoch {} does not match cluster identity epoch {identity_epoch}",
5178                metadata_epoch
5179            )));
5180        }
5181        Ok(())
5182    }
5183
5184    fn observe_identity(&mut self, selection: &PoolMetaIdentitySelection) -> Result<()> {
5185        self.identity_needs_repair = selection.needs_repair;
5186        self.identity_initialized = selection.identity.map(|identity| identity.initialized);
5187        self.identity_fresh_bootstrap_nonce = selection.identity.and_then(|identity| identity.fresh_bootstrap_nonce);
5188        self.identity_replicas_missing = selection
5189            .cas_tokens
5190            .iter()
5191            .any(|token| matches!(token, PoolMetaCasToken::Missing));
5192        self.identity_replicas_invalid = selection
5193            .valid_replicas
5194            .iter()
5195            .zip(&selection.cas_tokens)
5196            .any(|(valid, token)| !valid && !matches!(token, PoolMetaCasToken::Missing));
5197        if let Some(identity) = selection.identity {
5198            if identity.initialized {
5199                self.bootstrap_authority = PoolMetaBootstrapAuthority::None;
5200                self.pool_bootstrap_authorities.clear();
5201            }
5202            if let Some(metadata_epoch) = self.cluster_epoch
5203                && metadata_epoch != identity.epoch
5204            {
5205                self.block_writes();
5206                return Err(Error::other(format!(
5207                    "pool metadata recovery required: metadata epoch {metadata_epoch} does not match cluster identity epoch {}",
5208                    identity.epoch
5209                )));
5210            }
5211            self.cluster_epoch = Some(identity.epoch);
5212        }
5213        if !selection.repair_write_safe {
5214            self.block_writes();
5215        }
5216        Ok(())
5217    }
5218
5219    pub(crate) fn ensure_missing_metadata_can_initialize(&mut self) -> Result<()> {
5220        if !self.pool_meta_absent {
5221            return Ok(());
5222        }
5223        match self.validate_missing_metadata_can_initialize() {
5224            Ok(()) => Ok(()),
5225            Err(MissingMetadataRejection::BootstrapPending(err)) => Err(err),
5226            Err(MissingMetadataRejection::RecoveryRequired(err)) => {
5227                Err(block_pool_meta_validation(self, err, "metadata_absence"))
5228            }
5229        }
5230    }
5231
5232    fn validate_missing_metadata_can_initialize(&self) -> std::result::Result<(), MissingMetadataRejection> {
5233        use MissingMetadataRejection::{BootstrapPending, RecoveryRequired};
5234        match self.identity_initialized {
5235            Some(false)
5236                if self.identity_fresh_bootstrap_nonce.is_some()
5237                    && (self.bootstrap_identity_proven() || self.pending_identity_attested_by_every_pool()) =>
5238            {
5239                Ok(())
5240            }
5241            Some(false) if self.awaiting_creator_attestation() => Err(BootstrapPending(Error::other(
5242                "pool metadata bootstrap pending: waiting for every pool creator to attest the pending cluster identity",
5243            ))),
5244            Some(false) if self.is_non_elected_bootstrap_observer() && self.identity_fresh_bootstrap_nonce.is_some() => {
5245                Err(BootstrapPending(Error::other(
5246                    "pool metadata bootstrap pending: waiting for the elected writer to publish the initial pool.bin",
5247                )))
5248            }
5249            Some(false) => Err(RecoveryRequired(Error::other(
5250                "pool metadata recovery required: pending cluster identity exists but this startup has no verified fresh-bootstrap proof, legacy-adoption proof, or complete per-pool creator attestation",
5251            ))),
5252            Some(true) => Err(RecoveryRequired(Error::other(
5253                "pool metadata recovery required: initialized cluster identity exists but every pool.bin replica is missing",
5254            ))),
5255            None if self.is_non_elected_bootstrap_observer() => Err(BootstrapPending(Error::other(
5256                "pool metadata bootstrap pending: waiting for the elected writer to establish the cluster identity",
5257            ))),
5258            None => Err(RecoveryRequired(Error::other(
5259                "pool metadata recovery required: no durable bootstrap identity or pool.bin replica is available",
5260            ))),
5261        }
5262    }
5263
5264    pub(crate) fn identity_requires_repair(&self) -> bool {
5265        self.expected_cluster_id.is_some() && (self.identity_needs_repair || self.identity_initialized != Some(true))
5266    }
5267
5268    pub(crate) fn ensure_write_safe(&self, operation: &str) -> Result<()> {
5269        if !self.write_blocked && !self.aborted_transaction.load(Ordering::SeqCst) {
5270            return Ok(());
5271        }
5272        let mut context = self
5273            .block_context
5274            .clone()
5275            .or_else(|| {
5276                self.transaction_failure
5277                    .lock()
5278                    .unwrap_or_else(std::sync::PoisonError::into_inner)
5279                    .clone()
5280            })
5281            .unwrap_or_else(|| pool_metadata_error(crate::error::PoolMetadataFailure::FenceLost, "external_fence", None));
5282        context.operation = operation.to_owned();
5283        Err(Error::other(context))
5284    }
5285}
5286
5287fn pool_metadata_error(
5288    kind: crate::error::PoolMetadataFailure,
5289    phase: &'static str,
5290    source: Option<Arc<Error>>,
5291) -> crate::error::PoolMetadataError {
5292    crate::error::PoolMetadataError {
5293        kind,
5294        operation: phase.to_owned(),
5295        phase,
5296        since: OffsetDateTime::now_utc(),
5297        source,
5298    }
5299}
5300
5301fn record_pool_meta_block_once(
5302    started_at: &std::sync::Mutex<Option<OffsetDateTime>>,
5303    context: &mut crate::error::PoolMetadataError,
5304) {
5305    let mut started_at = started_at.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
5306    if let Some(since) = *started_at {
5307        context.since = since;
5308        return;
5309    }
5310    *started_at = Some(context.since);
5311    error!(
5312        event = EVENT_DECOMMISSION_STATE,
5313        component = LOG_COMPONENT_ECSTORE,
5314        subsystem = LOG_SUBSYSTEM_POOLS,
5315        state = "pool_metadata_blocked",
5316        reason = context.kind.as_str(),
5317        phase = context.phase,
5318        blocked_since = %context.since,
5319        "Pool metadata writes blocked"
5320    );
5321    metrics::counter!("rustfs_pool_metadata_blocks_total", "reason" => context.kind.as_str()).increment(1);
5322}
5323
5324#[derive(Debug)]
5325struct PoolMetaTransactionArm {
5326    aborted_transaction: Arc<AtomicBool>,
5327    failure: Arc<std::sync::Mutex<Option<crate::error::PoolMetadataError>>>,
5328    active_transactions: Arc<AtomicUsize>,
5329    block_started_at: Arc<std::sync::Mutex<Option<OffsetDateTime>>>,
5330    phase: Option<&'static str>,
5331    source: Option<Arc<Error>>,
5332}
5333
5334impl PoolMetaTransactionArm {
5335    fn failure(&mut self, err: Error) -> Error {
5336        let Some(phase) = self.phase else {
5337            return err;
5338        };
5339        let source = Arc::new(err);
5340        self.source = Some(Arc::clone(&source));
5341        Error::other(pool_metadata_error(
5342            crate::error::PoolMetadataFailure::TransactionUnknown,
5343            phase,
5344            Some(source),
5345        ))
5346    }
5347
5348    fn disarm(&mut self) {
5349        self.phase = None;
5350    }
5351}
5352
5353impl Drop for PoolMetaTransactionArm {
5354    fn drop(&mut self) {
5355        if let Some(phase) = self.phase {
5356            let mut context =
5357                pool_metadata_error(crate::error::PoolMetadataFailure::TransactionUnknown, phase, self.source.take());
5358            let mut failure = self.failure.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
5359            if !self.aborted_transaction.swap(true, Ordering::SeqCst) {
5360                record_pool_meta_block_once(&self.block_started_at, &mut context);
5361                *failure = Some(context);
5362            }
5363        }
5364        self.active_transactions.fetch_sub(1, Ordering::SeqCst);
5365    }
5366}
5367
5368#[derive(Debug)]
5369struct PoolMetaSelection {
5370    meta: PoolMeta,
5371    revision: PoolMetaRevision,
5372    canonical: Option<Vec<u8>>,
5373    replica_state: PoolMetaReplicaState,
5374    cas_tokens: Vec<PoolMetaCasToken>,
5375    absent: bool,
5376    generation_protocol_observed: bool,
5377    generation_identity: Option<(uuid::Uuid, u64)>,
5378}
5379
5380fn classify_pool_meta_tuple_decode_error(kind: &str, err: rmp_serde::decode::Error) -> PoolMetaReplica {
5381    let truncated = matches!(
5382        &err,
5383        rmp_serde::decode::Error::InvalidMarkerRead(source)
5384            | rmp_serde::decode::Error::InvalidDataRead(source)
5385            if source.kind() == std::io::ErrorKind::UnexpectedEof
5386    );
5387    if truncated {
5388        PoolMetaReplica::Corrupt(format!("{kind} tuple payload is truncated: {err}"))
5389    } else {
5390        PoolMetaReplica::Incompatible(format!("{kind} tuple payload is not decodable: {err}"))
5391    }
5392}
5393
5394fn parse_pool_meta_uuid(value: &str, field: &str) -> Result<uuid::Uuid> {
5395    let parsed = uuid::Uuid::parse_str(value)
5396        .map_err(|err| Error::other(format!("pool metadata corrupt: invalid {field} `{value}`: {err}")))?;
5397    if parsed.is_nil() || parsed == uuid::Uuid::max() {
5398        return Err(Error::other(format!("pool metadata corrupt: {field} must be a non-reserved UUID")));
5399    }
5400    Ok(parsed)
5401}
5402
5403fn pool_meta_generation_revision(
5404    cluster_id: &str,
5405    epoch: u64,
5406    generation: u64,
5407    transaction_id: &str,
5408) -> Result<PoolMetaRevision> {
5409    if epoch == 0 || generation == 0 {
5410        return Err(Error::other(
5411            "pool metadata corrupt: version 3 epoch and generation must both be non-zero",
5412        ));
5413    }
5414    Ok(PoolMetaRevision {
5415        version: POOL_META_GENERATION_VERSION,
5416        cluster_id: Some(parse_pool_meta_uuid(cluster_id, "cluster identity")?),
5417        epoch,
5418        generation,
5419        transaction_id: Some(parse_pool_meta_uuid(transaction_id, "transaction id")?),
5420    })
5421}
5422
5423fn pool_meta_from_v3_statuses(version: u16, pools: Vec<PersistedPoolStatus>) -> Result<PoolMeta> {
5424    if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) {
5425        return Err(Error::other(format!(
5426            "pool metadata corrupt: version 3 previous snapshot has unsupported source version {version}"
5427        )));
5428    }
5429    let meta = PoolMeta {
5430        version,
5431        pools: pools.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?,
5432        dont_save: false,
5433    };
5434    validate_decommission_capacity_model_cohort(&meta)?;
5435    Ok(meta)
5436}
5437
5438fn pool_meta_previous_candidate(value: PersistedPoolMetaV3Previous) -> Result<PoolMetaCommittedCandidate> {
5439    let revision = match value.version {
5440        POOL_META_V1_VERSION | POOL_META_VERSION => {
5441            if value.cluster_id.is_some() || value.epoch != 0 || value.generation != 0 || value.transaction_id.is_some() {
5442                return Err(Error::other(
5443                    "pool metadata corrupt: legacy previous snapshot carries version 3 revision fields",
5444                ));
5445            }
5446            PoolMetaRevision::legacy(value.version)
5447        }
5448        POOL_META_GENERATION_VERSION => pool_meta_generation_revision(
5449            value
5450                .cluster_id
5451                .as_deref()
5452                .ok_or_else(|| Error::other("pool metadata corrupt: version 3 previous snapshot has no cluster identity"))?,
5453            value.epoch,
5454            value.generation,
5455            value
5456                .transaction_id
5457                .as_deref()
5458                .ok_or_else(|| Error::other("pool metadata corrupt: version 3 previous snapshot has no transaction id"))?,
5459        )?,
5460        version => {
5461            return Err(Error::other(format!(
5462                "pool metadata corrupt: previous snapshot has unsupported version {version}"
5463            )));
5464        }
5465    };
5466    let meta = pool_meta_from_v3_statuses(value.version, value.pools)?;
5467    let canonical = if revision.is_generation_protocol() {
5468        encode_pool_meta_v3_envelope(&meta, revision, true, None)?
5469    } else {
5470        meta.encode_config_data_for_v2_gate(true)?
5471    };
5472    Ok(PoolMetaCommittedCandidate {
5473        canonical,
5474        meta,
5475        revision,
5476    })
5477}
5478
5479fn decode_pool_meta_v3(data: Vec<u8>) -> PoolMetaReplica {
5480    let persisted = match rmp_serde::from_slice::<PersistedPoolMetaV3>(&data[4..]) {
5481        Ok(persisted) => persisted,
5482        Err(err) => return classify_pool_meta_tuple_decode_error("v3", err),
5483    };
5484    if persisted.version != POOL_META_GENERATION_VERSION {
5485        return PoolMetaReplica::Corrupt(format!(
5486            "v3 payload has version {}, expected {POOL_META_GENERATION_VERSION}",
5487            persisted.version
5488        ));
5489    }
5490    let revision = match pool_meta_generation_revision(
5491        &persisted.cluster_id,
5492        persisted.epoch,
5493        persisted.generation,
5494        &persisted.transaction_id,
5495    ) {
5496        Ok(revision) => revision,
5497        Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5498    };
5499    let meta = match pool_meta_from_v3_statuses(POOL_META_GENERATION_VERSION, persisted.pools) {
5500        Ok(meta) => meta,
5501        Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5502    };
5503    let previous = match persisted.previous.map(pool_meta_previous_candidate).transpose() {
5504        Ok(previous) => previous,
5505        Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5506    };
5507    if persisted.committed && previous.is_some() {
5508        return PoolMetaReplica::Corrupt("committed v3 payload unexpectedly retains a previous snapshot".to_string());
5509    }
5510    if !persisted.committed {
5511        match previous.as_ref() {
5512            Some(previous) if previous.revision.is_generation_protocol() => {
5513                if previous.revision.cluster_id != revision.cluster_id
5514                    || previous.revision.epoch != revision.epoch
5515                    || previous.revision.generation.checked_add(1) != Some(revision.generation)
5516                {
5517                    return PoolMetaReplica::Corrupt(
5518                        "pending v3 payload does not advance exactly one generation from its previous snapshot".to_string(),
5519                    );
5520                }
5521            }
5522            Some(_) if revision.generation != 1 => {
5523                return PoolMetaReplica::Corrupt(
5524                    "first v3 generation must be generation 1 when migrating a legacy snapshot".to_string(),
5525                );
5526            }
5527            None if revision.generation != 1 => {
5528                return PoolMetaReplica::Corrupt(
5529                    "pending v3 payload without a previous snapshot must be the initial generation".to_string(),
5530                );
5531            }
5532            _ => {}
5533        }
5534    }
5535    let canonical = match encode_pool_meta_v3_envelope(&meta, revision, true, None) {
5536        Ok(canonical) => canonical,
5537        Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5538    };
5539    PoolMetaReplica::Valid {
5540        raw: data,
5541        canonical,
5542        meta,
5543        revision,
5544        committed: persisted.committed,
5545        previous: previous.map(Box::new),
5546    }
5547}
5548
5549fn decode_pool_meta_replica(data: Vec<u8>) -> PoolMetaReplica {
5550    if data.len() <= 4 {
5551        return PoolMetaReplica::Corrupt("metadata payload is empty or truncated".to_string());
5552    }
5553
5554    let format = LittleEndian::read_u16(&data[0..2]);
5555    if format != POOL_META_FORMAT {
5556        return PoolMetaReplica::Incompatible(format!("unsupported format {format}"));
5557    }
5558    let version = LittleEndian::read_u16(&data[2..4]);
5559    if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) {
5560        return PoolMetaReplica::Incompatible(format!("unsupported version {version}"));
5561    }
5562    if version == POOL_META_GENERATION_VERSION {
5563        return decode_pool_meta_v3(data);
5564    }
5565
5566    let payload = &data[4..];
5567    let meta = match (version, rmp::decode::read_array_len(&mut &payload[..])) {
5568        (POOL_META_VERSION, Ok(2)) => match rmp_serde::from_slice::<PersistedPoolMeta>(payload) {
5569            Ok(meta) => match PoolMeta::try_from(meta) {
5570                Ok(meta) => meta,
5571                Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5572            },
5573            Err(err) => return classify_pool_meta_tuple_decode_error("current", err),
5574        },
5575        (POOL_META_V1_VERSION, Ok(2)) => match rmp_serde::from_slice::<PersistedPoolMetaV1>(payload) {
5576            Ok(meta) => match PoolMeta::try_from(meta) {
5577                Ok(meta) => meta,
5578                Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5579            },
5580            Err(err) => return classify_pool_meta_tuple_decode_error("v1", err),
5581        },
5582        // The older V1 tuple includes the runtime-only `dont_save` flag.
5583        (POOL_META_V1_VERSION, Ok(3)) => match rmp_serde::from_slice::<LegacyPoolMeta>(payload) {
5584            Ok(meta) => match PoolMeta::try_from(meta) {
5585                Ok(meta) => meta,
5586                Err(err) => return PoolMetaReplica::Corrupt(err.to_string()),
5587            },
5588            Err(err) => return classify_pool_meta_tuple_decode_error("legacy v1", err),
5589        },
5590        (_, Ok(field_count)) if field_count < 2 => {
5591            return PoolMetaReplica::Corrupt(format!("pool metadata tuple has only {field_count} fields"));
5592        }
5593        (_, Ok(field_count)) => {
5594            return PoolMetaReplica::Incompatible(format!(
5595                "pool metadata version {version} tuple has unsupported field count {field_count}"
5596            ));
5597        }
5598        (_, Err(_)) => {
5599            let mut meta = PoolMeta::default();
5600            if let Err(err) = meta.load_from_config_data(data.clone()) {
5601                let reason = err.to_string();
5602                if reason.contains("unknown field") {
5603                    return PoolMetaReplica::Incompatible(format!("current-version payload uses unsupported fields: {reason}"));
5604                }
5605                return PoolMetaReplica::Corrupt(reason);
5606            }
5607            meta
5608        }
5609    };
5610
5611    match meta.encode_config_data_for_v2_gate(true) {
5612        Ok(canonical) => PoolMetaReplica::Valid {
5613            raw: data,
5614            canonical,
5615            meta,
5616            revision: PoolMetaRevision::legacy(version),
5617            committed: true,
5618            previous: None,
5619        },
5620        Err(err) => PoolMetaReplica::Corrupt(err.to_string()),
5621    }
5622}
5623
5624#[cfg(test)]
5625pub(crate) fn pool_meta_v3_commit_state_for_test(data: Vec<u8>) -> Result<(u64, bool)> {
5626    match decode_pool_meta_replica(data) {
5627        PoolMetaReplica::Valid { revision, committed, .. } if revision.is_generation_protocol() => {
5628            Ok((revision.generation, committed))
5629        }
5630        _ => Err(Error::other("test pool metadata is not a valid V3 replica")),
5631    }
5632}
5633
5634async fn read_pool_meta_replica<S>(pool: Arc<S>, no_lock: bool) -> PoolMetaReplicaRead
5635where
5636    S: EcstoreObjectIO,
5637{
5638    let result = if no_lock {
5639        read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_NAME)
5640            .await
5641            .map(|(data, object_info)| (data, object_info.etag))
5642    } else {
5643        read_config_preserve_empty(pool, POOL_META_NAME)
5644            .await
5645            .map(|data| (data, None))
5646    };
5647    match result {
5648        Ok((data, etag)) => PoolMetaReplicaRead {
5649            replica: decode_pool_meta_replica(data),
5650            cas: etag
5651                .filter(|etag| !etag.trim().is_empty())
5652                .map(PoolMetaCasToken::Existing)
5653                .unwrap_or(PoolMetaCasToken::Unsafe),
5654        },
5655        Err(Error::ConfigNotFound) => PoolMetaReplicaRead {
5656            replica: PoolMetaReplica::Missing,
5657            cas: PoolMetaCasToken::Missing,
5658        },
5659        Err(err) => PoolMetaReplicaRead {
5660            replica: PoolMetaReplica::Unreadable(Arc::new(err)),
5661            cas: PoolMetaCasToken::Unsafe,
5662        },
5663    }
5664}
5665
5666fn select_pool_meta_replica_reads(reads: Vec<PoolMetaReplicaRead>) -> Result<PoolMetaSelection> {
5667    if reads.is_empty() {
5668        return Err(Error::other("pool metadata recovery required: no storage pools available"));
5669    }
5670
5671    let cas_tokens = reads.iter().map(|read| read.cas.clone()).collect();
5672    let mut committed = Vec::<(usize, Vec<u8>, PoolMetaCommittedCandidate)>::new();
5673    let mut needs_repair = false;
5674    let mut repair_write_safe = true;
5675    let mut missing = 0usize;
5676    let mut unusable = Vec::new();
5677    let mut observed_version = POOL_META_V1_VERSION;
5678    let mut generation_protocol_observed = false;
5679    let mut generation_identity = None;
5680
5681    for (idx, read) in reads.into_iter().enumerate() {
5682        match read.replica {
5683            PoolMetaReplica::Missing => {
5684                missing += 1;
5685                needs_repair = true;
5686            }
5687            PoolMetaReplica::Corrupt(reason) => {
5688                needs_repair = true;
5689                unusable.push(format!("pool {idx} is corrupt: {reason}"));
5690            }
5691            PoolMetaReplica::Unreadable(reason) => {
5692                needs_repair = true;
5693                repair_write_safe = false;
5694                unusable.push(format!("pool {idx} is unreadable: {reason}"));
5695            }
5696            PoolMetaReplica::Incompatible(reason) => {
5697                return Err(Error::other(format!(
5698                    "pool metadata recovery required: pool {idx} is incompatible ({reason}); upgrade or restore a compatible replica without overwriting it"
5699                )));
5700            }
5701            PoolMetaReplica::Valid {
5702                raw,
5703                canonical,
5704                meta,
5705                revision,
5706                committed: is_committed,
5707                previous,
5708            } => {
5709                generation_protocol_observed |= revision.is_generation_protocol();
5710                if revision.is_generation_protocol() {
5711                    let identity = (
5712                        revision
5713                            .cluster_id
5714                            .expect("validated V3 revision should carry a cluster identity"),
5715                        revision.epoch,
5716                    );
5717                    if generation_identity.is_some_and(|current| current != identity) {
5718                        return Err(Error::other(
5719                            "pool metadata recovery required: V3 replicas disagree on cluster identity or epoch",
5720                        ));
5721                    }
5722                    generation_identity = Some(identity);
5723                }
5724                if is_committed {
5725                    observed_version = observed_version.max(meta.version);
5726                    committed.push((
5727                        idx,
5728                        raw,
5729                        PoolMetaCommittedCandidate {
5730                            canonical,
5731                            meta,
5732                            revision,
5733                        },
5734                    ));
5735                } else if let Some(previous) = previous {
5736                    observed_version = observed_version.max(previous.meta.version);
5737                    needs_repair = true;
5738                    committed.push((idx, raw, *previous));
5739                } else {
5740                    needs_repair = true;
5741                    unusable.push(format!("pool {idx} contains an uncommitted initial generation"));
5742                }
5743            }
5744        }
5745    }
5746
5747    if committed.is_empty() && missing > 0 && unusable.is_empty() {
5748        return Ok(PoolMetaSelection {
5749            meta: PoolMeta::default(),
5750            revision: PoolMetaRevision::legacy(0),
5751            canonical: None,
5752            replica_state: PoolMetaReplicaState {
5753                needs_repair: false,
5754                repair_write_safe: true,
5755            },
5756            cas_tokens,
5757            absent: true,
5758            generation_protocol_observed,
5759            generation_identity,
5760        });
5761    }
5762    if committed.is_empty() {
5763        return Err(Error::other(format!(
5764            "pool metadata recovery required: no valid committed replica is available ({})",
5765            unusable.join("; ")
5766        )));
5767    }
5768
5769    if committed
5770        .iter()
5771        .any(|(_, _, candidate)| candidate.revision.is_generation_protocol())
5772    {
5773        let highest = committed
5774            .iter()
5775            .filter(|(_, _, candidate)| candidate.revision.is_generation_protocol())
5776            .map(|(_, _, candidate)| candidate.revision.generation)
5777            .max()
5778            .ok_or_else(|| Error::other("pool metadata recovery required: no committed V3 generation is available"))?;
5779        let mut selected: Option<(usize, &PoolMetaCommittedCandidate)> = None;
5780        for (idx, _, candidate) in &committed {
5781            if !candidate.revision.is_generation_protocol() || candidate.revision.generation != highest {
5782                needs_repair = true;
5783                continue;
5784            }
5785            if let Some((selected_idx, selected_candidate)) = selected {
5786                if selected_candidate.canonical != candidate.canonical
5787                    || selected_candidate.revision.transaction_id != candidate.revision.transaction_id
5788                {
5789                    return Err(Error::other(format!(
5790                        "pool metadata recovery required: committed generation {highest} diverges between pools {selected_idx} and {idx}"
5791                    )));
5792                }
5793            } else {
5794                selected = Some((*idx, candidate));
5795            }
5796        }
5797        let (_, selected) = selected
5798            .ok_or_else(|| Error::other("pool metadata recovery required: highest V3 generation has no valid snapshot"))?;
5799        for (_, raw, candidate) in &committed {
5800            needs_repair |= candidate.canonical != selected.canonical || raw != &selected.canonical;
5801        }
5802        return Ok(PoolMetaSelection {
5803            meta: selected.meta.clone(),
5804            revision: selected.revision,
5805            canonical: Some(selected.canonical.clone()),
5806            replica_state: PoolMetaReplicaState {
5807                needs_repair,
5808                repair_write_safe,
5809            },
5810            cas_tokens,
5811            absent: false,
5812            generation_protocol_observed,
5813            generation_identity,
5814        });
5815    }
5816
5817    // Legacy V1/V2 has no durable generation. Pool zero remains the commit
5818    // record; divergent backups without it are ambiguous and fail closed.
5819    let mut selected: Option<(usize, Vec<u8>, PoolMetaCommittedCandidate)> = None;
5820    for (idx, raw, candidate) in committed {
5821        if let Some((selected_idx, selected_raw, selected_candidate)) = selected.as_ref() {
5822            if selected_candidate.canonical != candidate.canonical {
5823                if selected_candidate.meta.version == candidate.meta.version && *selected_idx == 0 {
5824                    needs_repair = true;
5825                } else {
5826                    return Err(Error::other(format!(
5827                        "pool metadata recovery required: valid replicas in pools {selected_idx} and {idx} diverge; restore one matching pool.bin snapshot before restart"
5828                    )));
5829                }
5830            } else {
5831                needs_repair |= selected_raw != &raw;
5832            }
5833        } else {
5834            selected = Some((idx, raw, candidate));
5835        }
5836    }
5837    let (_, _, mut selected) =
5838        selected.ok_or_else(|| Error::other("pool metadata recovery required: no valid legacy replica is available"))?;
5839    selected.meta.version = observed_version;
5840    selected.revision.version = observed_version;
5841    Ok(PoolMetaSelection {
5842        meta: selected.meta,
5843        revision: selected.revision,
5844        canonical: Some(selected.canonical),
5845        replica_state: PoolMetaReplicaState {
5846            needs_repair,
5847            repair_write_safe,
5848        },
5849        cas_tokens,
5850        absent: false,
5851        generation_protocol_observed,
5852        generation_identity,
5853    })
5854}
5855
5856#[cfg(test)]
5857fn select_pool_meta_replica(replicas: Vec<PoolMetaReplica>) -> Result<PoolMetaSelection> {
5858    select_pool_meta_replica_reads(
5859        replicas
5860            .into_iter()
5861            .map(|replica| PoolMetaReplicaRead {
5862                replica,
5863                cas: PoolMetaCasToken::Unsafe,
5864            })
5865            .collect(),
5866    )
5867}
5868
5869async fn read_pool_meta_replicas<S>(pools: Vec<Arc<S>>, no_lock: bool) -> Vec<PoolMetaReplicaRead>
5870where
5871    S: EcstoreObjectIO,
5872{
5873    let reads = join_all(pools.into_iter().map(|pool| read_pool_meta_replica(pool, no_lock))).await;
5874    #[cfg(feature = "e2e-test-hooks")]
5875    if STARTUP_CAS_OBSERVATION.try_with(|_| ()).is_ok() {
5876        let batch = uuid::Uuid::new_v4();
5877        for (pool, read) in reads.iter().enumerate() {
5878            let mut observation = serde_json::json!({
5879                "kind": "replica-read", "object": POOL_META_NAME, "batch": batch, "pool": pool,
5880                "cas": match &read.cas {
5881                    PoolMetaCasToken::Missing => "missing",
5882                    PoolMetaCasToken::Existing(_) => "existing",
5883                    PoolMetaCasToken::Unsafe => "unsafe",
5884                },
5885                "etag": match &read.cas { PoolMetaCasToken::Existing(etag) => Some(etag), _ => None },
5886            });
5887            match &read.replica {
5888                PoolMetaReplica::Valid {
5889                    raw,
5890                    canonical,
5891                    meta,
5892                    revision,
5893                    committed,
5894                    previous,
5895                } => {
5896                    observation["state"] = serde_json::json!("valid");
5897                    observation["committed"] = serde_json::json!(committed);
5898                    observation["version"] = serde_json::json!(revision.version);
5899                    observation["cluster_id"] = serde_json::json!(revision.cluster_id);
5900                    observation["epoch"] = serde_json::json!(revision.epoch);
5901                    observation["generation"] = serde_json::json!(revision.generation);
5902                    observation["transaction_id"] = serde_json::json!(revision.transaction_id);
5903                    observation["pool_count"] = serde_json::json!(meta.pools.len());
5904                    observation["payload_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(canonical)));
5905                    observation["raw_sha256"] = serde_json::json!(rustfs_utils::crypto::hex(Sha256::digest(raw)));
5906                    match pool_meta_test_pools_sha256(meta) {
5907                        Ok(digest) => observation["persisted_pools_sha256"] = serde_json::json!(digest),
5908                        Err(err) => observation["observation_error"] = serde_json::json!(err.to_string()),
5909                    }
5910                    if let Some(previous) = previous {
5911                        match pool_meta_test_snapshot(&previous.meta, previous.revision, &previous.canonical) {
5912                            Ok(summary) => observation["previous"] = summary,
5913                            Err(err) => observation["observation_error"] = serde_json::json!(err.to_string()),
5914                        }
5915                    }
5916                }
5917                PoolMetaReplica::Missing => observation["state"] = serde_json::json!("missing"),
5918                PoolMetaReplica::Corrupt(_) => observation["state"] = serde_json::json!("corrupt"),
5919                PoolMetaReplica::Incompatible(_) => observation["state"] = serde_json::json!("incompatible"),
5920                PoolMetaReplica::Unreadable(_) => observation["state"] = serde_json::json!("unreadable"),
5921            }
5922            startup_cas_test_observe(observation);
5923        }
5924    }
5925    reads
5926}
5927
5928fn select_pool_meta_replicas_observing<R>(write_state: &mut PoolMetaWriteState, replicas: Vec<R>) -> Result<PoolMetaSelection>
5929where
5930    R: Into<PoolMetaReplicaRead>,
5931{
5932    let replicas = replicas.into_iter().map(Into::into).collect::<Vec<_>>();
5933    if let Some(err) = pool_meta_replica_read_failure(&replicas, "pool_read") {
5934        return Err(err);
5935    }
5936    match select_pool_meta_replica_reads(replicas) {
5937        Ok(selection) => {
5938            if let Err(err) = write_state.observe_selection(&selection) {
5939                return Err(block_pool_meta_validation(write_state, err, "pool_selection"));
5940            }
5941            Ok(selection)
5942        }
5943        Err(err) => Err(block_pool_meta_validation(write_state, err, "pool_selection")),
5944    }
5945}
5946
5947fn block_pool_meta_validation(write_state: &mut PoolMetaWriteState, err: Error, phase: &'static str) -> Error {
5948    let context = pool_metadata_error(crate::error::PoolMetadataFailure::RecoveryRequired, phase, Some(Arc::new(err)));
5949    write_state.block_with_context(context.clone());
5950    Error::other(context)
5951}
5952
5953fn pool_meta_replica_read_failure(replicas: &[PoolMetaReplicaRead], operation: &str) -> Option<Error> {
5954    replicas.iter().find_map(|read| match &read.replica {
5955        PoolMetaReplica::Unreadable(source) => {
5956            let mut context =
5957                pool_metadata_error(crate::error::PoolMetadataFailure::ReadUnavailable, "pool_read", Some(Arc::clone(source)));
5958            context.operation = operation.to_owned();
5959            Some(Error::other(context))
5960        }
5961        _ => None,
5962    })
5963}
5964
5965fn select_pool_meta_replicas_for_read_probe<R>(
5966    write_state: &PoolMetaWriteState,
5967    replicas: Vec<R>,
5968    operation: &str,
5969) -> Result<PoolMetaSelection>
5970where
5971    R: Into<PoolMetaReplicaRead>,
5972{
5973    let replicas = replicas.into_iter().map(Into::into).collect::<Vec<_>>();
5974    if let Some(err) = pool_meta_replica_read_failure(&replicas, operation) {
5975        return Err(err);
5976    }
5977    let selection = select_pool_meta_replica_reads(replicas)?;
5978    write_state.validate_selection(&selection)?;
5979    selection.replica_state.ensure_write_safe(operation)?;
5980    if selection.absent && (write_state.expected_cluster_id.is_some() || write_state.identity_initialized.is_some()) {
5981        write_state
5982            .validate_missing_metadata_can_initialize()
5983            .map_err(MissingMetadataRejection::into_error)?;
5984    }
5985    Ok(selection)
5986}
5987
5988async fn load_pool_meta_replicas<S>(pools: Vec<Arc<S>>, no_lock: bool) -> Result<PoolMetaSelection>
5989where
5990    S: EcstoreObjectIO,
5991{
5992    select_pool_meta_replica_reads(read_pool_meta_replicas(pools, no_lock).await)
5993}
5994
5995async fn load_pool_meta_replicas_observing<S>(
5996    pools: Vec<Arc<S>>,
5997    no_lock: bool,
5998    write_state: &mut PoolMetaWriteState,
5999) -> Result<PoolMetaSelection>
6000where
6001    S: EcstoreObjectIO,
6002{
6003    let replicas = read_pool_meta_replicas(pools, no_lock).await;
6004    select_pool_meta_replicas_observing(write_state, replicas)
6005}
6006
6007async fn load_pool_meta_replicas_for_read_probe<S>(
6008    pools: Vec<Arc<S>>,
6009    no_lock: bool,
6010    write_state: &PoolMetaWriteState,
6011    operation: &str,
6012) -> Result<PoolMetaSelection>
6013where
6014    S: EcstoreObjectIO,
6015{
6016    let replicas = read_pool_meta_replicas(pools, no_lock).await;
6017    select_pool_meta_replicas_for_read_probe(write_state, replicas, operation).map_err(|err| {
6018        if err.pool_metadata_failure().is_some() {
6019            err
6020        } else {
6021            Error::other(pool_metadata_error(
6022                crate::error::PoolMetadataFailure::RecoveryRequired,
6023                "read_probe",
6024                Some(Arc::new(err)),
6025            ))
6026        }
6027    })
6028}
6029
6030#[derive(Debug, Clone, Serialize, Deserialize)]
6031#[serde(deny_unknown_fields)]
6032struct PersistedPoolMetaV3 {
6033    version: u16,
6034    cluster_id: String,
6035    epoch: u64,
6036    generation: u64,
6037    transaction_id: String,
6038    committed: bool,
6039    pools: Vec<PersistedPoolStatus>,
6040    previous: Option<PersistedPoolMetaV3Previous>,
6041}
6042
6043#[derive(Debug, Clone, Serialize, Deserialize)]
6044#[serde(deny_unknown_fields)]
6045struct PersistedPoolMetaV3Previous {
6046    version: u16,
6047    cluster_id: Option<String>,
6048    epoch: u64,
6049    generation: u64,
6050    transaction_id: Option<String>,
6051    pools: Vec<PersistedPoolStatus>,
6052}
6053
6054#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
6055#[serde(deny_unknown_fields)]
6056struct PersistedPoolMetaIdentity {
6057    version: u16,
6058    cluster_id: uuid::Uuid,
6059    epoch: u64,
6060    initialized: bool,
6061    fresh_bootstrap_nonce: Option<uuid::Uuid>,
6062}
6063
6064#[derive(Debug)]
6065enum PoolMetaIdentityReplica {
6066    Missing,
6067    Valid(PersistedPoolMetaIdentity),
6068    Corrupt(String),
6069    Incompatible(String),
6070    Unreadable(Arc<Error>),
6071}
6072
6073#[derive(Debug)]
6074struct PoolMetaIdentityRead {
6075    replica: PoolMetaIdentityReplica,
6076    cas: PoolMetaCasToken,
6077}
6078
6079#[derive(Debug)]
6080struct PoolMetaIdentitySelection {
6081    identity: Option<PersistedPoolMetaIdentity>,
6082    needs_repair: bool,
6083    repair_write_safe: bool,
6084    cas_tokens: Vec<PoolMetaCasToken>,
6085    /// Per pool: whether the replica decoded as a valid identity.
6086    valid_replicas: Vec<bool>,
6087}
6088
6089fn encode_pool_meta_identity(identity: PersistedPoolMetaIdentity) -> Result<Vec<u8>> {
6090    let mut data = Vec::new();
6091    data.write_u16::<LittleEndian>(POOL_META_IDENTITY_FORMAT)?;
6092    data.write_u16::<LittleEndian>(POOL_META_IDENTITY_VERSION)?;
6093    identity.serialize(&mut Serializer::new(&mut data))?;
6094    Ok(data)
6095}
6096
6097fn decode_pool_meta_identity(data: &[u8]) -> PoolMetaIdentityReplica {
6098    if data.len() <= 4 {
6099        return PoolMetaIdentityReplica::Corrupt("identity payload is empty or truncated".to_string());
6100    }
6101    let format = LittleEndian::read_u16(&data[0..2]);
6102    let version = LittleEndian::read_u16(&data[2..4]);
6103    if format != POOL_META_IDENTITY_FORMAT || version != POOL_META_IDENTITY_VERSION {
6104        return PoolMetaIdentityReplica::Incompatible(format!("unsupported identity format {format} version {version}"));
6105    }
6106    let identity = match rmp_serde::from_slice::<PersistedPoolMetaIdentity>(&data[4..]) {
6107        Ok(identity) => identity,
6108        Err(err) => return PoolMetaIdentityReplica::Corrupt(format!("identity payload is not decodable: {err}")),
6109    };
6110    let invalid_bootstrap_nonce = identity
6111        .fresh_bootstrap_nonce
6112        .is_some_and(|nonce| nonce.is_nil() || nonce == uuid::Uuid::max());
6113    if identity.version != POOL_META_IDENTITY_VERSION
6114        || identity.cluster_id.is_nil()
6115        || identity.cluster_id == uuid::Uuid::max()
6116        || identity.epoch == 0
6117        || invalid_bootstrap_nonce
6118        || identity.initialized == identity.fresh_bootstrap_nonce.is_some()
6119    {
6120        return PoolMetaIdentityReplica::Corrupt(
6121            "identity payload contains an invalid version, UUID, epoch, or fresh-bootstrap proof".to_string(),
6122        );
6123    }
6124    PoolMetaIdentityReplica::Valid(identity)
6125}
6126
6127#[cfg(test)]
6128pub(crate) fn pool_meta_identity_initialized_for_test(data: &[u8]) -> Result<bool> {
6129    match decode_pool_meta_identity(data) {
6130        PoolMetaIdentityReplica::Valid(identity) => Ok(identity.initialized),
6131        _ => Err(Error::other("test pool metadata identity is not valid")),
6132    }
6133}
6134
6135#[cfg(test)]
6136pub(crate) fn pending_pool_meta_identity_for_test(cluster_id: uuid::Uuid, epoch: u64, nonce: uuid::Uuid) -> Result<Vec<u8>> {
6137    encode_pool_meta_identity(PersistedPoolMetaIdentity {
6138        version: POOL_META_IDENTITY_VERSION,
6139        cluster_id,
6140        epoch,
6141        initialized: false,
6142        fresh_bootstrap_nonce: Some(nonce),
6143    })
6144}
6145
6146#[cfg(test)]
6147pub(crate) fn initialized_pool_meta_identity_for_test(cluster_id: uuid::Uuid, epoch: u64) -> Result<Vec<u8>> {
6148    encode_pool_meta_identity(PersistedPoolMetaIdentity {
6149        version: POOL_META_IDENTITY_VERSION,
6150        cluster_id,
6151        epoch,
6152        initialized: true,
6153        fresh_bootstrap_nonce: None,
6154    })
6155}
6156
6157async fn read_pool_meta_identity_replica<S>(pool: Arc<S>) -> PoolMetaIdentityRead
6158where
6159    S: EcstoreObjectIO,
6160{
6161    match read_config_no_lock_preserve_empty_with_metadata(pool, POOL_META_IDENTITY_NAME).await {
6162        Ok((data, object_info)) => PoolMetaIdentityRead {
6163            replica: decode_pool_meta_identity(&data),
6164            cas: object_info
6165                .etag
6166                .filter(|etag| !etag.trim().is_empty())
6167                .map(PoolMetaCasToken::Existing)
6168                .unwrap_or(PoolMetaCasToken::Unsafe),
6169        },
6170        Err(Error::ConfigNotFound) => PoolMetaIdentityRead {
6171            replica: PoolMetaIdentityReplica::Missing,
6172            cas: PoolMetaCasToken::Missing,
6173        },
6174        Err(err) => PoolMetaIdentityRead {
6175            replica: PoolMetaIdentityReplica::Unreadable(Arc::new(err)),
6176            cas: PoolMetaCasToken::Unsafe,
6177        },
6178    }
6179}
6180
6181fn select_pool_meta_identity(
6182    reads: Vec<PoolMetaIdentityRead>,
6183    expected_cluster_id: uuid::Uuid,
6184) -> Result<PoolMetaIdentitySelection> {
6185    let cas_tokens = reads.iter().map(|read| read.cas.clone()).collect();
6186    let valid_replicas = reads
6187        .iter()
6188        .map(|read| matches!(read.replica, PoolMetaIdentityReplica::Valid(_)))
6189        .collect();
6190    let mut selected: Option<PersistedPoolMetaIdentity> = None;
6191    let mut needs_repair = false;
6192    let mut repair_write_safe = true;
6193    let mut unusable = Vec::new();
6194    for (idx, read) in reads.into_iter().enumerate() {
6195        match read.replica {
6196            PoolMetaIdentityReplica::Missing => needs_repair = true,
6197            PoolMetaIdentityReplica::Corrupt(reason) => {
6198                needs_repair = true;
6199                unusable.push(format!("pool {idx} identity is corrupt: {reason}"));
6200            }
6201            PoolMetaIdentityReplica::Unreadable(reason) => {
6202                needs_repair = true;
6203                repair_write_safe = false;
6204                unusable.push(format!("pool {idx} identity is unreadable: {reason}"));
6205            }
6206            PoolMetaIdentityReplica::Incompatible(reason) => {
6207                return Err(Error::other(format!("pool metadata incompatible: pool {idx} identity uses {reason}")));
6208            }
6209            PoolMetaIdentityReplica::Valid(identity) => {
6210                if identity.cluster_id != expected_cluster_id {
6211                    return Err(Error::other(format!(
6212                        "pool metadata incompatible: pool {idx} identity {} does not match deployment {expected_cluster_id}",
6213                        identity.cluster_id
6214                    )));
6215                }
6216                if let Some(current) = selected {
6217                    if current.cluster_id != identity.cluster_id || current.epoch != identity.epoch {
6218                        return Err(Error::other(
6219                            "pool metadata recovery required: identity replicas disagree on cluster identity or epoch",
6220                        ));
6221                    }
6222                    if current.initialized != identity.initialized {
6223                        needs_repair = true;
6224                        selected = Some(if current.initialized { current } else { identity });
6225                    } else if !current.initialized && current.fresh_bootstrap_nonce != identity.fresh_bootstrap_nonce {
6226                        return Err(Error::other(
6227                            "pool metadata recovery required: pending identity replicas disagree on fresh-bootstrap proof",
6228                        ));
6229                    }
6230                } else {
6231                    selected = Some(identity);
6232                }
6233            }
6234        }
6235    }
6236    if selected.is_none() && !unusable.is_empty() {
6237        return Err(Error::other(format!(
6238            "pool metadata recovery required: no valid cluster identity replica is available ({})",
6239            unusable.join("; ")
6240        )));
6241    }
6242    Ok(PoolMetaIdentitySelection {
6243        identity: selected,
6244        needs_repair,
6245        repair_write_safe,
6246        cas_tokens,
6247        valid_replicas,
6248    })
6249}
6250
6251async fn load_pool_meta_identity<S>(pools: Vec<Arc<S>>, expected_cluster_id: uuid::Uuid) -> Result<PoolMetaIdentitySelection>
6252where
6253    S: EcstoreObjectIO,
6254{
6255    let reads = join_all(pools.into_iter().map(read_pool_meta_identity_replica)).await;
6256    if let Some(source) = reads.iter().find_map(|read| match &read.replica {
6257        PoolMetaIdentityReplica::Unreadable(source) => Some(Arc::clone(source)),
6258        _ => None,
6259    }) {
6260        return Err(Error::other(pool_metadata_error(
6261            crate::error::PoolMetadataFailure::ReadUnavailable,
6262            "identity_read",
6263            Some(source),
6264        )));
6265    }
6266    select_pool_meta_identity(reads, expected_cluster_id)
6267}
6268
6269async fn load_pool_meta_identity_selection_observing<S>(
6270    pools: Vec<Arc<S>>,
6271    write_state: &mut PoolMetaWriteState,
6272    expected_cluster_id: uuid::Uuid,
6273) -> Result<PoolMetaIdentitySelection>
6274where
6275    S: EcstoreObjectIO,
6276{
6277    let selection = match load_pool_meta_identity(pools, expected_cluster_id).await {
6278        Ok(selection) => selection,
6279        Err(err) => {
6280            if err
6281                .pool_metadata_failure()
6282                .is_some_and(|context| context.kind == crate::error::PoolMetadataFailure::ReadUnavailable)
6283            {
6284                return Err(err);
6285            }
6286            return Err(block_pool_meta_validation(write_state, err, "identity_selection"));
6287        }
6288    };
6289    if let Err(err) = write_state.observe_identity(&selection) {
6290        return Err(block_pool_meta_validation(write_state, err, "identity_selection"));
6291    }
6292    Ok(selection)
6293}
6294
6295fn persisted_pool_meta_v3_previous(candidate: &PoolMetaCommittedCandidate) -> PersistedPoolMetaV3Previous {
6296    PersistedPoolMetaV3Previous {
6297        version: candidate.revision.version,
6298        cluster_id: candidate.revision.cluster_id.map(|id| id.to_string()),
6299        epoch: candidate.revision.epoch,
6300        generation: candidate.revision.generation,
6301        transaction_id: candidate.revision.transaction_id.map(|id| id.to_string()),
6302        pools: candidate.meta.pools.iter().map(Into::into).collect(),
6303    }
6304}
6305
6306fn encode_pool_meta_v3_envelope(
6307    meta: &PoolMeta,
6308    revision: PoolMetaRevision,
6309    committed: bool,
6310    previous: Option<&PoolMetaCommittedCandidate>,
6311) -> Result<Vec<u8>> {
6312    if meta.dont_save {
6313        return Ok(Vec::new());
6314    }
6315    let cluster_id = revision
6316        .cluster_id
6317        .ok_or_else(|| Error::other("pool metadata V3 save failed: cluster identity is not initialized"))?;
6318    let transaction_id = revision
6319        .transaction_id
6320        .ok_or_else(|| Error::other("pool metadata V3 save failed: transaction id is not initialized"))?;
6321    if revision.version != POOL_META_GENERATION_VERSION || revision.epoch == 0 || revision.generation == 0 {
6322        return Err(Error::other("pool metadata V3 save failed: invalid durable revision"));
6323    }
6324    let persisted = PersistedPoolMetaV3 {
6325        version: POOL_META_GENERATION_VERSION,
6326        cluster_id: cluster_id.to_string(),
6327        epoch: revision.epoch,
6328        generation: revision.generation,
6329        transaction_id: transaction_id.to_string(),
6330        committed,
6331        pools: meta.pools.iter().map(Into::into).collect(),
6332        previous: previous.map(persisted_pool_meta_v3_previous),
6333    };
6334    let mut data = Vec::new();
6335    data.write_u16::<LittleEndian>(POOL_META_FORMAT)?;
6336    data.write_u16::<LittleEndian>(POOL_META_GENERATION_VERSION)?;
6337    persisted.serialize(&mut Serializer::new(&mut data))?;
6338    Ok(data)
6339}
6340
6341enum PoolMetaPersistenceFence<'a> {
6342    Distributed(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>),
6343    Activation(&'a PoolRebalanceActivationFence),
6344}
6345
6346impl PoolMetaPersistenceFence<'_> {
6347    fn ensure_held(&self) -> Result<()> {
6348        match self {
6349            Self::Distributed(Some(signal)) if signal.is_lost() => {
6350                Err(Error::other("pool metadata distributed fence was lost before a replica write"))
6351            }
6352            Self::Activation(fence) => fence.ensure_held(),
6353            _ => Ok(()),
6354        }
6355    }
6356
6357    fn add_to_options(&self, opts: &mut ObjectOptions) {
6358        match self {
6359            Self::Distributed(Some(signal)) => opts.add_namespace_lock_lost_signal(Arc::clone(signal)),
6360            Self::Activation(fence) => fence.add_namespace_lock_fence(opts),
6361            Self::Distributed(None) => {}
6362        }
6363    }
6364
6365    fn is_activation(&self) -> bool {
6366        matches!(self, Self::Activation(_))
6367    }
6368}
6369
6370fn pool_meta_cas_preconditions(token: &PoolMetaCasToken, object: &str) -> Result<HTTPPreconditions> {
6371    match token {
6372        PoolMetaCasToken::Missing => Ok(HTTPPreconditions {
6373            if_none_match: Some("*".to_string()),
6374            ..Default::default()
6375        }),
6376        PoolMetaCasToken::Existing(etag) => Ok(HTTPPreconditions {
6377            if_match: Some(etag.clone()),
6378            ..Default::default()
6379        }),
6380        PoolMetaCasToken::Unsafe => Err(Error::other(format!(
6381            "pool metadata recovery required: {object} replica has no safe conditional-write revision"
6382        ))),
6383    }
6384}
6385
6386#[cfg(feature = "e2e-test-hooks")]
6387struct StartupCasObservation {
6388    attempt: uuid::Uuid,
6389    phase: &'static str,
6390    pools: Vec<usize>,
6391}
6392
6393#[cfg(feature = "e2e-test-hooks")]
6394tokio::task_local! {
6395    static STARTUP_CAS_OBSERVATION: StartupCasObservation;
6396}
6397
6398// This scope follows only the directly polled startup future. Spawned work
6399// does not inherit it; receiver evidence retains its existing RPC tuple.
6400#[cfg(feature = "e2e-test-hooks")]
6401pub(crate) async fn startup_cas_test_scope<S, F: std::future::Future>(
6402    attempt: uuid::Uuid,
6403    phase: &'static str,
6404    pools: &[Arc<S>],
6405    future: F,
6406) -> F::Output {
6407    STARTUP_CAS_OBSERVATION
6408        .scope(
6409            StartupCasObservation {
6410                attempt,
6411                phase,
6412                // These identities are never dereferenced or logged. The
6413                // caller and operation keep the same pool Arcs alive.
6414                pools: pools.iter().map(|pool| Arc::as_ptr(pool) as usize).collect(),
6415            },
6416            future,
6417        )
6418        .await
6419}
6420
6421// Direct JSON diagnostics are independent of the startup tracing subscriber.
6422#[cfg(feature = "e2e-test-hooks")]
6423pub(crate) fn startup_cas_test_observe(mut observation: serde_json::Value) {
6424    let Some(nonce) = std::env::var("RUSTFS_E2E_STARTUP_CAS_NONCE")
6425        .ok()
6426        .and_then(|value| uuid::Uuid::parse_str(&value).ok())
6427    else {
6428        return;
6429    };
6430    observation["nonce"] = serde_json::json!(nonce);
6431    observation["pid"] = serde_json::json!(std::process::id());
6432    let _ = STARTUP_CAS_OBSERVATION.try_with(|scope| {
6433        observation["attempt"] = serde_json::json!(scope.attempt);
6434        observation["startup_phase"] = serde_json::json!(scope.phase);
6435    });
6436    let line = format!("RUSTFS_E2E_STARTUP_CAS {observation}\n");
6437    let _ = std::io::Write::write_all(&mut std::io::stderr().lock(), line.as_bytes());
6438}
6439
6440#[cfg(feature = "e2e-test-hooks")]
6441fn pool_meta_test_pools_sha256(meta: &PoolMeta) -> Result<String> {
6442    let pools = meta.pools.iter().map(PersistedPoolStatus::from).collect::<Vec<_>>();
6443    Ok(rustfs_utils::crypto::hex(Sha256::digest(serde_json::to_vec(&pools)?)))
6444}
6445
6446#[cfg(feature = "e2e-test-hooks")]
6447fn pool_meta_test_snapshot(meta: &PoolMeta, revision: PoolMetaRevision, canonical: &[u8]) -> Result<serde_json::Value> {
6448    Ok(serde_json::json!({
6449        "version": revision.version, "cluster_id": revision.cluster_id, "epoch": revision.epoch,
6450        "generation": revision.generation, "transaction_id": revision.transaction_id,
6451        "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(canonical)),
6452        "persisted_pools_sha256": pool_meta_test_pools_sha256(meta)?,
6453    }))
6454}
6455
6456#[cfg(feature = "e2e-test-hooks")]
6457#[derive(Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
6458#[serde(rename_all = "snake_case")]
6459enum PoolMetaPhaseBarrierCase {
6460    PrepareSubset,
6461    PreparedAll,
6462    CommitOne,
6463    BeforePublish,
6464}
6465
6466#[cfg(feature = "e2e-test-hooks")]
6467#[derive(Deserialize)]
6468#[serde(deny_unknown_fields)]
6469struct PoolMetaPhaseBarrierArm {
6470    nonce: uuid::Uuid,
6471    case: PoolMetaPhaseBarrierCase,
6472}
6473
6474#[cfg(feature = "e2e-test-hooks")]
6475struct PoolMetaPhaseBarrier {
6476    directory: std::path::PathBuf,
6477    arm: PoolMetaPhaseBarrierArm,
6478    transaction_id: uuid::Uuid,
6479    event_gate: tokio::sync::Mutex<()>,
6480    blocked_pools: AtomicUsize,
6481    completed_writes: AtomicUsize,
6482    ready_emitted: AtomicBool,
6483}
6484
6485#[cfg(feature = "e2e-test-hooks")]
6486impl PoolMetaPhaseBarrier {
6487    async fn bind(
6488        previous: &PoolMetaCommittedCandidate,
6489        candidate: &PoolMeta,
6490        revision: PoolMetaRevision,
6491        durable: &[u8],
6492    ) -> Result<Option<Self>> {
6493        let Some(directory) = std::env::var_os("RUSTFS_E2E_POOL_META_BARRIER_DIR").map(std::path::PathBuf::from) else {
6494            return Ok(None);
6495        };
6496        Self::bind_in_directory(directory, previous, candidate, revision, durable).await
6497    }
6498
6499    async fn bind_in_directory(
6500        directory: std::path::PathBuf,
6501        previous: &PoolMetaCommittedCandidate,
6502        candidate: &PoolMeta,
6503        revision: PoolMetaRevision,
6504        durable: &[u8],
6505    ) -> Result<Option<Self>> {
6506        let data = match tokio::fs::read(directory.join("arm.json")).await {
6507            Ok(data) => data,
6508            Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
6509            Err(err) => return Err(err.into()),
6510        };
6511        let arm: PoolMetaPhaseBarrierArm = serde_json::from_slice(&data)?;
6512        if arm.nonce.is_nil() || !directory.is_absolute() {
6513            return Err(Error::other(
6514                "pool metadata test barrier requires an absolute directory and non-nil nonce",
6515            ));
6516        }
6517        let transaction_id = revision
6518            .transaction_id
6519            .ok_or_else(|| Error::other("pool metadata test barrier requires a V3 transaction"))?;
6520        // A new revision can still carry the same persisted pool state. Leave
6521        // the external arm available until recovery can distinguish P from G.
6522        if pool_meta_test_pools_sha256(&previous.meta)? == pool_meta_test_pools_sha256(candidate)? {
6523            return Ok(None);
6524        }
6525        // One external arm binds one attempt, including if its CAS subsequently retries.
6526        let claim = tokio::fs::OpenOptions::new()
6527            .write(true)
6528            .create_new(true)
6529            .open(directory.join("claimed"))
6530            .await;
6531        match claim {
6532            Ok(_) => {}
6533            Err(err) if err.kind() == std::io::ErrorKind::AlreadyExists => return Ok(None),
6534            Err(err) => return Err(err.into()),
6535        }
6536        let barrier = Self {
6537            directory,
6538            arm,
6539            transaction_id,
6540            event_gate: tokio::sync::Mutex::new(()),
6541            blocked_pools: AtomicUsize::new(0),
6542            completed_writes: AtomicUsize::new(0),
6543            ready_emitted: AtomicBool::new(false),
6544        };
6545        barrier
6546            .observe(serde_json::json!({
6547                "kind": "armed",
6548                "previous": pool_meta_test_snapshot(&previous.meta, previous.revision, &previous.canonical)?,
6549                "candidate": pool_meta_test_snapshot(candidate, revision, durable)?,
6550            }))
6551            .await?;
6552        Ok(Some(barrier))
6553    }
6554
6555    async fn observe(&self, mut event: serde_json::Value) -> Result<()> {
6556        use tokio::io::AsyncWriteExt as _;
6557        event["nonce"] = serde_json::json!(self.arm.nonce);
6558        event["pid"] = serde_json::json!(std::process::id());
6559        event["transaction_id"] = serde_json::json!(self.transaction_id);
6560        event["case"] = serde_json::json!(self.arm.case);
6561        let mut line = serde_json::to_vec(&event)?;
6562        line.push(b'\n');
6563        let _event_guard = self.event_gate.lock().await;
6564        let mut file = tokio::fs::OpenOptions::new()
6565            .create(true)
6566            .append(true)
6567            .open(self.directory.join("events.jsonl"))
6568            .await?;
6569        file.write_all(&line).await?;
6570        file.flush().await?;
6571        Ok(())
6572    }
6573
6574    async fn wait_for_release(&self) -> Result<()> {
6575        tokio::time::timeout(std::time::Duration::from_secs(120), async {
6576            loop {
6577                match tokio::fs::metadata(self.directory.join("release")).await {
6578                    Ok(_) => return Ok(()),
6579                    Err(err) if err.kind() == std::io::ErrorKind::NotFound => {}
6580                    Err(err) => return Err(Error::from(err)),
6581                }
6582                tokio::time::sleep(std::time::Duration::from_millis(10)).await;
6583            }
6584        })
6585        .await
6586        .map_err(|_| Error::other("pool metadata test barrier release timed out"))?
6587    }
6588
6589    async fn ready(&self) -> Result<()> {
6590        self.observe(serde_json::json!({"kind": "ready", "cutpoint": self.arm.case}))
6591            .await?;
6592        self.wait_for_release().await
6593    }
6594
6595    async fn ready_after_all_participants(&self) -> Result<()> {
6596        let (blocked, completed) = match self.arm.case {
6597            PoolMetaPhaseBarrierCase::PrepareSubset => (0b0101, 0b1010),
6598            PoolMetaPhaseBarrierCase::CommitOne => (0b0111, 0b1000),
6599            _ => return Ok(()),
6600        };
6601        if self.blocked_pools.load(Ordering::SeqCst) == blocked
6602            && self.completed_writes.load(Ordering::SeqCst) == completed
6603            && !self.ready_emitted.swap(true, Ordering::SeqCst)
6604        {
6605            self.ready().await?;
6606        }
6607        Ok(())
6608    }
6609
6610    async fn before_write(&self, phase: &'static str, pool: usize, data: &[u8]) -> Result<()> {
6611        let blocked = match (self.arm.case, phase) {
6612            (PoolMetaPhaseBarrierCase::PrepareSubset, "prepare_cas") => ![1, 3].contains(&pool),
6613            (PoolMetaPhaseBarrierCase::CommitOne, "commit_cas") => pool != 3,
6614            _ => false,
6615        };
6616        if blocked {
6617            self.observe(serde_json::json!({
6618                "kind": "blocked", "phase": phase, "pool": pool,
6619                "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(data)),
6620            }))
6621            .await?;
6622            self.blocked_pools.fetch_or(1 << pool, Ordering::SeqCst);
6623            self.ready_after_all_participants().await?;
6624            self.wait_for_release().await?;
6625        }
6626        self.observe(serde_json::json!({
6627            "kind": "before-dispatch", "phase": phase, "pool": pool,
6628            "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(data)),
6629        }))
6630        .await
6631    }
6632
6633    async fn after_write(&self, phase: &'static str, pool: usize, data: &[u8], outcome: &PoolMetaCasWriteOutcome) -> Result<()> {
6634        self.observe(serde_json::json!({
6635            "kind": "after-cas", "phase": phase, "pool": pool,
6636            "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(data)),
6637            "ok": outcome.result.is_ok(), "tail_drained": outcome.result.is_ok(),
6638            "etag": outcome.result.as_ref().ok().and_then(|info| info.etag.as_deref()),
6639            "may_have_mutated": outcome.may_have_mutated,
6640        }))
6641        .await?;
6642        if outcome.result.is_ok() {
6643            match (self.arm.case, phase, pool) {
6644                (PoolMetaPhaseBarrierCase::PrepareSubset, "prepare_cas", 1 | 3)
6645                | (PoolMetaPhaseBarrierCase::CommitOne, "commit_cas", 3) => {
6646                    self.completed_writes.fetch_or(1 << pool, Ordering::SeqCst);
6647                    self.ready_after_all_participants().await?;
6648                }
6649                _ => {}
6650            }
6651        }
6652        Ok(())
6653    }
6654}
6655
6656fn pool_meta_cas_options(object: &str, token: &PoolMetaCasToken, fence: &PoolMetaPersistenceFence<'_>) -> Result<ObjectOptions> {
6657    fence.ensure_held()?;
6658    let mut opts = ObjectOptions {
6659        max_parity: true,
6660        write_completion: crate::object_api::WriteCompletion::TailDrained,
6661        no_lock: true,
6662        http_preconditions: Some(pool_meta_cas_preconditions(token, object)?),
6663        ..Default::default()
6664    };
6665    fence.add_to_options(&mut opts);
6666    Ok(opts)
6667}
6668
6669struct PoolMetaCasWriteOutcome {
6670    result: Result<crate::object_api::ObjectInfo>,
6671    source: Option<Arc<Error>>,
6672    may_have_mutated: bool,
6673}
6674
6675impl PoolMetaCasWriteOutcome {
6676    fn not_dispatched(err: Error) -> Self {
6677        Self {
6678            result: Err(err),
6679            source: None,
6680            may_have_mutated: false,
6681        }
6682    }
6683}
6684
6685async fn execute_pool_meta_object_cas<S>(
6686    pool: Arc<S>,
6687    object: &str,
6688    data: Vec<u8>,
6689    opts: ObjectOptions,
6690    fence: &PoolMetaPersistenceFence<'_>,
6691    phase: &'static str,
6692) -> PoolMetaCasWriteOutcome
6693where
6694    S: EcstoreObjectIO,
6695{
6696    // A bounded phase can wait for another replica before this write is polled.
6697    if let Err(err) = fence.ensure_held() {
6698        return PoolMetaCasWriteOutcome::not_dispatched(err);
6699    }
6700    #[cfg(feature = "e2e-test-hooks")]
6701    let observation = std::env::var_os("RUSTFS_E2E_STARTUP_CAS_NONCE").map(|_| {
6702        serde_json::json!({
6703            "kind": "cas", "object": object, "phase": phase,
6704            "pool": STARTUP_CAS_OBSERVATION.try_with(|scope| {
6705                scope.pools.iter().position(|identity| *identity == Arc::as_ptr(&pool) as usize)
6706            }).ok().flatten(),
6707            "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&data)),
6708            "if_match": opts.http_preconditions.as_ref().and_then(|p| p.if_match.as_deref()),
6709            "if_none_match": opts.http_preconditions.as_ref().and_then(|p| p.if_none_match.as_deref()),
6710            "tail_drained": opts.write_completion == crate::object_api::WriteCompletion::TailDrained,
6711            "no_lock": opts.no_lock,
6712        })
6713    });
6714    let result = save_config_with_opts_and_metadata(pool, object, data, &opts).await;
6715    let may_have_mutated = !matches!(&result, Err(Error::PreconditionFailed));
6716    if !may_have_mutated {
6717        record_pool_meta_stale_write_rejection(phase);
6718    }
6719    let mut source = None;
6720    let result = match result {
6721        Ok(object_info) => fence.ensure_held().map(|()| object_info),
6722        Err(err) => {
6723            let original = Arc::new(err);
6724            source = Some(Arc::clone(&original));
6725            if matches!(original.as_ref(), Error::PreconditionFailed) {
6726                Err(Error::PreconditionFailed)
6727            } else {
6728                Err(Error::other(pool_metadata_error(
6729                    crate::error::PoolMetadataFailure::TransactionUnknown,
6730                    phase,
6731                    Some(original),
6732                )))
6733            }
6734        }
6735    };
6736    #[cfg(feature = "e2e-test-hooks")]
6737    if let Some(mut observation) = observation {
6738        observation["ok"] = serde_json::json!(result.is_ok());
6739        observation["etag"] = serde_json::json!(result.as_ref().ok().and_then(|info| info.etag.as_deref()));
6740        observation["mod_time"] = serde_json::json!(
6741            result
6742                .as_ref()
6743                .ok()
6744                .and_then(|info| info.mod_time)
6745                .map(|time| time.unix_timestamp_nanos().to_string())
6746        );
6747        observation["error"] = serde_json::json!(result.as_ref().err().map(ToString::to_string));
6748        startup_cas_test_observe(observation);
6749    }
6750    PoolMetaCasWriteOutcome {
6751        result,
6752        source,
6753        may_have_mutated,
6754    }
6755}
6756
6757async fn save_pool_meta_object_cas<S>(
6758    pool: Arc<S>,
6759    object: &str,
6760    data: Vec<u8>,
6761    token: &PoolMetaCasToken,
6762    fence: &PoolMetaPersistenceFence<'_>,
6763    phase: &'static str,
6764    transaction_arm: &mut PoolMetaTransactionArm,
6765) -> Result<crate::object_api::ObjectInfo>
6766where
6767    S: EcstoreObjectIO,
6768{
6769    let opts = pool_meta_cas_options(object, token, fence)?;
6770    // Cancellation can happen at the very first poll of the storage future.
6771    // Arm before dispatch, but not during read/encode/fence preflight.
6772    let previous_phase = transaction_arm.phase;
6773    transaction_arm.phase = Some(phase);
6774    let outcome = execute_pool_meta_object_cas(pool, object, data, opts, fence, phase).await;
6775    if !outcome.may_have_mutated {
6776        transaction_arm.phase = previous_phase;
6777    }
6778    if let Some(source) = outcome.source
6779        && (!matches!(source.as_ref(), Error::PreconditionFailed)
6780            || transaction_arm
6781                .source
6782                .as_ref()
6783                .is_none_or(|previous| matches!(previous.as_ref(), Error::PreconditionFailed)))
6784    {
6785        transaction_arm.source = Some(source);
6786    }
6787    outcome.result
6788}
6789
6790async fn save_pool_meta_phase<S>(
6791    pools: &[Arc<S>],
6792    data: &[u8],
6793    tokens: &[PoolMetaCasToken],
6794    fence: &PoolMetaPersistenceFence<'_>,
6795    phase: &'static str,
6796    transaction_arm: &mut PoolMetaTransactionArm,
6797    #[cfg(feature = "e2e-test-hooks")] barrier: Option<&PoolMetaPhaseBarrier>,
6798) -> Vec<Result<crate::object_api::ObjectInfo>>
6799where
6800    S: EcstoreObjectIO,
6801{
6802    if pools.len() != tokens.len() {
6803        return vec![Err(Error::other("pool metadata phase has inconsistent replica revisions"))];
6804    }
6805    let options = tokens
6806        .iter()
6807        .map(|token| pool_meta_cas_options(POOL_META_NAME, token, fence))
6808        .collect::<Vec<_>>();
6809    let previous_phase = transaction_arm.phase;
6810    if options.iter().any(Result::is_ok) {
6811        transaction_arm.phase = Some(phase);
6812    }
6813    // The caller owns one arm and the save/namespace guards for the whole phase.
6814    // No replica may restore that arm while another write is still in flight.
6815    let writes = futures::stream::iter(pools.iter().cloned().zip(options).enumerate().map(
6816        |(pool_index, (pool, opts))| async move {
6817            #[cfg(feature = "e2e-test-hooks")]
6818            if opts.is_ok()
6819                && let Some(barrier) = barrier
6820                && let Err(err) = barrier.before_write(phase, pool_index, data).await
6821            {
6822                return (pool_index, PoolMetaCasWriteOutcome::not_dispatched(err));
6823            }
6824            let outcome = match opts {
6825                Ok(opts) => execute_pool_meta_object_cas(pool, POOL_META_NAME, data.to_vec(), opts, fence, phase).await,
6826                Err(err) => PoolMetaCasWriteOutcome::not_dispatched(err),
6827            };
6828            #[cfg(feature = "e2e-test-hooks")]
6829            let outcome = {
6830                let mut outcome = outcome;
6831                if let Some(barrier) = barrier
6832                    && let Err(err) = barrier.after_write(phase, pool_index, data, &outcome).await
6833                    && outcome.result.is_ok()
6834                {
6835                    outcome.result = Err(err);
6836                }
6837                outcome
6838            };
6839            (pool_index, outcome)
6840        },
6841    ))
6842    .buffer_unordered(4);
6843    futures::pin_mut!(writes);
6844    let mut results = Vec::with_capacity(pools.len());
6845    let mut may_have_mutated = false;
6846    let mut source_pool_index: Option<(bool, usize)> = None;
6847    while let Some((pool_index, outcome)) = writes.next().await {
6848        may_have_mutated |= outcome.may_have_mutated;
6849        if let Some(source) = outcome.source
6850            && (outcome.may_have_mutated
6851                || transaction_arm
6852                    .source
6853                    .as_ref()
6854                    .is_none_or(|previous| matches!(previous.as_ref(), Error::PreconditionFailed)))
6855            && source_pool_index.is_none_or(|(previous_mutated, previous_index)| {
6856                (outcome.may_have_mutated && !previous_mutated)
6857                    || (outcome.may_have_mutated == previous_mutated && pool_index < previous_index)
6858            })
6859        {
6860            // A later CAS rejection must not erase an I/O failure if draining is cancelled.
6861            transaction_arm.source = Some(source);
6862            source_pool_index = Some((outcome.may_have_mutated, pool_index));
6863        }
6864        results.push((pool_index, outcome.result));
6865    }
6866    if !may_have_mutated {
6867        transaction_arm.phase = previous_phase;
6868    }
6869    results.sort_unstable_by_key(|(pool_index, _)| *pool_index);
6870    results.into_iter().map(|(_, result)| result).collect()
6871}
6872
6873/// Which pool replicas a cluster-identity write may touch.
6874#[derive(Debug, Clone, Copy)]
6875enum PoolMetaIdentityWriteScope<'a> {
6876    /// Every pool. Creating a pending identity here requires deployment-wide
6877    /// fresh-bootstrap or legacy-adoption proof.
6878    All,
6879    /// Only the listed pools, each of which this process formatted or adopted
6880    /// first-hand. Multi-pool bootstraps whose pools have distinct format
6881    /// creators use this scope: the first pool's creator mints the deployment
6882    /// nonce and every other creator copies it to its own pool, so the elected
6883    /// writer can verify a complete, agreeing pending set instead of trusting
6884    /// an in-process flag it cannot observe on another node.
6885    Pools(&'a [usize]),
6886}
6887
6888fn identity_write_satisfied(
6889    selection: &PoolMetaIdentitySelection,
6890    identity: PersistedPoolMetaIdentity,
6891    scope: PoolMetaIdentityWriteScope<'_>,
6892    targets: &[usize],
6893) -> bool {
6894    if selection.identity != Some(identity) {
6895        return false;
6896    }
6897    match scope {
6898        PoolMetaIdentityWriteScope::All => !selection.needs_repair,
6899        PoolMetaIdentityWriteScope::Pools(_) => targets
6900            .iter()
6901            .all(|pool_idx| selection.valid_replicas.get(*pool_idx).copied().unwrap_or(false)),
6902    }
6903}
6904
6905async fn persist_pool_meta_identity<S>(
6906    pools: Vec<Arc<S>>,
6907    write_state: &mut PoolMetaWriteState,
6908    initialized: bool,
6909    scope: PoolMetaIdentityWriteScope<'_>,
6910    fence: &PoolMetaPersistenceFence<'_>,
6911    transaction_arm: &mut PoolMetaTransactionArm,
6912) -> Result<()>
6913where
6914    S: EcstoreObjectIO,
6915{
6916    let Some(cluster_id) = write_state.expected_cluster_id else {
6917        return Ok(());
6918    };
6919    let targets: Vec<usize> = match scope {
6920        PoolMetaIdentityWriteScope::All => (0..pools.len()).collect(),
6921        PoolMetaIdentityWriteScope::Pools(indices) => {
6922            if initialized {
6923                return Err(Error::other("pool metadata identity commit must address every pool"));
6924            }
6925            if indices
6926                .iter()
6927                .any(|pool_idx| *pool_idx >= pools.len() || !write_state.pool_bootstrap_authority_proven(*pool_idx))
6928            {
6929                return Err(Error::other(
6930                    "pool metadata recovery required: a pending cluster identity can only be attested for pools this startup formatted or adopted first-hand",
6931                ));
6932            }
6933            indices.to_vec()
6934        }
6935    };
6936    for attempt in 0..POOL_META_CAS_MAX_ATTEMPTS {
6937        let selection = load_pool_meta_identity_selection_observing(pools.clone(), write_state, cluster_id).await?;
6938        if !selection.repair_write_safe {
6939            write_state.block_writes();
6940            return Err(Error::other(
6941                "pool metadata recovery required: cluster identity has an unreadable replica",
6942            ));
6943        }
6944        let identity = match (selection.identity, scope) {
6945            // An initialized deployment (for example a pool expansion) never
6946            // reopens bootstrap: first-hand proof for a new pool is not a
6947            // reason to publish a pending identity.
6948            (Some(identity), PoolMetaIdentityWriteScope::Pools(_)) if identity.initialized => return Ok(()),
6949            (Some(identity), _) if identity.initialized || initialized => PersistedPoolMetaIdentity {
6950                initialized: true,
6951                fresh_bootstrap_nonce: None,
6952                ..identity
6953            },
6954            (Some(identity), _) => identity,
6955            // Only the first pool's creator mints the deployment nonce; every
6956            // other creator waits until it is durable and copies it, so two
6957            // concurrent creators can never publish disagreeing replicas.
6958            (None, PoolMetaIdentityWriteScope::Pools(indices)) => {
6959                if !indices.contains(&0) {
6960                    return Ok(());
6961                }
6962                PersistedPoolMetaIdentity {
6963                    version: POOL_META_IDENTITY_VERSION,
6964                    cluster_id,
6965                    epoch: write_state.cluster_epoch.unwrap_or(POOL_META_INITIAL_EPOCH),
6966                    initialized: false,
6967                    fresh_bootstrap_nonce: Some(uuid::Uuid::new_v4()),
6968                }
6969            }
6970            (None, PoolMetaIdentityWriteScope::All) if !initialized && !write_state.bootstrap_identity_proven() => {
6971                write_state.block_writes();
6972                return Err(Error::other(
6973                    "pool metadata recovery required: cannot create a pending cluster identity without verified fresh-bootstrap proof or legacy-adoption proof",
6974                ));
6975            }
6976            (None, PoolMetaIdentityWriteScope::All) => PersistedPoolMetaIdentity {
6977                version: POOL_META_IDENTITY_VERSION,
6978                cluster_id,
6979                epoch: write_state.cluster_epoch.unwrap_or(POOL_META_INITIAL_EPOCH),
6980                initialized,
6981                fresh_bootstrap_nonce: (!initialized).then(uuid::Uuid::new_v4),
6982            },
6983        };
6984        if identity_write_satisfied(&selection, identity, scope, &targets) {
6985            return Ok(());
6986        }
6987        let data = encode_pool_meta_identity(identity)?;
6988        let mut conflict = false;
6989        for (pool_idx, (pool, token)) in pools.iter().cloned().zip(&selection.cas_tokens).enumerate() {
6990            if !targets.contains(&pool_idx) {
6991                continue;
6992            }
6993            match save_pool_meta_object_cas(
6994                pool,
6995                POOL_META_IDENTITY_NAME,
6996                data.clone(),
6997                token,
6998                fence,
6999                "identity_cas",
7000                transaction_arm,
7001            )
7002            .await
7003            {
7004                Ok(_) => {}
7005                Err(Error::PreconditionFailed) => {
7006                    conflict = true;
7007                    break;
7008                }
7009                Err(err) => return Err(err),
7010            }
7011        }
7012        if conflict {
7013            if attempt + 1 < POOL_META_CAS_MAX_ATTEMPTS {
7014                continue;
7015            }
7016            return Err(Error::PreconditionFailed);
7017        }
7018        let confirmed = load_pool_meta_identity_selection_observing(pools.clone(), write_state, cluster_id).await?;
7019        if identity_write_satisfied(&confirmed, identity, scope, &targets) {
7020            return Ok(());
7021        }
7022    }
7023    Err(Error::PreconditionFailed)
7024}
7025
7026pub(crate) async fn load_pool_meta_identity_observing<S>(pools: Vec<Arc<S>>, write_state: &mut PoolMetaWriteState) -> Result<()>
7027where
7028    S: EcstoreObjectIO,
7029{
7030    let Some(cluster_id) = write_state.expected_cluster_id else {
7031        return Ok(());
7032    };
7033    load_pool_meta_identity_selection_observing(pools, write_state, cluster_id)
7034        .await
7035        .map(|_| ())
7036}
7037
7038pub(crate) async fn persist_pool_meta_identity_for_startup<S>(
7039    pools: Vec<Arc<S>>,
7040    write_state: &mut PoolMetaWriteState,
7041    initialized: bool,
7042) -> Result<()>
7043where
7044    S: EcstoreObjectIO,
7045{
7046    let mut transaction_arm = write_state.arm_transaction();
7047    persist_pool_meta_identity(
7048        pools,
7049        write_state,
7050        initialized,
7051        PoolMetaIdentityWriteScope::All,
7052        &PoolMetaPersistenceFence::Distributed(None),
7053        &mut transaction_arm,
7054    )
7055    .await?;
7056    transaction_arm.disarm();
7057    Ok(())
7058}
7059
7060/// Attest, during startup, the pending cluster identity for the pools this
7061/// process formatted or adopted first-hand. The first pool's creator mints the
7062/// deployment nonce; every other creator copies it once it is durable. Nothing
7063/// is written while the deployment is already initialized or while the nonce
7064/// is not yet durable, so callers simply retry through the startup loop.
7065pub(crate) async fn persist_pool_meta_identity_for_attested_pools<S>(
7066    pools: Vec<Arc<S>>,
7067    write_state: &mut PoolMetaWriteState,
7068    pool_indices: &[usize],
7069) -> Result<()>
7070where
7071    S: EcstoreObjectIO,
7072{
7073    let mut transaction_arm = write_state.arm_transaction();
7074    persist_pool_meta_identity(
7075        pools,
7076        write_state,
7077        false,
7078        PoolMetaIdentityWriteScope::Pools(pool_indices),
7079        &PoolMetaPersistenceFence::Distributed(None),
7080        &mut transaction_arm,
7081    )
7082    .await?;
7083    transaction_arm.disarm();
7084    Ok(())
7085}
7086
7087#[derive(Debug, Clone, Serialize, Deserialize)]
7088#[serde(deny_unknown_fields)]
7089struct PersistedPoolMeta {
7090    pub version: u16,
7091    pub pools: Vec<PersistedPoolStatus>,
7092}
7093
7094#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
7095#[serde(deny_unknown_fields)]
7096struct PersistedPoolStatus {
7097    #[serde(rename = "id")]
7098    pub id: usize,
7099    #[serde(rename = "cmdline")]
7100    pub cmd_line: String,
7101    #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
7102    pub last_update: OffsetDateTime,
7103    #[serde(rename = "decommissionInfo")]
7104    pub decommission: Option<PersistedPoolDecommissionInfo>,
7105}
7106
7107#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
7108#[serde(deny_unknown_fields)]
7109struct PersistedPoolDecommissionInfo {
7110    #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
7111    pub start_time: Option<OffsetDateTime>,
7112    #[serde(rename = "startSize")]
7113    pub start_size: usize,
7114    #[serde(rename = "totalSize")]
7115    pub total_size: usize,
7116    #[serde(rename = "currentSize")]
7117    pub current_size: usize,
7118    #[serde(rename = "complete")]
7119    pub complete: bool,
7120    #[serde(rename = "failed")]
7121    pub failed: bool,
7122    #[serde(rename = "canceled")]
7123    pub canceled: bool,
7124    #[serde(rename = "queued", default)]
7125    pub queued: bool,
7126    #[serde(rename = "queuedBuckets", default)]
7127    pub queued_buckets: Vec<String>,
7128    #[serde(rename = "decommissionedBuckets", default)]
7129    pub decommissioned_buckets: Vec<String>,
7130    #[serde(rename = "bucket", default)]
7131    pub bucket: String,
7132    #[serde(rename = "prefix", default)]
7133    pub prefix: String,
7134    #[serde(rename = "object", default)]
7135    pub object: String,
7136    #[serde(rename = "objectsDecommissioned")]
7137    pub items_decommissioned: usize,
7138    #[serde(rename = "objectsDecommissionedFailed")]
7139    pub items_decommission_failed: usize,
7140    #[serde(rename = "bytesDecommissioned")]
7141    pub bytes_done: usize,
7142    #[serde(rename = "bytesDecommissionedFailed")]
7143    pub bytes_failed: usize,
7144    #[serde(rename = "terminalReloadAttemptAt", with = "time::serde::rfc3339::option", default)]
7145    pub terminal_reload_attempt_at: Option<OffsetDateTime>,
7146    #[serde(rename = "terminalReloadFailures", default)]
7147    pub terminal_reload_failures: Vec<String>,
7148    #[serde(rename = "unresolvedEntries", default)]
7149    pub unresolved_entries: Vec<DecommissionUnresolvedEntry>,
7150    #[serde(rename = "capacityReservation", default)]
7151    pub capacity_reservation: Option<DecommissionCapacityReservation>,
7152    #[serde(rename = "capacityBlockedReason", default)]
7153    pub capacity_blocked_reason: Option<String>,
7154}
7155
7156#[derive(Debug, Clone, Serialize, Deserialize)]
7157#[serde(deny_unknown_fields)]
7158struct PersistedPoolMetaV1 {
7159    pub version: u16,
7160    pub pools: Vec<PersistedPoolStatusV1>,
7161}
7162
7163#[derive(Debug, Clone, Serialize, Deserialize)]
7164#[serde(deny_unknown_fields)]
7165struct PersistedPoolStatusV1 {
7166    #[serde(rename = "id")]
7167    pub id: usize,
7168    #[serde(rename = "cmdline")]
7169    pub cmd_line: String,
7170    #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
7171    pub last_update: OffsetDateTime,
7172    #[serde(rename = "decommissionInfo")]
7173    pub decommission: Option<PersistedPoolDecommissionInfoV1>,
7174}
7175
7176#[derive(Debug, Clone, Default, Serialize, Deserialize)]
7177#[serde(deny_unknown_fields)]
7178struct PersistedPoolDecommissionInfoV1 {
7179    #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
7180    pub start_time: Option<OffsetDateTime>,
7181    #[serde(rename = "startSize")]
7182    pub start_size: usize,
7183    #[serde(rename = "totalSize")]
7184    pub total_size: usize,
7185    #[serde(rename = "currentSize")]
7186    pub current_size: usize,
7187    #[serde(rename = "complete")]
7188    pub complete: bool,
7189    #[serde(rename = "failed")]
7190    pub failed: bool,
7191    #[serde(rename = "canceled")]
7192    pub canceled: bool,
7193    #[serde(rename = "queued", default)]
7194    pub queued: bool,
7195    #[serde(rename = "queuedBuckets", default)]
7196    pub queued_buckets: Vec<String>,
7197    #[serde(rename = "decommissionedBuckets", default)]
7198    pub decommissioned_buckets: Vec<String>,
7199    #[serde(rename = "bucket", default)]
7200    pub bucket: String,
7201    #[serde(rename = "prefix", default)]
7202    pub prefix: String,
7203    #[serde(rename = "object", default)]
7204    pub object: String,
7205    #[serde(rename = "objectsDecommissioned")]
7206    pub items_decommissioned: usize,
7207    #[serde(rename = "objectsDecommissionedFailed")]
7208    pub items_decommission_failed: usize,
7209    #[serde(rename = "bytesDecommissioned")]
7210    pub bytes_done: usize,
7211    #[serde(rename = "bytesDecommissionedFailed")]
7212    pub bytes_failed: usize,
7213    #[serde(rename = "terminalReloadAttemptAt", with = "time::serde::rfc3339::option", default)]
7214    pub terminal_reload_attempt_at: Option<OffsetDateTime>,
7215    #[serde(rename = "terminalReloadFailures", default)]
7216    pub terminal_reload_failures: Vec<String>,
7217}
7218
7219#[derive(Debug, Clone, Serialize, Deserialize)]
7220#[serde(deny_unknown_fields)]
7221struct LegacyPoolMeta {
7222    pub version: u16,
7223    pub pools: Vec<LegacyPoolStatus>,
7224    pub dont_save: bool,
7225}
7226
7227#[derive(Debug, Clone, Serialize, Deserialize)]
7228#[serde(deny_unknown_fields)]
7229struct LegacyPoolStatus {
7230    #[serde(rename = "id")]
7231    pub id: usize,
7232    #[serde(rename = "cmdline")]
7233    pub cmd_line: String,
7234    #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
7235    pub last_update: OffsetDateTime,
7236    #[serde(rename = "decommissionInfo")]
7237    pub decommission: Option<LegacyPoolDecommissionInfo>,
7238}
7239
7240#[derive(Debug, Clone, Default, Serialize, Deserialize)]
7241#[serde(deny_unknown_fields)]
7242struct LegacyPoolDecommissionInfo {
7243    #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
7244    pub start_time: Option<OffsetDateTime>,
7245    #[serde(rename = "startSize")]
7246    pub start_size: usize,
7247    #[serde(rename = "totalSize")]
7248    pub total_size: usize,
7249    #[serde(rename = "currentSize")]
7250    pub current_size: usize,
7251    #[serde(rename = "complete")]
7252    pub complete: bool,
7253    #[serde(rename = "failed")]
7254    pub failed: bool,
7255    #[serde(rename = "canceled")]
7256    pub canceled: bool,
7257    #[serde(rename = "objectsDecommissioned")]
7258    pub items_decommissioned: usize,
7259    #[serde(rename = "objectsDecommissionedFailed")]
7260    pub items_decommission_failed: usize,
7261    #[serde(rename = "bytesDecommissioned")]
7262    pub bytes_done: usize,
7263    #[serde(rename = "bytesDecommissionedFailed")]
7264    pub bytes_failed: usize,
7265}
7266
7267fn ensure_pool_meta_payload_version(actual: u16, expected: u16, kind: &str) -> Result<()> {
7268    if actual == expected {
7269        return Ok(());
7270    }
7271    Err(Error::other(format!(
7272        "pool metadata {kind} payload has version {actual}, expected {expected}"
7273    )))
7274}
7275
7276impl TryFrom<PersistedPoolMeta> for PoolMeta {
7277    type Error = Error;
7278
7279    fn try_from(value: PersistedPoolMeta) -> Result<Self> {
7280        ensure_pool_meta_payload_version(value.version, POOL_META_VERSION, "current")?;
7281        let meta = Self {
7282            version: POOL_META_VERSION,
7283            pools: value.pools.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?,
7284            dont_save: false,
7285        };
7286        validate_decommission_capacity_model_cohort(&meta)?;
7287        Ok(meta)
7288    }
7289}
7290
7291impl TryFrom<PersistedPoolMetaV1> for PoolMeta {
7292    type Error = Error;
7293
7294    fn try_from(value: PersistedPoolMetaV1) -> Result<Self> {
7295        ensure_pool_meta_payload_version(value.version, POOL_META_V1_VERSION, "v1")?;
7296        Ok(Self {
7297            version: POOL_META_V1_VERSION,
7298            pools: value.pools.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?,
7299            dont_save: false,
7300        })
7301    }
7302}
7303
7304impl TryFrom<LegacyPoolMeta> for PoolMeta {
7305    type Error = Error;
7306
7307    fn try_from(value: LegacyPoolMeta) -> Result<Self> {
7308        let LegacyPoolMeta {
7309            version,
7310            pools,
7311            dont_save: _,
7312        } = value;
7313        ensure_pool_meta_payload_version(version, POOL_META_V1_VERSION, "legacy v1")?;
7314        Ok(Self {
7315            version: POOL_META_V1_VERSION,
7316            pools: pools.into_iter().map(TryInto::try_into).collect::<Result<Vec<_>>>()?,
7317            dont_save: false,
7318        })
7319    }
7320}
7321
7322impl TryFrom<PersistedPoolStatus> for PoolStatus {
7323    type Error = Error;
7324
7325    fn try_from(value: PersistedPoolStatus) -> Result<Self> {
7326        let decommission = value.decommission.map(TryInto::try_into).transpose()?;
7327        if decommission
7328            .as_ref()
7329            .and_then(|info: &PoolDecommissionInfo| info.capacity_reservation.as_ref())
7330            .is_some_and(|reservation| reservation.source_pool_index != value.id)
7331        {
7332            return Err(Error::other(
7333                "pool metadata load failed: decommission capacity reservation source pool does not match its owner",
7334            ));
7335        }
7336        Ok(Self {
7337            id: value.id,
7338            cmd_line: value.cmd_line,
7339            last_update: value.last_update,
7340            decommission,
7341        })
7342    }
7343}
7344
7345impl TryFrom<PersistedPoolStatusV1> for PoolStatus {
7346    type Error = Error;
7347
7348    fn try_from(value: PersistedPoolStatusV1) -> Result<Self> {
7349        Ok(Self {
7350            id: value.id,
7351            cmd_line: value.cmd_line,
7352            last_update: value.last_update,
7353            decommission: value.decommission.map(TryInto::try_into).transpose()?,
7354        })
7355    }
7356}
7357
7358impl TryFrom<LegacyPoolStatus> for PoolStatus {
7359    type Error = Error;
7360
7361    fn try_from(value: LegacyPoolStatus) -> Result<Self> {
7362        Ok(Self {
7363            id: value.id,
7364            cmd_line: value.cmd_line,
7365            last_update: value.last_update,
7366            decommission: value.decommission.map(TryInto::try_into).transpose()?,
7367        })
7368    }
7369}
7370
7371impl TryFrom<PersistedPoolDecommissionInfo> for PoolDecommissionInfo {
7372    type Error = Error;
7373
7374    fn try_from(value: PersistedPoolDecommissionInfo) -> Result<Self> {
7375        validate_decommission_terminal_state(value.complete, value.failed, value.canceled)?;
7376        validate_decommission_capacity_reservation(value.capacity_reservation.as_ref())?;
7377        Ok(Self {
7378            start_time: value.start_time,
7379            start_size: value.start_size,
7380            total_size: value.total_size,
7381            current_size: value.current_size,
7382            complete: value.complete,
7383            failed: value.failed,
7384            canceled: value.canceled,
7385            queued: value.queued,
7386            queued_buckets: value.queued_buckets,
7387            decommissioned_buckets: value.decommissioned_buckets,
7388            bucket: value.bucket,
7389            prefix: value.prefix,
7390            object: value.object,
7391            stage: String::new(),
7392            items_decommissioned: value.items_decommissioned,
7393            items_decommission_failed: value.items_decommission_failed,
7394            bytes_done: value.bytes_done,
7395            bytes_failed: value.bytes_failed,
7396            terminal_reload_attempt_at: value.terminal_reload_attempt_at,
7397            terminal_reload_failures: value.terminal_reload_failures,
7398            capacity_reservation: value.capacity_reservation,
7399            capacity_blocked_reason: value.capacity_blocked_reason,
7400            unresolved_entries: value.unresolved_entries,
7401            progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed),
7402            progress_save_retry_after: None,
7403            progress_save_last_at: None,
7404        })
7405    }
7406}
7407
7408impl TryFrom<PersistedPoolDecommissionInfoV1> for PoolDecommissionInfo {
7409    type Error = Error;
7410
7411    fn try_from(value: PersistedPoolDecommissionInfoV1) -> Result<Self> {
7412        validate_decommission_terminal_state(value.complete, value.failed, value.canceled)?;
7413        Ok(Self {
7414            start_time: value.start_time,
7415            start_size: value.start_size,
7416            total_size: value.total_size,
7417            current_size: value.current_size,
7418            complete: value.complete,
7419            failed: value.failed,
7420            canceled: value.canceled,
7421            queued: value.queued,
7422            queued_buckets: value.queued_buckets,
7423            decommissioned_buckets: value.decommissioned_buckets,
7424            bucket: value.bucket,
7425            prefix: value.prefix,
7426            object: value.object,
7427            stage: String::new(),
7428            items_decommissioned: value.items_decommissioned,
7429            items_decommission_failed: value.items_decommission_failed,
7430            bytes_done: value.bytes_done,
7431            bytes_failed: value.bytes_failed,
7432            terminal_reload_attempt_at: value.terminal_reload_attempt_at,
7433            terminal_reload_failures: value.terminal_reload_failures,
7434            capacity_reservation: None,
7435            capacity_blocked_reason: None,
7436            unresolved_entries: Vec::new(),
7437            progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed),
7438            progress_save_retry_after: None,
7439            progress_save_last_at: None,
7440        })
7441    }
7442}
7443
7444impl TryFrom<LegacyPoolDecommissionInfo> for PoolDecommissionInfo {
7445    type Error = Error;
7446
7447    fn try_from(value: LegacyPoolDecommissionInfo) -> Result<Self> {
7448        validate_decommission_terminal_state(value.complete, value.failed, value.canceled)?;
7449        Ok(Self {
7450            start_time: value.start_time,
7451            start_size: value.start_size,
7452            total_size: value.total_size,
7453            current_size: value.current_size,
7454            complete: value.complete,
7455            failed: value.failed,
7456            canceled: value.canceled,
7457            queued: false,
7458            queued_buckets: Vec::new(),
7459            decommissioned_buckets: Vec::new(),
7460            bucket: String::new(),
7461            prefix: String::new(),
7462            object: String::new(),
7463            stage: String::new(),
7464            items_decommissioned: value.items_decommissioned,
7465            items_decommission_failed: value.items_decommission_failed,
7466            bytes_done: value.bytes_done,
7467            bytes_failed: value.bytes_failed,
7468            terminal_reload_attempt_at: None,
7469            terminal_reload_failures: Vec::new(),
7470            capacity_reservation: None,
7471            capacity_blocked_reason: None,
7472            unresolved_entries: Vec::new(),
7473            progress_save_item_baseline: value.items_decommissioned.saturating_add(value.items_decommission_failed),
7474            progress_save_retry_after: None,
7475            progress_save_last_at: None,
7476        })
7477    }
7478}
7479
7480impl From<&PoolMeta> for PersistedPoolMeta {
7481    fn from(value: &PoolMeta) -> Self {
7482        Self {
7483            version: POOL_META_VERSION,
7484            pools: value.pools.iter().map(Into::into).collect(),
7485        }
7486    }
7487}
7488
7489impl From<&PoolMeta> for PersistedPoolMetaV1 {
7490    fn from(value: &PoolMeta) -> Self {
7491        Self {
7492            version: POOL_META_V1_VERSION,
7493            pools: value.pools.iter().map(Into::into).collect(),
7494        }
7495    }
7496}
7497
7498impl From<&PoolStatus> for PersistedPoolStatus {
7499    fn from(value: &PoolStatus) -> Self {
7500        Self {
7501            id: value.id,
7502            cmd_line: value.cmd_line.clone(),
7503            last_update: value.last_update,
7504            decommission: value.decommission.as_ref().map(Into::into),
7505        }
7506    }
7507}
7508
7509impl From<&PoolStatus> for PersistedPoolStatusV1 {
7510    fn from(value: &PoolStatus) -> Self {
7511        Self {
7512            id: value.id,
7513            cmd_line: value.cmd_line.clone(),
7514            last_update: value.last_update,
7515            decommission: value.decommission.as_ref().map(Into::into),
7516        }
7517    }
7518}
7519
7520impl From<&PoolDecommissionInfo> for PersistedPoolDecommissionInfo {
7521    fn from(value: &PoolDecommissionInfo) -> Self {
7522        Self {
7523            start_time: value.start_time,
7524            start_size: value.start_size,
7525            total_size: value.total_size,
7526            current_size: value.current_size,
7527            complete: value.complete,
7528            failed: value.failed,
7529            canceled: value.canceled,
7530            queued: value.queued,
7531            queued_buckets: value.queued_buckets.clone(),
7532            decommissioned_buckets: value.decommissioned_buckets.clone(),
7533            bucket: value.bucket.clone(),
7534            prefix: value.prefix.clone(),
7535            object: value.object.clone(),
7536            items_decommissioned: value.items_decommissioned,
7537            items_decommission_failed: value.items_decommission_failed,
7538            bytes_done: value.bytes_done,
7539            bytes_failed: value.bytes_failed,
7540            terminal_reload_attempt_at: value.terminal_reload_attempt_at,
7541            terminal_reload_failures: value.terminal_reload_failures.clone(),
7542            capacity_reservation: value.capacity_reservation.clone(),
7543            capacity_blocked_reason: value.capacity_blocked_reason.clone(),
7544            unresolved_entries: value.unresolved_entries.clone(),
7545        }
7546    }
7547}
7548
7549impl From<&PoolDecommissionInfo> for PersistedPoolDecommissionInfoV1 {
7550    fn from(value: &PoolDecommissionInfo) -> Self {
7551        Self {
7552            start_time: value.start_time,
7553            start_size: value.start_size,
7554            total_size: value.total_size,
7555            current_size: value.current_size,
7556            complete: value.complete,
7557            failed: value.failed,
7558            canceled: value.canceled,
7559            queued: value.queued,
7560            queued_buckets: value.queued_buckets.clone(),
7561            decommissioned_buckets: value.decommissioned_buckets.clone(),
7562            bucket: value.bucket.clone(),
7563            prefix: value.prefix.clone(),
7564            object: value.object.clone(),
7565            items_decommissioned: value.items_decommissioned,
7566            items_decommission_failed: value.items_decommission_failed,
7567            bytes_done: value.bytes_done,
7568            bytes_failed: value.bytes_failed,
7569            terminal_reload_attempt_at: value.terminal_reload_attempt_at,
7570            terminal_reload_failures: value.terminal_reload_failures.clone(),
7571        }
7572    }
7573}
7574
7575#[derive(Debug)]
7576struct PoolMetaSaveOutcome {
7577    transaction_arm: PoolMetaTransactionArm,
7578    committed: PoolMeta,
7579}
7580
7581/// Reconcile only a known, interrupted pool.bin transaction. The caller keeps
7582/// the original block installed and holds the namespace writer through runtime
7583/// publication. Never feed speculative in-memory progress into this repair.
7584async fn load_pool_meta_for_transaction_recovery<S>(
7585    pools: Vec<Arc<S>>,
7586    write_state: &mut PoolMetaWriteState,
7587) -> Result<PoolMetaSelection>
7588where
7589    S: EcstoreObjectIO,
7590{
7591    let result = async {
7592        if let Some(cluster_id) = write_state.expected_cluster_id {
7593            let reads = join_all(pools.iter().cloned().map(read_pool_meta_identity_replica)).await;
7594            for read in &reads {
7595                match &read.replica {
7596                    PoolMetaIdentityReplica::Unreadable(source) => {
7597                        return Err(Error::other(pool_metadata_error(
7598                            crate::error::PoolMetadataFailure::ReadUnavailable,
7599                            "recovery_identity_read",
7600                            Some(Arc::clone(source)),
7601                        )));
7602                    }
7603                    PoolMetaIdentityReplica::Corrupt(_) | PoolMetaIdentityReplica::Incompatible(_) => {
7604                        return Err(Error::other("pool metadata recovery requires operator repair of cluster identity"));
7605                    }
7606                    _ => {}
7607                }
7608            }
7609            let identity = select_pool_meta_identity(reads, cluster_id)?;
7610            if !identity.identity.is_some_and(|identity| identity.initialized) {
7611                return Err(Error::other("pool metadata recovery requires an initialized durable cluster identity"));
7612            }
7613            write_state.observe_identity(&identity)?;
7614        }
7615        let reads = read_pool_meta_replicas(pools, true).await;
7616        if let Some(err) = pool_meta_replica_read_failure(&reads, "pool metadata transaction recovery") {
7617            return Err(err);
7618        }
7619        if reads
7620            .iter()
7621            .any(|read| matches!(&read.replica, PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)))
7622        {
7623            return Err(Error::other(
7624                "pool metadata recovery requires operator repair of corrupt or incompatible replicas",
7625            ));
7626        }
7627        let selection = select_pool_meta_replica_reads(reads)?;
7628        write_state.observe_selection(&selection)?;
7629        if selection.absent || selection.meta.pools.is_empty() {
7630            return Err(Error::other("pool metadata recovery cannot initialize missing metadata"));
7631        }
7632        selection
7633            .replica_state
7634            .ensure_write_safe("pool metadata transaction recovery")?;
7635        Ok(selection)
7636    }
7637    .await;
7638    result.map_err(|err: Error| {
7639        if err.pool_metadata_failure().is_some() {
7640            err
7641        } else {
7642            Error::other(pool_metadata_error(
7643                crate::error::PoolMetadataFailure::RecoveryRequired,
7644                "recovery_validation",
7645                Some(Arc::new(err)),
7646            ))
7647        }
7648    })
7649}
7650
7651async fn repair_pool_meta_transaction<S>(
7652    pools: Vec<Arc<S>>,
7653    write_state: &mut PoolMetaWriteState,
7654    selection: PoolMetaSelection,
7655    fence: &PoolMetaPersistenceFence<'_>,
7656) -> Result<PoolMetaSaveOutcome>
7657where
7658    S: EcstoreObjectIO,
7659{
7660    let mut transaction_arm = write_state.arm_transaction();
7661    let mut expected_meta = selection.meta.clone();
7662    let mut expected_revision = selection.revision;
7663    // A failed first V3 prepare still establishes the format floor. Commit the
7664    // previous authoritative state as V3 instead of erasing that observation.
7665    if selection.generation_protocol_observed && !expected_revision.is_generation_protocol() {
7666        let (cluster_id, epoch) = selection
7667            .generation_identity
7668            .ok_or_else(|| Error::other("pool metadata recovery has no V3 cluster identity"))?;
7669        expected_meta.version = POOL_META_GENERATION_VERSION;
7670        expected_revision = PoolMetaRevision {
7671            version: POOL_META_GENERATION_VERSION,
7672            cluster_id: Some(cluster_id),
7673            epoch,
7674            generation: 1,
7675            transaction_id: Some(uuid::Uuid::new_v4()),
7676        };
7677    }
7678    let expected_canonical = if expected_revision.is_generation_protocol() {
7679        encode_pool_meta_v3_envelope(&expected_meta, expected_revision, true, None)?
7680    } else {
7681        expected_meta.encode_config_data_for_v2_gate(true)?
7682    };
7683    if selection.replica_state.needs_repair {
7684        let data = if expected_revision.is_generation_protocol() {
7685            expected_canonical.clone()
7686        } else {
7687            expected_meta.encode_config_data_for_v2_gate(expected_revision.version == POOL_META_VERSION)?
7688        };
7689        for (pool, token) in pools.iter().cloned().zip(&selection.cas_tokens) {
7690            save_pool_meta_object_cas(pool, POOL_META_NAME, data.clone(), token, fence, "recovery_cas", &mut transaction_arm)
7691                .await?;
7692        }
7693    }
7694    persist_pool_meta_identity(
7695        pools.clone(),
7696        write_state,
7697        true,
7698        PoolMetaIdentityWriteScope::All,
7699        fence,
7700        &mut transaction_arm,
7701    )
7702    .await?;
7703    let confirmed = load_pool_meta_for_transaction_recovery(pools, write_state).await?;
7704    if confirmed.revision != expected_revision
7705        || confirmed.canonical.as_ref() != Some(&expected_canonical)
7706        || confirmed.replica_state.needs_repair
7707        || write_state.identity_requires_repair()
7708    {
7709        return Err(Error::PreconditionFailed);
7710    }
7711    fence.ensure_held()?;
7712    Ok(PoolMetaSaveOutcome {
7713        transaction_arm,
7714        committed: confirmed.meta,
7715    })
7716}
7717
7718impl PoolMetaSaveOutcome {
7719    fn disarm(mut self) {
7720        self.transaction_arm.disarm();
7721    }
7722
7723    fn into_committed(mut self) -> PoolMeta {
7724        self.transaction_arm.disarm();
7725        self.committed
7726    }
7727}
7728
7729impl PoolMeta {
7730    fn current_decommission_movement_update(&self) -> Option<OffsetDateTime> {
7731        self.pools
7732            .iter()
7733            .filter(|pool| pool.decommission.is_some())
7734            .map(|pool| pool.last_update)
7735            .max()
7736    }
7737
7738    fn current_rebalance_movement_update(rebalance_meta: Option<&RebalanceMeta>) -> Option<OffsetDateTime> {
7739        rebalance_meta
7740            .into_iter()
7741            .flat_map(|meta| {
7742                meta.stopped_at.into_iter().chain(
7743                    meta.pool_stats
7744                        .iter()
7745                        .flat_map(|pool| [pool.info.start_time, pool.info.end_time])
7746                        .flatten(),
7747                )
7748            })
7749            .max()
7750    }
7751
7752    pub(crate) fn next_scanner_data_movement_update(
7753        &self,
7754        now: OffsetDateTime,
7755        rebalance_meta: Option<&RebalanceMeta>,
7756    ) -> OffsetDateTime {
7757        let max_seen = self
7758            .current_decommission_movement_update()
7759            .max(Self::current_rebalance_movement_update(rebalance_meta));
7760        match max_seen {
7761            Some(max_seen) => max_seen
7762                .checked_add(Duration::nanoseconds(1))
7763                .map(|next| now.max(next))
7764                .unwrap_or(max_seen),
7765            None => now,
7766        }
7767    }
7768
7769    fn decode_pool_meta_payload(version: u16, payload: &[u8]) -> Result<Self> {
7770        match version {
7771            POOL_META_VERSION => rmp_serde::from_slice::<PersistedPoolMeta>(payload)
7772                .map_err(|err| Error::other(format!("PoolMeta v{POOL_META_VERSION} decode failed: {err}")))?
7773                .try_into(),
7774            POOL_META_V1_VERSION => match rmp_serde::from_slice::<PersistedPoolMetaV1>(payload) {
7775                Ok(meta) => meta.try_into(),
7776                Err(persisted_err) => {
7777                    let legacy: LegacyPoolMeta = rmp_serde::from_slice(payload).map_err(|legacy_err| {
7778                        Error::other(format!(
7779                            "PoolMeta v1 decode failed for both persisted and legacy formats: persisted={persisted_err}; legacy={legacy_err}"
7780                        ))
7781                    })?;
7782                    legacy.try_into()
7783                }
7784            },
7785            _ => Err(Error::other(format!("pool metadata load failed: unknown version {version}"))),
7786        }
7787    }
7788
7789    pub fn new(pools: &[Arc<Sets>], prev_meta: &PoolMeta) -> Self {
7790        let mut new_meta = Self {
7791            version: if prev_meta.version == POOL_META_GENERATION_VERSION || pool_meta_v3_writer_enabled() {
7792                POOL_META_GENERATION_VERSION
7793            } else if prev_meta.version == POOL_META_VERSION || pool_meta_v2_writer_enabled() {
7794                POOL_META_VERSION
7795            } else {
7796                POOL_META_V1_VERSION
7797            },
7798            pools: Vec::new(),
7799            ..Default::default()
7800        };
7801
7802        for (idx, pool) in pools.iter().enumerate() {
7803            let mut skip = false;
7804
7805            for current_pool in prev_meta.pools.iter() {
7806                if current_pool.cmd_line == pool.endpoints.cmd_line {
7807                    new_meta.pools.push(current_pool.clone());
7808                    skip = true;
7809                    break;
7810                }
7811            }
7812
7813            if skip {
7814                continue;
7815            }
7816
7817            new_meta.pools.push(PoolStatus {
7818                cmd_line: pool.endpoints.cmd_line.clone(),
7819                id: idx,
7820                last_update: OffsetDateTime::now_utc(),
7821                decommission: None,
7822            });
7823        }
7824
7825        new_meta
7826    }
7827
7828    pub fn is_suspended(&self, idx: usize) -> bool {
7829        self.pools
7830            .get(idx)
7831            .and_then(|pool| pool.decommission.as_ref())
7832            .is_some_and(is_decommission_suspended)
7833    }
7834
7835    pub(crate) fn has_active_decommission_capacity_reservation(&self, idx: usize) -> bool {
7836        self.pools
7837            .get(idx)
7838            .and_then(|pool| pool.decommission.as_ref())
7839            .is_some_and(|info| {
7840                info.has_decommission_state()
7841                    && is_decommission_active(info.complete, info.failed, info.canceled)
7842                    && info
7843                        .capacity_reservation
7844                        .as_ref()
7845                        .is_some_and(DecommissionCapacityReservation::active)
7846            })
7847    }
7848
7849    pub(crate) fn scanner_pause_backlog_pool_writable(&self, idx: usize) -> bool {
7850        self.pools.get(idx).is_some_and(|pool| {
7851            !pool
7852                .decommission
7853                .as_ref()
7854                .is_some_and(|info| info.has_decommission_state() && !info.failed && !info.canceled)
7855        })
7856    }
7857
7858    fn mark_decommission_progress_saved(&mut self) {
7859        for pool in &mut self.pools {
7860            if let Some(info) = pool.decommission.as_mut() {
7861                info.mark_progress_saved();
7862                info.progress_save_last_at = Some(pool.last_update);
7863            }
7864        }
7865    }
7866
7867    fn decommission_progress_checkpoint(
7868        &self,
7869        idx: usize,
7870        duration: Duration,
7871        now: OffsetDateTime,
7872        rebalance_meta: Option<&RebalanceMeta>,
7873    ) -> Result<Option<DecommissionProgressCheckpoint>> {
7874        let pool_count = self.pools.len();
7875        ensure_valid_decommission_pool_index(pool_count, idx)?;
7876
7877        let Some(pool) = self.pools.get(idx) else {
7878            return Err(invalid_decommission_pool_index_error(pool_count, idx));
7879        };
7880        let Some(info) = pool.decommission.as_ref() else {
7881            return Err(decommission_metadata_not_initialized_error("update decommission metadata timestamp"));
7882        };
7883
7884        if info.progress_save_retry_after.is_some_and(|retry_after| now < retry_after) {
7885            return Ok(None);
7886        }
7887
7888        let last_progress_save = info.progress_save_last_at.unwrap_or(pool.last_update);
7889        let time_threshold_reached = now.unix_timestamp() - last_progress_save.unix_timestamp() >= duration.whole_seconds();
7890        let item_threshold_reached = info.items_since_last_progress_save() >= DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD;
7891        if !time_threshold_reached && !item_threshold_reached {
7892            return Ok(None);
7893        }
7894
7895        Ok(Some(DecommissionProgressCheckpoint {
7896            start_time: info.start_time,
7897            queued: info.queued,
7898            counted_items: info.counted_items(),
7899            checkpoint_at: self.next_scanner_data_movement_update(now, rebalance_meta),
7900            capacity_operation_id: info
7901                .capacity_reservation
7902                .as_ref()
7903                .filter(|reservation| reservation.lease_active_at(now))
7904                .map(|reservation| reservation.operation_id),
7905            capacity_owner_nonce: info
7906                .capacity_reservation
7907                .as_ref()
7908                .filter(|reservation| reservation.lease_active_at(now))
7909                .map(|reservation| reservation.owner_nonce),
7910            capacity_lease_expires_at: info
7911                .capacity_reservation
7912                .as_ref()
7913                .filter(|reservation| reservation.lease_active_at(now))
7914                .map(|_| now + DECOMMISSION_CAPACITY_RESERVATION_TTL),
7915        }))
7916    }
7917
7918    fn commit_decommission_progress_checkpoint(&mut self, idx: usize, checkpoint: DecommissionProgressCheckpoint) -> bool {
7919        let Some(pool) = self.pools.get_mut(idx) else {
7920            return false;
7921        };
7922        let Some(info) = pool.decommission.as_mut() else {
7923            return false;
7924        };
7925
7926        if info.start_time != checkpoint.start_time
7927            || info.queued != checkpoint.queued
7928            || !is_decommission_active(info.complete, info.failed, info.canceled)
7929        {
7930            return false;
7931        }
7932
7933        info.progress_save_item_baseline = info.progress_save_item_baseline.max(checkpoint.counted_items);
7934        info.progress_save_retry_after = None;
7935        info.progress_save_last_at = Some(
7936            info.progress_save_last_at
7937                .map_or(checkpoint.checkpoint_at, |at| at.max(checkpoint.checkpoint_at)),
7938        );
7939        if let (Some(operation_id), Some(owner_nonce), Some(expires_at), Some(reservation)) = (
7940            checkpoint.capacity_operation_id,
7941            checkpoint.capacity_owner_nonce,
7942            checkpoint.capacity_lease_expires_at,
7943            info.capacity_reservation.as_mut(),
7944        ) && reservation.operation_id == operation_id
7945            && reservation.owner_nonce == owner_nonce
7946            && reservation.active()
7947        {
7948            reservation.renewed_at = checkpoint.checkpoint_at;
7949            reservation.expires_at = expires_at;
7950        }
7951        pool.last_update = pool.last_update.max(checkpoint.checkpoint_at);
7952        true
7953    }
7954
7955    fn defer_decommission_progress_checkpoint(
7956        &mut self,
7957        idx: usize,
7958        checkpoint: DecommissionProgressCheckpoint,
7959        retry_after: OffsetDateTime,
7960    ) {
7961        let Some(pool) = self.pools.get_mut(idx) else {
7962            return;
7963        };
7964        let Some(info) = pool.decommission.as_mut() else {
7965            return;
7966        };
7967
7968        if info.start_time == checkpoint.start_time
7969            && info.queued == checkpoint.queued
7970            && is_decommission_active(info.complete, info.failed, info.canceled)
7971        {
7972            info.progress_save_retry_after = Some(retry_after);
7973        }
7974    }
7975
7976    fn load_from_config_data(&mut self, data: Vec<u8>) -> Result<()> {
7977        if data.is_empty() {
7978            return Ok(());
7979        } else if data.len() <= 4 {
7980            return Err(Error::other("pool metadata load failed: metadata payload is too short"));
7981        }
7982
7983        let format = LittleEndian::read_u16(&data[0..2]);
7984        if format != POOL_META_FORMAT {
7985            return Err(Error::other(format!("pool metadata load failed: unknown format {format}")));
7986        }
7987        let version = LittleEndian::read_u16(&data[2..4]);
7988        if !matches!(version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) {
7989            return Err(Error::other(format!("pool metadata load failed: unknown version {version}")));
7990        }
7991
7992        if version == POOL_META_GENERATION_VERSION {
7993            let PoolMetaReplica::Valid {
7994                meta, committed: true, ..
7995            } = decode_pool_meta_replica(data)
7996            else {
7997                return Err(Error::other(
7998                    "pool metadata load failed: V3 payload is corrupt, incompatible, or not committed",
7999                ));
8000            };
8001            *self = meta;
8002            return Ok(());
8003        }
8004
8005        *self = Self::decode_pool_meta_payload(version, &data[4..])?;
8006
8007        if !matches!(self.version, POOL_META_V1_VERSION | POOL_META_VERSION | POOL_META_GENERATION_VERSION) {
8008            return Err(Error::other(format!(
8009                "pool metadata load failed: unexpected decoded version {}",
8010                self.version
8011            )));
8012        }
8013        Ok(())
8014    }
8015
8016    pub async fn load(&mut self, pool: Arc<Sets>, pools: Vec<Arc<Sets>>) -> Result<()> {
8017        let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
8018        let _pool_meta_guard = pool_meta_lock.get_read_lock(get_lock_acquire_timeout()).await?;
8019        let replica_state = self.load_no_lock_from_replicas(pools).await?;
8020        replica_state.ensure_write_safe("pool metadata load failed")?;
8021        Ok(())
8022    }
8023
8024    /// Loads every pool metadata replica while the caller owns the metadata fence
8025    /// or before the namespace-lock RPC surface is ready during startup.
8026    pub(crate) async fn load_no_lock_from_replicas<S>(&mut self, pools: Vec<Arc<S>>) -> Result<PoolMetaReplicaState>
8027    where
8028        S: EcstoreObjectIO,
8029    {
8030        let selection = load_pool_meta_replicas(pools, true).await?;
8031        *self = selection.meta;
8032        Ok(selection.replica_state)
8033    }
8034
8035    pub(crate) async fn load_no_lock_from_replicas_observing<S>(
8036        &mut self,
8037        pools: Vec<Arc<S>>,
8038        write_state: &mut PoolMetaWriteState,
8039    ) -> Result<PoolMetaReplicaState>
8040    where
8041        S: EcstoreObjectIO,
8042    {
8043        let selection = load_pool_meta_replicas_observing(pools, true, write_state).await?;
8044        *self = selection.meta;
8045        Ok(selection.replica_state)
8046    }
8047
8048    pub(crate) async fn load_for_startup_observing<S>(
8049        &mut self,
8050        pools: Vec<Arc<S>>,
8051        write_state: &mut PoolMetaWriteState,
8052    ) -> Result<PoolMetaReplicaState>
8053    where
8054        S: EcstoreObjectIO,
8055    {
8056        // Startup may install a readable replica without authorizing writes.
8057        // Runtime preflight retries must not inherit this bootstrap policy.
8058        if let Some(cluster_id) = write_state.expected_cluster_id {
8059            let reads = join_all(pools.iter().cloned().map(read_pool_meta_identity_replica)).await;
8060            if let Some(source) = reads.iter().find_map(|read| match &read.replica {
8061                PoolMetaIdentityReplica::Unreadable(source) => Some(Arc::clone(source)),
8062                _ => None,
8063            }) {
8064                write_state.block_with_context(pool_metadata_error(
8065                    crate::error::PoolMetadataFailure::RecoveryRequired,
8066                    "startup_identity_read",
8067                    Some(source),
8068                ));
8069            }
8070            let identity = select_pool_meta_identity(reads, cluster_id)
8071                .map_err(|err| block_pool_meta_validation(write_state, err, "startup_identity"))?;
8072            write_state.observe_identity(&identity)?;
8073        }
8074        let reads = read_pool_meta_replicas(pools, true).await;
8075        if let Some(err) = pool_meta_replica_read_failure(&reads, "pool metadata load") {
8076            write_state.block_with_context(pool_metadata_error(
8077                crate::error::PoolMetadataFailure::RecoveryRequired,
8078                "startup_read",
8079                Some(Arc::new(err)),
8080            ));
8081        }
8082        let selection = select_pool_meta_replica_reads(reads)
8083            .map_err(|err| block_pool_meta_validation(write_state, err, "startup_selection"))?;
8084        write_state
8085            .observe_selection(&selection)
8086            .map_err(|err| block_pool_meta_validation(write_state, err, "startup_selection"))?;
8087        *self = selection.meta;
8088        Ok(selection.replica_state)
8089    }
8090
8091    #[cfg(test)]
8092    fn encode_config_data(&self) -> Result<Vec<u8>> {
8093        self.encode_config_data_for_v2_gate(pool_meta_v2_writer_enabled())
8094    }
8095
8096    fn encode_config_data_for_v2_gate(&self, v2_enabled: bool) -> Result<Vec<u8>> {
8097        if self.dont_save {
8098            return Ok(Vec::new());
8099        }
8100        if self.version == POOL_META_GENERATION_VERSION {
8101            return Err(Error::other(
8102                "pool metadata V3 save requires cluster identity and generation transaction context",
8103            ));
8104        }
8105        if !matches!(self.version, 0 | POOL_META_V1_VERSION | POOL_META_VERSION) {
8106            return Err(Error::other(format!(
8107                "pool metadata save failed: unexpected runtime version {}",
8108                self.version
8109            )));
8110        }
8111        let version = if self.version == POOL_META_VERSION || v2_enabled {
8112            POOL_META_VERSION
8113        } else {
8114            POOL_META_V1_VERSION
8115        };
8116        if version == POOL_META_V1_VERSION {
8117            let mut decommission_infos = self.pools.iter().filter_map(|pool| pool.decommission.as_ref());
8118            if decommission_infos.clone().any(|info| !info.unresolved_entries.is_empty()) {
8119                return Err(Error::other(format!(
8120                    "pool metadata V2 is required to persist unresolved decommission entries; enable both {} and {} only after every reader and writer supports V2",
8121                    rustfs_config::ENV_POOL_META_V2_WRITE,
8122                    rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
8123                )));
8124            }
8125            if decommission_infos.any(|info| info.capacity_reservation.is_some()) {
8126                return Err(Error::DecommissionCapacity(format!(
8127                    "pool metadata V2 is required to persist decommission capacity reservations; enable both {} and {} only after every reader and writer supports V2",
8128                    rustfs_config::ENV_POOL_META_V2_WRITE,
8129                    rustfs_config::ENV_POOL_META_V2_FLEET_CONFIRMED,
8130                )));
8131            }
8132        }
8133        let mut data = Vec::new();
8134        data.write_u16::<LittleEndian>(POOL_META_FORMAT)?;
8135        data.write_u16::<LittleEndian>(version)?;
8136        let mut buf = Vec::new();
8137        match version {
8138            POOL_META_V1_VERSION => PersistedPoolMetaV1::from(self).serialize(&mut Serializer::new(&mut buf))?,
8139            POOL_META_VERSION => PersistedPoolMeta::from(self).serialize(&mut Serializer::new(&mut buf))?,
8140            _ => unreachable!("pool metadata writer selected an unsupported version"),
8141        }
8142        data.write_all(&buf)?;
8143        Ok(data)
8144    }
8145
8146    #[cfg(test)]
8147    pub(crate) fn encode_config_data_for_test(&self) -> Result<Vec<u8>> {
8148        self.encode_config_data_for_v2_gate(true)
8149    }
8150
8151    pub async fn save(&self, pools: Vec<Arc<Sets>>) -> Result<()> {
8152        let pool = pools
8153            .first()
8154            .cloned()
8155            .ok_or_else(|| Error::other("pool metadata save failed: no storage pools available"))?;
8156        let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
8157        let pool_meta_guard = pool_meta_lock.get_write_lock(get_lock_acquire_timeout()).await?;
8158        let mut write_state = PoolMetaWriteState::default();
8159        let indices = (0..self.pools.len()).collect::<Vec<_>>();
8160        let outcome = self
8161            .save_no_lock_armed_scoped(pools, &mut write_state, pool_meta_guard.lock_lost_signal(), Some(&indices))
8162            .await?;
8163        outcome.disarm();
8164        Ok(())
8165    }
8166
8167    /// Startup has a single elected local writer, so it must not depend on namespace locks here.
8168    #[cfg(any(test, feature = "test-util"))]
8169    pub(crate) async fn save_for_startup<S>(&self, pools: Vec<Arc<S>>) -> Result<()>
8170    where
8171        S: EcstoreObjectIO,
8172    {
8173        let mut write_state = PoolMetaWriteState::for_test_bootstrap();
8174        self.save_for_startup_observing(pools, &mut write_state).await.map(|_| ())
8175    }
8176
8177    pub(crate) async fn save_for_startup_observing<S>(
8178        &self,
8179        pools: Vec<Arc<S>>,
8180        write_state: &mut PoolMetaWriteState,
8181    ) -> Result<PoolMeta>
8182    where
8183        S: EcstoreObjectIO,
8184    {
8185        let outcome = self.save_no_lock_armed_scoped(pools, write_state, None, None).await?;
8186        Ok(outcome.into_committed())
8187    }
8188
8189    async fn save_no_lock_with_fence<S>(
8190        &self,
8191        pools: Vec<Arc<S>>,
8192        lock_lost: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
8193        indices: &[usize],
8194    ) -> Result<()>
8195    where
8196        S: EcstoreObjectIO,
8197    {
8198        let mut write_state = PoolMetaWriteState::default();
8199        let outcome = self
8200            .save_no_lock_armed_scoped(pools, &mut write_state, lock_lost, Some(indices))
8201            .await?;
8202        outcome.disarm();
8203        Ok(())
8204    }
8205
8206    async fn save_no_lock_with_activation_fence<S>(
8207        &self,
8208        pools: Vec<Arc<S>>,
8209        write_state: &mut PoolMetaWriteState,
8210        activation_fence: &PoolRebalanceActivationFence,
8211        indices: &[usize],
8212    ) -> Result<PoolMetaSaveOutcome>
8213    where
8214        S: EcstoreObjectIO,
8215    {
8216        write_state.ensure_write_safe("pool metadata activation save failed")?;
8217        let mut transaction_arm = write_state.arm_transaction();
8218        let fence = PoolMetaPersistenceFence::Activation(activation_fence);
8219        let committed = self
8220            .save_no_lock_transaction(pools, write_state, &fence, Some(indices), &mut transaction_arm)
8221            .await
8222            .map_err(|err| transaction_arm.failure(err))?;
8223        if transaction_arm.phase.is_some() {
8224            transaction_arm.phase = Some("publication");
8225        }
8226        Ok(PoolMetaSaveOutcome {
8227            transaction_arm,
8228            committed,
8229        })
8230    }
8231
8232    async fn save_no_lock_armed<S>(
8233        &self,
8234        pools: Vec<Arc<S>>,
8235        write_state: &mut PoolMetaWriteState,
8236        lock_lost: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
8237        indices: &[usize],
8238    ) -> Result<PoolMetaSaveOutcome>
8239    where
8240        S: EcstoreObjectIO,
8241    {
8242        self.save_no_lock_armed_scoped(pools, write_state, lock_lost, Some(indices))
8243            .await
8244    }
8245
8246    async fn save_no_lock_armed_scoped<S>(
8247        &self,
8248        pools: Vec<Arc<S>>,
8249        write_state: &mut PoolMetaWriteState,
8250        lock_lost: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
8251        indices: Option<&[usize]>,
8252    ) -> Result<PoolMetaSaveOutcome>
8253    where
8254        S: EcstoreObjectIO,
8255    {
8256        write_state.ensure_write_safe("pool metadata save failed")?;
8257        // The arm does not block its own transaction. If this future or its
8258        // returned outcome is dropped before publication, Drop latches the
8259        // sticky recovery gate.
8260        let mut transaction_arm = write_state.arm_transaction();
8261        let fence = PoolMetaPersistenceFence::Distributed(lock_lost);
8262        let committed = self
8263            .save_no_lock_transaction(pools, write_state, &fence, indices, &mut transaction_arm)
8264            .await
8265            .map_err(|err| transaction_arm.failure(err))?;
8266        if transaction_arm.phase.is_some() {
8267            transaction_arm.phase = Some("publication");
8268        }
8269        Ok(PoolMetaSaveOutcome {
8270            transaction_arm,
8271            committed,
8272        })
8273    }
8274
8275    async fn save_no_lock_transaction<S>(
8276        &self,
8277        pools: Vec<Arc<S>>,
8278        write_state: &mut PoolMetaWriteState,
8279        fence: &PoolMetaPersistenceFence<'_>,
8280        indices: Option<&[usize]>,
8281        transaction_arm: &mut PoolMetaTransactionArm,
8282    ) -> Result<PoolMeta>
8283    where
8284        S: EcstoreObjectIO,
8285    {
8286        if pools.is_empty() {
8287            return Err(Error::other("pool metadata save failed: no storage pools available"));
8288        }
8289        if self.dont_save {
8290            return Ok(self.clone());
8291        }
8292        for attempt in 0..POOL_META_CAS_MAX_ATTEMPTS {
8293            match self
8294                .save_no_lock_transaction_once(pools.clone(), write_state, fence, indices, transaction_arm)
8295                .await
8296            {
8297                Ok(committed) => return Ok(committed),
8298                Err(Error::PreconditionFailed) if attempt + 1 < POOL_META_CAS_MAX_ATTEMPTS => continue,
8299                Err(err) => return Err(err),
8300            }
8301        }
8302        Err(Error::PreconditionFailed)
8303    }
8304
8305    async fn save_no_lock_transaction_once<S>(
8306        &self,
8307        pools: Vec<Arc<S>>,
8308        write_state: &mut PoolMetaWriteState,
8309        fence: &PoolMetaPersistenceFence<'_>,
8310        indices: Option<&[usize]>,
8311        transaction_arm: &mut PoolMetaTransactionArm,
8312    ) -> Result<PoolMeta>
8313    where
8314        S: EcstoreObjectIO,
8315    {
8316        fence.ensure_held()?;
8317        let selection = load_pool_meta_replicas_observing(pools.clone(), true, write_state).await?;
8318        write_state.observe_replicas(selection.replica_state);
8319        write_state.ensure_write_safe("pool metadata save failed")?;
8320        let mut bootstrap_generation_required = false;
8321        if let Some(cluster_id) = write_state.expected_cluster_id {
8322            let identity = load_pool_meta_identity_selection_observing(pools.clone(), write_state, cluster_id).await?;
8323            bootstrap_generation_required = selection.absent || write_state.identity_initialized == Some(false);
8324            if !identity.repair_write_safe {
8325                write_state.block_writes();
8326                return Err(Error::other(
8327                    "pool metadata recovery required: cluster identity has an unreadable replica",
8328                ));
8329            }
8330            if let Some(identity) = identity.identity
8331                && selection.revision.is_generation_protocol()
8332                && identity.epoch != selection.revision.epoch
8333            {
8334                write_state.block_writes();
8335                return Err(Error::other(format!(
8336                    "pool metadata recovery required: committed epoch {} does not match cluster identity epoch {}",
8337                    selection.revision.epoch, identity.epoch
8338                )));
8339            }
8340            if !selection.absent && write_state.identity_requires_repair() {
8341                let initialized = write_state.identity_initialized != Some(false) || selection.revision.is_generation_protocol();
8342                persist_pool_meta_identity(
8343                    pools.clone(),
8344                    write_state,
8345                    initialized,
8346                    PoolMetaIdentityWriteScope::All,
8347                    fence,
8348                    transaction_arm,
8349                )
8350                .await?;
8351            }
8352        }
8353        // Startup is the only path allowed to create an all-missing metadata
8354        // set. Runtime callers without an identity context must fail closed.
8355        write_state.ensure_missing_metadata_can_initialize()?;
8356
8357        let mut committed = if let Some(indices) = indices {
8358            if selection.meta.pools.is_empty() {
8359                self.clone()
8360            } else {
8361                let mut requested = self.clone();
8362                requested.version = selection.meta.version;
8363                let mut latest = selection.meta.clone();
8364                merge_pool_meta_updates_for_save(&mut latest, &requested, indices, "pool metadata save failed")?;
8365                latest
8366            }
8367        } else {
8368            self.clone()
8369        };
8370        let target_version = if selection.generation_protocol_observed
8371            || selection.revision.is_generation_protocol()
8372            || pool_meta_v3_writer_enabled()
8373            || bootstrap_generation_required
8374        {
8375            POOL_META_GENERATION_VERSION
8376        } else if selection.meta.version == POOL_META_VERSION
8377            || self.version == POOL_META_VERSION
8378            || pool_meta_v2_writer_enabled()
8379        {
8380            POOL_META_VERSION
8381        } else {
8382            POOL_META_V1_VERSION
8383        };
8384        committed.version = target_version;
8385
8386        if target_version != POOL_META_GENERATION_VERSION {
8387            let data = committed.encode_config_data_for_v2_gate(target_version == POOL_META_VERSION)?;
8388            let mut canonical_saved = false;
8389            for (pool_index, (pool, token)) in pools.iter().cloned().zip(&selection.cas_tokens).enumerate() {
8390                let result = save_pool_meta_object_cas(
8391                    pool.clone(),
8392                    POOL_META_NAME,
8393                    data.clone(),
8394                    token,
8395                    fence,
8396                    "legacy_cas",
8397                    transaction_arm,
8398                )
8399                .await;
8400                if result.is_ok() && pool_index == 0 {
8401                    canonical_saved = true;
8402                    #[cfg(test)]
8403                    if let PoolMetaPersistenceFence::Activation(activation_fence) = fence {
8404                        pause_pool_activation_after_durable_save(&pool, activation_fence).await;
8405                    }
8406                }
8407                if let Err(err) = result {
8408                    if canonical_saved && fence.is_activation() {
8409                        warn!(
8410                            event = EVENT_DECOMMISSION_STATE,
8411                            component = LOG_COMPONENT_ECSTORE,
8412                            subsystem = LOG_SUBSYSTEM_POOLS,
8413                            pool_index,
8414                            state = "activation_replica_repair_pending",
8415                            error = %err,
8416                            "Decommission activation replica repair pending"
8417                        );
8418                        continue;
8419                    }
8420                    return Err(err);
8421                }
8422            }
8423            let confirmed = if fence.is_activation() {
8424                load_pool_meta_replicas(pools, true).await?
8425            } else {
8426                let confirmed = load_pool_meta_replicas_observing(pools, true, write_state).await?;
8427                write_state.observe_replicas(confirmed.replica_state);
8428                confirmed
8429            };
8430            let expected = committed.encode_config_data_for_v2_gate(true)?;
8431            if confirmed.canonical.as_ref() != Some(&expected) {
8432                record_pool_meta_stale_write_rejection("legacy_verify");
8433                return Err(Error::PreconditionFailed);
8434            }
8435            return Ok(confirmed.meta);
8436        }
8437
8438        let cluster_id = selection
8439            .revision
8440            .cluster_id
8441            .or(selection.generation_identity.map(|(cluster_id, _)| cluster_id))
8442            .or(write_state.expected_cluster_id)
8443            .ok_or_else(|| Error::other("pool metadata V3 save failed: cluster identity is not initialized"))?;
8444        let epoch = if selection.revision.is_generation_protocol() {
8445            selection.revision.epoch
8446        } else {
8447            selection
8448                .generation_identity
8449                .map(|(_, epoch)| epoch)
8450                .or(write_state.cluster_epoch)
8451                .unwrap_or(POOL_META_INITIAL_EPOCH)
8452        };
8453        let generation = if selection.revision.is_generation_protocol() {
8454            selection
8455                .revision
8456                .generation
8457                .checked_add(1)
8458                .ok_or_else(|| Error::other("pool metadata V3 generation exhausted"))?
8459        } else {
8460            1
8461        };
8462        let revision = PoolMetaRevision {
8463            version: POOL_META_GENERATION_VERSION,
8464            cluster_id: Some(cluster_id),
8465            epoch,
8466            generation,
8467            transaction_id: Some(uuid::Uuid::new_v4()),
8468        };
8469        let previous = if let Some(canonical) = selection.canonical.clone() {
8470            PoolMetaCommittedCandidate {
8471                canonical,
8472                meta: selection.meta.clone(),
8473                revision: selection.revision,
8474            }
8475        } else {
8476            let empty = PoolMeta {
8477                version: POOL_META_V1_VERSION,
8478                ..Default::default()
8479            };
8480            PoolMetaCommittedCandidate {
8481                canonical: empty.encode_config_data_for_v2_gate(true)?,
8482                meta: empty,
8483                revision: PoolMetaRevision::legacy(POOL_META_V1_VERSION),
8484            }
8485        };
8486        let pending = encode_pool_meta_v3_envelope(&committed, revision, false, Some(&previous))?;
8487        let durable = encode_pool_meta_v3_envelope(&committed, revision, true, None)?;
8488        let concurrent_phases = pools.len() > 1
8489            && selection.revision.is_generation_protocol()
8490            && !selection.replica_state.needs_repair
8491            && !bootstrap_generation_required
8492            && matches!(fence, PoolMetaPersistenceFence::Distributed(Some(_)));
8493        #[cfg(feature = "e2e-test-hooks")]
8494        let phase_barrier = if concurrent_phases && pools.len() == 4 {
8495            PoolMetaPhaseBarrier::bind(&previous, &committed, revision, &durable).await?
8496        } else {
8497            None
8498        };
8499        let mut pending_tokens = Vec::with_capacity(pools.len());
8500        if concurrent_phases {
8501            // Drain the entire prepare phase before validating ETags or returning an error.
8502            for result in save_pool_meta_phase(
8503                pools.as_slice(),
8504                &pending,
8505                &selection.cas_tokens,
8506                fence,
8507                "prepare_cas",
8508                transaction_arm,
8509                #[cfg(feature = "e2e-test-hooks")]
8510                phase_barrier.as_ref(),
8511            )
8512            .await
8513            {
8514                let etag = result?
8515                    .etag
8516                    .filter(|etag| !etag.trim().is_empty())
8517                    .ok_or_else(|| Error::other("pool metadata V3 prepare succeeded without a conditional-write revision"))?;
8518                pending_tokens.push(PoolMetaCasToken::Existing(etag));
8519            }
8520        } else {
8521            for (pool, token) in pools.iter().cloned().zip(&selection.cas_tokens) {
8522                let object_info = save_pool_meta_object_cas(
8523                    pool,
8524                    POOL_META_NAME,
8525                    pending.clone(),
8526                    token,
8527                    fence,
8528                    "prepare_cas",
8529                    transaction_arm,
8530                )
8531                .await?;
8532                let etag = object_info
8533                    .etag
8534                    .filter(|etag| !etag.trim().is_empty())
8535                    .ok_or_else(|| Error::other("pool metadata V3 prepare succeeded without a conditional-write revision"))?;
8536                pending_tokens.push(PoolMetaCasToken::Existing(etag));
8537            }
8538        }
8539
8540        #[cfg(feature = "e2e-test-hooks")]
8541        if let Some(barrier) = &phase_barrier
8542            && barrier.arm.case == PoolMetaPhaseBarrierCase::PreparedAll
8543        {
8544            barrier.ready().await?;
8545        }
8546        let mut commit_error = None;
8547        let mut commit_succeeded = false;
8548        if concurrent_phases {
8549            for result in save_pool_meta_phase(
8550                pools.as_slice(),
8551                &durable,
8552                &pending_tokens,
8553                fence,
8554                "commit_cas",
8555                transaction_arm,
8556                #[cfg(feature = "e2e-test-hooks")]
8557                phase_barrier.as_ref(),
8558            )
8559            .await
8560            {
8561                match result {
8562                    Ok(_) => commit_succeeded = true,
8563                    Err(err) => {
8564                        commit_error.get_or_insert(err);
8565                    }
8566                }
8567            }
8568        } else {
8569            #[cfg(test)]
8570            let mut first_pool = true;
8571            for (pool, token) in pools.iter().cloned().zip(&pending_tokens) {
8572                match save_pool_meta_object_cas(
8573                    pool.clone(),
8574                    POOL_META_NAME,
8575                    durable.clone(),
8576                    token,
8577                    fence,
8578                    "commit_cas",
8579                    transaction_arm,
8580                )
8581                .await
8582                {
8583                    Ok(_) => {
8584                        commit_succeeded = true;
8585                        #[cfg(test)]
8586                        if first_pool && let PoolMetaPersistenceFence::Activation(activation_fence) = fence {
8587                            pause_pool_activation_after_durable_save(&pool, activation_fence).await;
8588                        }
8589                    }
8590                    Err(err) => {
8591                        commit_error.get_or_insert(err);
8592                    }
8593                }
8594                #[cfg(test)]
8595                {
8596                    first_pool = false;
8597                }
8598            }
8599        }
8600        let confirmed = if fence.is_activation() {
8601            load_pool_meta_replicas(pools.clone(), true).await?
8602        } else {
8603            let confirmed = load_pool_meta_replicas_observing(pools.clone(), true, write_state).await?;
8604            write_state.observe_replicas(confirmed.replica_state);
8605            confirmed
8606        };
8607        if confirmed.revision == revision && confirmed.canonical.as_ref() == Some(&durable) {
8608            persist_pool_meta_identity(pools, write_state, true, PoolMetaIdentityWriteScope::All, fence, transaction_arm).await?;
8609            #[cfg(feature = "e2e-test-hooks")]
8610            startup_cas_test_observe(serde_json::json!({
8611                "kind": "confirmed", "object": POOL_META_NAME,
8612                "payload_sha256": rustfs_utils::crypto::hex(Sha256::digest(&durable)),
8613                "generation": confirmed.revision.generation,
8614                "transaction_id": confirmed.revision.transaction_id,
8615            }));
8616            #[cfg(feature = "e2e-test-hooks")]
8617            if let Some(barrier) = &phase_barrier
8618                && barrier.arm.case == PoolMetaPhaseBarrierCase::BeforePublish
8619            {
8620                barrier.ready().await?;
8621            }
8622            return Ok(confirmed.meta);
8623        }
8624        if !commit_succeeded {
8625            return Err(commit_error.unwrap_or(Error::PreconditionFailed));
8626        }
8627        Err(commit_error.unwrap_or_else(|| {
8628            Error::other("pool metadata recovery required: committed V3 transaction was not selected after write")
8629        }))
8630    }
8631
8632    #[cfg(test)]
8633    async fn save_no_lock_observing<S>(&self, pools: Vec<Arc<S>>, write_state: &mut PoolMetaWriteState) -> Result<()>
8634    where
8635        S: EcstoreObjectIO,
8636    {
8637        let indices = (0..self.pools.len()).collect::<Vec<_>>();
8638        let outcome = self.save_no_lock_armed(pools, write_state, None, &indices).await?;
8639        outcome.disarm();
8640        Ok(())
8641    }
8642
8643    pub fn decommission_cancel(&mut self, idx: usize) -> bool {
8644        self.decommission_cancel_at(idx, OffsetDateTime::now_utc(), None)
8645    }
8646
8647    #[cfg(test)]
8648    pub(crate) fn decommission_cancel_at_for_test(
8649        &mut self,
8650        idx: usize,
8651        now: OffsetDateTime,
8652        rebalance_meta: Option<&RebalanceMeta>,
8653    ) -> bool {
8654        self.decommission_cancel_at(idx, now, rebalance_meta)
8655    }
8656
8657    fn decommission_cancel_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool {
8658        let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else {
8659            return false;
8660        };
8661        if d.canceled {
8662            return false;
8663        }
8664
8665        let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8666        let mut pd = d.clone();
8667        pd.canceled = true;
8668        pd.failed = false;
8669        pd.complete = false;
8670        pd.start_time = None;
8671        pd.terminal_reload_attempt_at = None;
8672        pd.terminal_reload_failures.clear();
8673        pd.capacity_blocked_reason = None;
8674        release_decommission_capacity_reservation(&mut pd, DECOMMISSION_CAPACITY_RELEASE_CANCELED, last_update);
8675
8676        let Some(stats) = self.pools.get_mut(idx) else {
8677            return false;
8678        };
8679        stats.last_update = last_update;
8680        stats.decommission = Some(pd);
8681        true
8682    }
8683    pub fn decommission_failed(&mut self, idx: usize) -> bool {
8684        self.decommission_failed_at(idx, OffsetDateTime::now_utc(), None)
8685    }
8686
8687    #[cfg(test)]
8688    pub(crate) fn decommission_failed_at_for_test(
8689        &mut self,
8690        idx: usize,
8691        now: OffsetDateTime,
8692        rebalance_meta: Option<&RebalanceMeta>,
8693    ) -> bool {
8694        self.decommission_failed_at(idx, now, rebalance_meta)
8695    }
8696
8697    fn decommission_failed_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool {
8698        let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else {
8699            return false;
8700        };
8701        if !is_decommission_active(d.complete, d.failed, d.canceled) {
8702            return false;
8703        }
8704
8705        let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8706        let mut pd = d.clone();
8707        pd.canceled = false;
8708        pd.failed = true;
8709        pd.complete = false;
8710        pd.start_time = None;
8711        pd.terminal_reload_attempt_at = None;
8712        pd.terminal_reload_failures.clear();
8713        pd.capacity_blocked_reason = None;
8714        release_decommission_capacity_reservation(&mut pd, DECOMMISSION_CAPACITY_RELEASE_FAILED, last_update);
8715
8716        let Some(stats) = self.pools.get_mut(idx) else {
8717            return false;
8718        };
8719        stats.last_update = last_update;
8720        stats.decommission = Some(pd);
8721        true
8722    }
8723
8724    pub fn clear_decommission(&mut self, idx: usize) -> Result<bool> {
8725        self.clear_decommission_at(idx, OffsetDateTime::now_utc(), None)
8726    }
8727
8728    #[cfg(test)]
8729    pub(crate) fn clear_decommission_at_for_test(
8730        &mut self,
8731        idx: usize,
8732        now: OffsetDateTime,
8733        rebalance_meta: Option<&RebalanceMeta>,
8734    ) -> Result<bool> {
8735        self.clear_decommission_at(idx, now, rebalance_meta)
8736    }
8737
8738    fn clear_decommission_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> Result<bool> {
8739        let pool_count = self.pools.len();
8740        ensure_valid_decommission_pool_index(pool_count, idx)?;
8741
8742        let Some(pool) = self.pools.get(idx) else {
8743            return Err(invalid_decommission_pool_index_error(pool_count, idx));
8744        };
8745
8746        let (decommission_present, complete, failed, canceled, unresolved_entries) = pool
8747            .decommission
8748            .as_ref()
8749            .map(|info| {
8750                (
8751                    info.has_decommission_state(),
8752                    info.complete,
8753                    info.failed,
8754                    info.canceled,
8755                    info.unresolved_entries.len(),
8756                )
8757            })
8758            .unwrap_or((false, false, false, false, 0));
8759
8760        ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled, unresolved_entries)?;
8761
8762        let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8763        let Some(pool) = self.pools.get_mut(idx) else {
8764            return Err(invalid_decommission_pool_index_error(pool_count, idx));
8765        };
8766        pool.last_update = last_update;
8767        // Preserve a state-empty tombstone so scanner catch-up can recover the
8768        // durable movement generation after a clear followed by a restart.
8769        pool.decommission = Some(PoolDecommissionInfo::default());
8770        Ok(true)
8771    }
8772
8773    pub fn decommission_complete(&mut self, idx: usize) -> bool {
8774        self.decommission_complete_at(idx, OffsetDateTime::now_utc(), None)
8775    }
8776
8777    #[cfg(test)]
8778    pub(crate) fn decommission_complete_at_for_test(
8779        &mut self,
8780        idx: usize,
8781        now: OffsetDateTime,
8782        rebalance_meta: Option<&RebalanceMeta>,
8783    ) -> bool {
8784        self.decommission_complete_at(idx, now, rebalance_meta)
8785    }
8786
8787    fn decommission_complete_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool {
8788        let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else {
8789            return false;
8790        };
8791        if !is_decommission_active(d.complete, d.failed, d.canceled) {
8792            return false;
8793        }
8794
8795        let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8796        let mut pd = d.clone();
8797        pd.canceled = false;
8798        pd.failed = false;
8799        pd.complete = true;
8800        pd.terminal_reload_attempt_at = None;
8801        pd.terminal_reload_failures.clear();
8802        pd.capacity_blocked_reason = None;
8803        release_decommission_capacity_reservation(&mut pd, DECOMMISSION_CAPACITY_RELEASE_COMPLETED, last_update);
8804
8805        let Some(stats) = self.pools.get_mut(idx) else {
8806            return false;
8807        };
8808        stats.last_update = last_update;
8809        stats.decommission = Some(pd);
8810        true
8811    }
8812    fn set_decommission_state_at(
8813        &mut self,
8814        idx: usize,
8815        pi: PoolSpaceInfo,
8816        queued: bool,
8817        now: OffsetDateTime,
8818        rebalance_meta: Option<&RebalanceMeta>,
8819    ) -> Result<()> {
8820        let pool_count = self.pools.len();
8821        ensure_valid_decommission_pool_index(pool_count, idx)?;
8822
8823        let Some(pool) = self.pools.get(idx) else {
8824            return Err(invalid_decommission_pool_index_error(pool_count, idx));
8825        };
8826
8827        ensure_decommission_start_allowed(decommission_start_pool_state(Some(pool)))?;
8828
8829        let generation = self.next_scanner_data_movement_update(now, rebalance_meta);
8830        let Some(pool) = self.pools.get_mut(idx) else {
8831            return Err(invalid_decommission_pool_index_error(pool_count, idx));
8832        };
8833        let previous = pool.decommission.as_ref();
8834        pool.last_update = generation;
8835        pool.decommission = Some(build_decommission_start_state(pi, queued, generation, previous));
8836
8837        Ok(())
8838    }
8839
8840    fn set_decommission_state(&mut self, idx: usize, pi: PoolSpaceInfo, queued: bool) -> Result<()> {
8841        self.set_decommission_state_at(idx, pi, queued, OffsetDateTime::now_utc(), None)
8842    }
8843
8844    #[cfg(test)]
8845    pub(crate) fn decommission_at_for_test(
8846        &mut self,
8847        idx: usize,
8848        pi: PoolSpaceInfo,
8849        now: OffsetDateTime,
8850        rebalance_meta: Option<&RebalanceMeta>,
8851    ) -> Result<()> {
8852        self.set_decommission_state_at(idx, pi, false, now, rebalance_meta)
8853    }
8854
8855    pub fn decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> {
8856        self.set_decommission_state(idx, pi, false)
8857    }
8858
8859    pub fn queue_decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> {
8860        self.set_decommission_state(idx, pi, true)
8861    }
8862
8863    pub fn record_decommission_terminal_reload_failure(&mut self, idx: usize, stage: &str, message: String) -> Result<bool> {
8864        self.record_decommission_terminal_reload_failure_at(idx, stage, message, OffsetDateTime::now_utc(), None)
8865    }
8866
8867    #[cfg(test)]
8868    pub(crate) fn record_decommission_terminal_reload_failure_at_for_test(
8869        &mut self,
8870        idx: usize,
8871        stage: &str,
8872        message: String,
8873        now: OffsetDateTime,
8874        rebalance_meta: Option<&RebalanceMeta>,
8875    ) -> Result<bool> {
8876        self.record_decommission_terminal_reload_failure_at(idx, stage, message, now, rebalance_meta)
8877    }
8878
8879    fn record_decommission_terminal_reload_failure_at(
8880        &mut self,
8881        idx: usize,
8882        stage: &str,
8883        message: String,
8884        now: OffsetDateTime,
8885        rebalance_meta: Option<&RebalanceMeta>,
8886    ) -> Result<bool> {
8887        let pool_count = self.pools.len();
8888        ensure_valid_decommission_pool_index(pool_count, idx)?;
8889
8890        let Some(pool) = self.pools.get(idx) else {
8891            return Err(invalid_decommission_pool_index_error(pool_count, idx));
8892        };
8893        let Some(info) = pool.decommission.as_ref() else {
8894            return Err(decommission_metadata_not_initialized_error("record decommission terminal reload failure"));
8895        };
8896
8897        let failure = format!("{stage}: {message}");
8898        if info.terminal_reload_failures.last().is_some_and(|last| last == &failure) {
8899            return Ok(false);
8900        }
8901
8902        let last_update = self.next_scanner_data_movement_update(now, rebalance_meta);
8903        let Some(pool) = self.pools.get_mut(idx) else {
8904            return Err(invalid_decommission_pool_index_error(pool_count, idx));
8905        };
8906        let Some(info) = pool.decommission.as_mut() else {
8907            return Err(decommission_metadata_not_initialized_error("record decommission terminal reload failure"));
8908        };
8909        pool.last_update = last_update;
8910        info.terminal_reload_attempt_at = Some(last_update);
8911        info.terminal_reload_failures.push(failure);
8912        Ok(true)
8913    }
8914
8915    fn mark_decommission_capacity_blocked(&mut self, idx: usize, reason: String, now: OffsetDateTime) -> Result<bool> {
8916        let pool_count = self.pools.len();
8917        let pool = self
8918            .pools
8919            .get_mut(idx)
8920            .ok_or_else(|| invalid_decommission_pool_index_error(pool_count, idx))?;
8921        let info = pool
8922            .decommission
8923            .as_mut()
8924            .ok_or_else(|| decommission_metadata_not_initialized_error("pause decommission for target capacity"))?;
8925        if !is_decommission_active(info.complete, info.failed, info.canceled) {
8926            return Ok(false);
8927        }
8928        if let Some(reservation) = info.capacity_reservation.as_mut().filter(|reservation| reservation.active()) {
8929            renew_decommission_capacity_reservation(reservation, now, true);
8930        }
8931        let changed = info.capacity_blocked_reason.as_deref() != Some(reason.as_str());
8932        info.capacity_blocked_reason = Some(reason);
8933        pool.last_update = now;
8934        Ok(changed)
8935    }
8936
8937    pub fn promote_queued_decommission(&mut self, idx: usize) -> bool {
8938        self.promote_queued_decommission_at(idx, OffsetDateTime::now_utc(), None)
8939    }
8940
8941    #[cfg(test)]
8942    pub(crate) fn promote_queued_decommission_at_for_test(
8943        &mut self,
8944        idx: usize,
8945        now: OffsetDateTime,
8946        rebalance_meta: Option<&RebalanceMeta>,
8947    ) -> bool {
8948        self.promote_queued_decommission_at(idx, now, rebalance_meta)
8949    }
8950
8951    fn promote_queued_decommission_at(
8952        &mut self,
8953        idx: usize,
8954        now: OffsetDateTime,
8955        rebalance_meta: Option<&RebalanceMeta>,
8956    ) -> bool {
8957        let Some(info) = self.pools.get(idx).and_then(|pool| pool.decommission.as_ref()) else {
8958            return false;
8959        };
8960        if !info.queued || !is_decommission_active(info.complete, info.failed, info.canceled) {
8961            return false;
8962        }
8963
8964        let generation = self.next_scanner_data_movement_update(now, rebalance_meta);
8965        let Some(pool) = self.pools.get_mut(idx) else {
8966            return false;
8967        };
8968        let Some(info) = pool.decommission.as_mut() else {
8969            return false;
8970        };
8971        pool.last_update = generation;
8972        info.queued = false;
8973        info.start_time = Some(generation);
8974        for entry in &mut info.unresolved_entries {
8975            entry.source_generation = generation;
8976        }
8977        true
8978    }
8979    pub fn queue_buckets(&mut self, idx: usize, bks: Vec<DecomBucketInfo>) {
8980        if let Some(pool) = self.pools.get_mut(idx)
8981            && let Some(dec) = pool.decommission.as_mut()
8982        {
8983            for bk in bks.iter() {
8984                dec.bucket_push(bk);
8985            }
8986        }
8987    }
8988    pub fn pending_buckets(&self, idx: usize) -> Vec<DecomBucketInfo> {
8989        let mut list = Vec::new();
8990
8991        if let Some(pool) = self.pools.get(idx)
8992            && let Some(ref info) = pool.decommission
8993        {
8994            for bk in info.queued_buckets.iter() {
8995                let (name, prefix) = path2_bucket_object(bk);
8996                list.push(DecomBucketInfo { name, prefix });
8997            }
8998        }
8999
9000        list
9001    }
9002
9003    pub fn is_bucket_decommissioned(&self, idx: usize, bucket: String) -> bool {
9004        self.pools
9005            .get(idx)
9006            .and_then(|pool| pool.decommission.as_ref())
9007            .is_some_and(|info| info.is_bucket_decommissioned(&bucket))
9008    }
9009
9010    pub fn bucket_done(&mut self, idx: usize, bucket: String) -> bool {
9011        if let Some(pool) = self.pools.get_mut(idx) {
9012            if let Some(info) = pool.decommission.as_mut() {
9013                info.bucket_pop(&bucket)
9014            } else {
9015                false
9016            }
9017        } else {
9018            false
9019        }
9020    }
9021
9022    pub fn count_item(&mut self, idx: usize, size: usize, failed: bool) {
9023        if let Some(pool) = self.pools.get_mut(idx)
9024            && let Some(info) = pool.decommission.as_mut()
9025        {
9026            if failed {
9027                info.items_decommission_failed += 1;
9028                info.bytes_failed += size;
9029            } else {
9030                info.items_decommissioned += 1;
9031                info.bytes_done += size;
9032            }
9033        }
9034    }
9035
9036    pub fn track_current_bucket_object(&mut self, idx: usize, bucket: String, object: String) {
9037        self.track_current_bucket_object_stage(idx, bucket, object, String::new());
9038    }
9039
9040    pub fn track_current_bucket_object_stage(&mut self, idx: usize, bucket: String, object: String, stage: String) {
9041        if self.pools.get(idx).is_none_or(|v| v.decommission.is_none()) {
9042            return;
9043        }
9044
9045        if let Some(pool) = self.pools.get_mut(idx)
9046            && let Some(info) = pool.decommission.as_mut()
9047        {
9048            info.object = object;
9049            info.bucket = bucket;
9050            info.stage = stage;
9051        }
9052    }
9053
9054    pub fn update_after(&mut self, idx: usize, duration: Duration) -> Result<bool> {
9055        Ok(self
9056            .decommission_progress_checkpoint(idx, duration, OffsetDateTime::now_utc(), None)?
9057            .is_some())
9058    }
9059
9060    pub fn validate(&self, pools: Vec<Arc<Sets>>) -> Result<bool> {
9061        struct PoolInfo {
9062            position: usize,
9063            completed: bool,
9064            #[allow(dead_code, reason = "written but never read back (backlog#1823)")]
9065            decom_started: bool,
9066        }
9067
9068        let mut remembered_pools = HashMap::new();
9069        for (idx, pool) in self.pools.iter().enumerate() {
9070            let mut complete = false;
9071            let mut decom_started = false;
9072            if let Some(decommission) = &pool.decommission {
9073                if decommission.complete {
9074                    complete = true;
9075                }
9076                decom_started = true;
9077            }
9078            remembered_pools.insert(
9079                pool.cmd_line.clone(),
9080                PoolInfo {
9081                    position: idx,
9082                    completed: complete,
9083                    decom_started,
9084                },
9085            );
9086        }
9087
9088        let mut specified_pools = HashMap::new();
9089        for (idx, pool) in pools.iter().enumerate() {
9090            specified_pools.insert(pool.endpoints.cmd_line.clone(), idx);
9091        }
9092
9093        let mut update = false;
9094
9095        // Determine whether the selected pool should be removed from the retired list.
9096        for k in specified_pools.keys() {
9097            if let Some(pi) = remembered_pools.get(k) {
9098                ensure_pool_not_left_in_cmdline_after_decommission(pi.position, k, pi.completed)?;
9099            } else {
9100                // If the previous pool no longer exists, allow updates because a new pool may have been added.
9101                update = true;
9102            }
9103        }
9104
9105        if specified_pools.len() == remembered_pools.len() {
9106            for (k, pi) in remembered_pools.iter() {
9107                if let Some(pos) = specified_pools.get(k)
9108                    && *pos != pi.position
9109                {
9110                    update = true; // Pool order changed, allow the update.
9111                }
9112            }
9113        }
9114
9115        if !update {
9116            update = specified_pools.len() != remembered_pools.len();
9117        }
9118
9119        Ok(update)
9120    }
9121
9122    pub fn return_resumable_pools(&self) -> Vec<PoolStatus> {
9123        resumable_decommission_queue_indices(self)
9124            .into_iter()
9125            .map(|idx| self.pools[idx].clone())
9126            .collect()
9127    }
9128}
9129
9130pub fn path2_bucket_object(name: &str) -> (String, String) {
9131    path_to_bucket_object(name)
9132}
9133
9134pub fn path2_bucket_object_with_base_path(base_path: &str, path: &str) -> (String, String) {
9135    path_to_bucket_object_with_base_path(base_path, path)
9136}
9137
9138#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
9139#[serde(deny_unknown_fields)]
9140pub struct DecommissionUnresolvedEntry {
9141    pub bucket: String,
9142    pub object: String,
9143    #[serde(rename = "poolIndex")]
9144    pub pool_index: usize,
9145    #[serde(rename = "setIndex")]
9146    pub set_index: usize,
9147    #[serde(rename = "sourceGeneration", with = "time::serde::rfc3339")]
9148    pub source_generation: OffsetDateTime,
9149    #[serde(rename = "candidateCount")]
9150    pub candidate_count: usize,
9151    #[serde(rename = "diskErrorCount")]
9152    pub disk_error_count: usize,
9153    #[serde(rename = "observedAt", with = "time::serde::rfc3339")]
9154    pub observed_at: OffsetDateTime,
9155    pub reason: String,
9156}
9157
9158#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)]
9159#[serde(rename_all = "camelCase", deny_unknown_fields)]
9160pub struct DecommissionErasureLayout {
9161    pub data: usize,
9162    pub parity: usize,
9163}
9164
9165impl DecommissionErasureLayout {
9166    fn width(self) -> usize {
9167        self.data.saturating_add(self.parity)
9168    }
9169
9170    fn is_valid(self) -> bool {
9171        self.data > 0 && self.data.checked_add(self.parity).is_some()
9172    }
9173}
9174
9175#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
9176#[serde(rename_all = "camelCase", deny_unknown_fields)]
9177pub struct DecommissionCapacityTarget {
9178    pub pool_index: usize,
9179    pub layout: DecommissionErasureLayout,
9180    pub physical_total_at_reservation: usize,
9181    pub physical_free_at_reservation: usize,
9182    pub reserved_physical_bytes: usize,
9183    #[serde(default)]
9184    pub consumed_physical_bytes: usize,
9185    #[serde(default)]
9186    pub observed_physical_bytes: usize,
9187    #[serde(default)]
9188    pub inflight_physical_bytes: usize,
9189    #[serde(default)]
9190    pub pending_physical_bytes: usize,
9191    #[serde(default)]
9192    pub pending_mutation_id: Option<uuid::Uuid>,
9193    #[serde(default, skip_serializing_if = "Vec::is_empty")]
9194    pub temporary_mutations: Vec<DecommissionCapacityTemporaryMutation>,
9195}
9196
9197#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
9198#[serde(rename_all = "camelCase", deny_unknown_fields)]
9199pub struct DecommissionCapacityTemporaryMutation {
9200    pub mutation_id: uuid::Uuid,
9201    pub physical_bytes: usize,
9202}
9203
9204#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
9205#[serde(rename_all = "camelCase", deny_unknown_fields)]
9206pub struct DecommissionCapacityReservation {
9207    pub model_version: u16,
9208    pub operation_id: uuid::Uuid,
9209    pub generation: u64,
9210    pub owner_nonce: uuid::Uuid,
9211    pub source_pool_index: usize,
9212    pub source_layout: DecommissionErasureLayout,
9213    pub source_physical_total_bytes: usize,
9214    pub source_physical_bytes: usize,
9215    pub source_data_equivalent_bytes: usize,
9216    pub predicted_physical_bytes: usize,
9217    pub temporary_copies: usize,
9218    pub temporary_physical_bytes: usize,
9219    pub peak_physical_bytes: usize,
9220    pub committed_data_bytes: usize,
9221    #[serde(default)]
9222    pub consumed_target_physical_bytes: usize,
9223    #[serde(default)]
9224    pub observed_target_physical_bytes: usize,
9225    #[serde(default)]
9226    pub inflight_target_physical_bytes: usize,
9227    #[serde(default)]
9228    pub pending_target_physical_bytes: usize,
9229    pub prediction_error_bytes: i64,
9230    pub targets: Vec<DecommissionCapacityTarget>,
9231    #[serde(with = "time::serde::rfc3339")]
9232    pub created_at: OffsetDateTime,
9233    #[serde(with = "time::serde::rfc3339")]
9234    pub renewed_at: OffsetDateTime,
9235    #[serde(with = "time::serde::rfc3339")]
9236    pub expires_at: OffsetDateTime,
9237    #[serde(with = "time::serde::rfc3339::option", default)]
9238    pub recovered_at: Option<OffsetDateTime>,
9239    #[serde(with = "time::serde::rfc3339::option", default)]
9240    pub released_at: Option<OffsetDateTime>,
9241    #[serde(default, skip_serializing_if = "Option::is_none")]
9242    pub release_reason: Option<String>,
9243}
9244
9245#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9246enum DecommissionCapacityMutationMode {
9247    Durable,
9248    Temporary,
9249    NonGrowingReplacement,
9250    TemporaryRelease,
9251}
9252
9253#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9254struct DecommissionCapacityReleaseProof {
9255    confirmed_absent: bool,
9256    clear_pending: bool,
9257}
9258
9259impl DecommissionCapacityReleaseProof {
9260    const fn confirmed_absence(clear_pending: bool) -> Self {
9261        Self {
9262            confirmed_absent: true,
9263            clear_pending,
9264        }
9265    }
9266}
9267
9268#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9269pub(crate) struct DecommissionCapacityOwner {
9270    pub(crate) source_pool_index: usize,
9271    pub(crate) operation_id: uuid::Uuid,
9272    pub(crate) generation: u64,
9273    pub(crate) owner_nonce: uuid::Uuid,
9274    pub(crate) mutation_id: Option<uuid::Uuid>,
9275}
9276
9277#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
9278struct DecommissionCapacityTargetPermitKey {
9279    store_id: uuid::Uuid,
9280    target_pool_index: usize,
9281    source_pool_index: usize,
9282    operation_id: uuid::Uuid,
9283    generation: u64,
9284    owner_nonce: uuid::Uuid,
9285    mutation_id: uuid::Uuid,
9286}
9287
9288impl DecommissionCapacityTargetPermitKey {
9289    fn new(store_id: uuid::Uuid, target_pool_index: usize, owner: DecommissionCapacityOwner) -> Option<Self> {
9290        Some(Self {
9291            store_id,
9292            target_pool_index,
9293            source_pool_index: owner.source_pool_index,
9294            operation_id: owner.operation_id,
9295            generation: owner.generation,
9296            owner_nonce: owner.owner_nonce,
9297            mutation_id: owner.mutation_id?,
9298        })
9299    }
9300
9301    fn owns_same_mutation(&self, store_id: uuid::Uuid, owner: DecommissionCapacityOwner) -> bool {
9302        owner.mutation_id.is_some_and(|mutation_id| {
9303            self.store_id == store_id
9304                && self.source_pool_index == owner.source_pool_index
9305                && self.operation_id == owner.operation_id
9306                && self.generation == owner.generation
9307                && self.owner_nonce == owner.owner_nonce
9308                && self.mutation_id == mutation_id
9309        })
9310    }
9311}
9312
9313struct DecommissionCapacityTargetPermit {
9314    key: DecommissionCapacityTargetPermitKey,
9315}
9316
9317// Busy recovery crosses option rebuilding and a spawned migration task. Keep
9318// the distributed guard in one exact-mutation side table so target selection
9319// can pin that target and the formal capacity mutation can consume the guard.
9320static DECOMMISSION_CAPACITY_TARGET_PERMITS: std::sync::OnceLock<
9321    std::sync::Mutex<HashMap<DecommissionCapacityTargetPermitKey, rustfs_lock::NamespaceLockGuard>>,
9322> = std::sync::OnceLock::new();
9323
9324fn decommission_capacity_target_permits()
9325-> &'static std::sync::Mutex<HashMap<DecommissionCapacityTargetPermitKey, rustfs_lock::NamespaceLockGuard>> {
9326    DECOMMISSION_CAPACITY_TARGET_PERMITS.get_or_init(|| std::sync::Mutex::new(HashMap::new()))
9327}
9328
9329fn install_decommission_capacity_target_permit(
9330    store_id: uuid::Uuid,
9331    target_pool_index: usize,
9332    owner: DecommissionCapacityOwner,
9333    guard: rustfs_lock::NamespaceLockGuard,
9334) -> Result<DecommissionCapacityTargetPermit> {
9335    let key = DecommissionCapacityTargetPermitKey::new(store_id, target_pool_index, owner)
9336        .ok_or_else(|| Error::other("decommission target permit is missing its mutation identity"))?;
9337    let mut permits = decommission_capacity_target_permits()
9338        .lock()
9339        .unwrap_or_else(std::sync::PoisonError::into_inner);
9340    if permits.keys().any(|candidate| candidate.owns_same_mutation(store_id, owner)) {
9341        return Err(Error::other("decommission target permit already exists for this mutation"));
9342    }
9343    permits.insert(key, guard);
9344    Ok(DecommissionCapacityTargetPermit { key })
9345}
9346
9347fn take_decommission_capacity_target_permit(
9348    store_id: uuid::Uuid,
9349    target_pool_index: usize,
9350    owner: DecommissionCapacityOwner,
9351) -> Option<rustfs_lock::NamespaceLockGuard> {
9352    let key = DecommissionCapacityTargetPermitKey::new(store_id, target_pool_index, owner)?;
9353    decommission_capacity_target_permits()
9354        .lock()
9355        .unwrap_or_else(std::sync::PoisonError::into_inner)
9356        .remove(&key)
9357}
9358
9359fn decommission_capacity_target_permit_index(store_id: uuid::Uuid, owner: DecommissionCapacityOwner) -> Option<usize> {
9360    decommission_capacity_target_permits()
9361        .lock()
9362        .unwrap_or_else(std::sync::PoisonError::into_inner)
9363        .keys()
9364        .find(|key| key.owns_same_mutation(store_id, owner))
9365        .map(|key| key.target_pool_index)
9366}
9367
9368fn discard_decommission_capacity_target_permit_except(
9369    store_id: uuid::Uuid,
9370    owner: DecommissionCapacityOwner,
9371    retained_target_pool_index: Option<usize>,
9372) {
9373    let guard = {
9374        let mut permits = decommission_capacity_target_permits()
9375            .lock()
9376            .unwrap_or_else(std::sync::PoisonError::into_inner);
9377        let key = permits
9378            .keys()
9379            .find(|key| key.owns_same_mutation(store_id, owner) && retained_target_pool_index != Some(key.target_pool_index))
9380            .copied();
9381        key.and_then(|key| permits.remove(&key))
9382    };
9383    drop(guard);
9384}
9385
9386impl Drop for DecommissionCapacityTargetPermit {
9387    fn drop(&mut self) {
9388        let guard = decommission_capacity_target_permits()
9389            .lock()
9390            .unwrap_or_else(std::sync::PoisonError::into_inner)
9391            .remove(&self.key);
9392        drop(guard);
9393    }
9394}
9395
9396#[derive(Debug, Clone, PartialEq, Eq)]
9397struct DecommissionCapacityTerminalFencePlan {
9398    model_version: u16,
9399    operation_id: uuid::Uuid,
9400    generation: u64,
9401    owner_nonce: uuid::Uuid,
9402    target_pool_indices: Vec<usize>,
9403}
9404
9405fn decommission_capacity_terminal_fence_plan(
9406    meta: &PoolMeta,
9407    source_pool_index: usize,
9408) -> Result<Option<DecommissionCapacityTerminalFencePlan>> {
9409    let Some(reservation) = meta
9410        .pools
9411        .get(source_pool_index)
9412        .and_then(|pool| pool.decommission.as_ref())
9413        .and_then(|info| info.capacity_reservation.as_ref())
9414        .filter(|reservation| reservation.active())
9415    else {
9416        return Ok(None);
9417    };
9418    if reservation.source_pool_index != source_pool_index {
9419        return Err(Error::DecommissionCapacity(
9420            "decommission terminal transition found a mismatched capacity source".to_string(),
9421        ));
9422    }
9423    if !matches!(
9424        reservation.model_version,
9425        DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION | DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
9426    ) {
9427        return Err(Error::DecommissionCapacity(format!(
9428            "decommission terminal transition found unsupported capacity lock model {}",
9429            reservation.model_version
9430        )));
9431    }
9432    let mut target_pool_indices = reservation.targets.iter().map(|target| target.pool_index).collect::<Vec<_>>();
9433    target_pool_indices.sort_unstable();
9434    Ok(Some(DecommissionCapacityTerminalFencePlan {
9435        model_version: reservation.model_version,
9436        operation_id: reservation.operation_id,
9437        generation: reservation.generation,
9438        owner_nonce: reservation.owner_nonce,
9439        target_pool_indices,
9440    }))
9441}
9442
9443#[derive(Debug, Clone, PartialEq, Eq)]
9444pub(crate) struct DecommissionDurableIlmCheckpointTarget {
9445    pub(crate) source_pool_index: usize,
9446    pub(crate) target_pool_index: usize,
9447    pub(crate) capacity_owner: DecommissionCapacityOwner,
9448    pub(crate) already_committed: bool,
9449    pub(crate) target_etag: Option<String>,
9450}
9451
9452impl DecommissionCapacityOwner {
9453    pub(crate) fn apply_to(self, opts: &mut ObjectOptions) {
9454        opts.src_pool_idx = self.source_pool_index;
9455        let admission = opts
9456            .decommission_capacity
9457            .get_or_insert_with(|| Box::new(DecommissionCapacityOptions::default()));
9458        admission.operation_id = Some(self.operation_id);
9459        admission.generation = Some(self.generation);
9460        admission.owner_nonce = Some(self.owner_nonce);
9461        admission.mutation_id = self.mutation_id;
9462    }
9463
9464    pub(crate) fn from_options(opts: &ObjectOptions) -> Option<Self> {
9465        let capacity = opts.decommission_capacity.as_deref()?;
9466        Some(Self {
9467            source_pool_index: opts.src_pool_idx,
9468            operation_id: capacity.operation_id?,
9469            generation: capacity.generation?,
9470            owner_nonce: capacity.owner_nonce?,
9471            mutation_id: capacity.mutation_id,
9472        })
9473    }
9474
9475    pub(crate) fn with_mutation_id(self, mutation_id: uuid::Uuid) -> Self {
9476        Self {
9477            mutation_id: Some(mutation_id),
9478            ..self
9479        }
9480    }
9481}
9482
9483pub(crate) fn decommission_capacity_mutation_id(
9484    owner: DecommissionCapacityOwner,
9485    bucket: &str,
9486    object: &str,
9487    version_id: Option<&str>,
9488    delete_marker: bool,
9489    mod_time: Option<OffsetDateTime>,
9490) -> uuid::Uuid {
9491    let mut hasher = Sha256::new();
9492    hasher.update(owner.operation_id.as_bytes());
9493    hasher.update(owner.generation.to_le_bytes());
9494    hasher.update(owner.source_pool_index.to_le_bytes());
9495    for value in [bucket, object] {
9496        hasher.update((value.len() as u64).to_le_bytes());
9497        hasher.update(value.as_bytes());
9498    }
9499    hasher.update([u8::from(delete_marker)]);
9500    if let Some(version_id) = version_id {
9501        hasher.update([1]);
9502        hasher.update((version_id.len() as u64).to_le_bytes());
9503        hasher.update(version_id.as_bytes());
9504    } else {
9505        hasher.update([0]);
9506    }
9507    if let Some(mod_time) = mod_time {
9508        hasher.update([1]);
9509        hasher.update(mod_time.unix_timestamp_nanos().to_le_bytes());
9510    } else {
9511        hasher.update([0]);
9512    }
9513    let digest = hasher.finalize();
9514    let mut bytes = [0; 16];
9515    bytes.copy_from_slice(&digest[..16]);
9516    uuid::Uuid::from_bytes(bytes)
9517}
9518
9519#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9520struct ExactDeleteCapacityReconciliation {
9521    source_pool_index: usize,
9522    target_pool_index: usize,
9523    mutation_id: uuid::Uuid,
9524    expected_data_bytes: usize,
9525    expected_target_physical_bytes: usize,
9526}
9527
9528fn plan_exact_delete_capacity_reconciliations(
9529    meta: &PoolMeta,
9530    object: &str,
9531    exact: &ObjectInfo,
9532) -> Result<Vec<ExactDeleteCapacityReconciliation>> {
9533    let version_id = exact.version_id.map(|version_id| version_id.to_string());
9534    let mut matches = Vec::new();
9535
9536    for (source_pool_index, pool) in meta.pools.iter().enumerate() {
9537        let Some(reservation) = pool
9538            .decommission
9539            .as_ref()
9540            .and_then(|info| info.capacity_reservation.as_ref())
9541            .filter(|reservation| reservation.active())
9542        else {
9543            continue;
9544        };
9545        let owner = DecommissionCapacityOwner {
9546            source_pool_index,
9547            operation_id: reservation.operation_id,
9548            generation: reservation.generation,
9549            owner_nonce: reservation.owner_nonce,
9550            mutation_id: None,
9551        };
9552        let logical_mutation_id = decommission_capacity_mutation_id(
9553            owner,
9554            &exact.bucket,
9555            &exact.name,
9556            version_id.as_deref(),
9557            exact.delete_marker,
9558            exact.mod_time,
9559        );
9560        // Existing data-movement producers persist directory-key intents using
9561        // either the logical name or its internal `__XLDIR__` representation.
9562        // Accept both while retaining the exact persisted identity for CAS.
9563        let internal_mutation_id = if object == exact.name {
9564            logical_mutation_id
9565        } else {
9566            decommission_capacity_mutation_id(
9567                owner,
9568                &exact.bucket,
9569                object,
9570                version_id.as_deref(),
9571                exact.delete_marker,
9572                exact.mod_time,
9573            )
9574        };
9575        let mut source_match = None;
9576
9577        for target in &reservation.targets {
9578            if target.pending_physical_bytes == 0 {
9579                continue;
9580            }
9581            let Some(pending_mutation_id) = target.pending_mutation_id else {
9582                return Err(decommission_capacity_blocked_error(format!(
9583                    "source pool {source_pool_index} target pool {} has pending capacity without an object identity",
9584                    target.pool_index
9585                )));
9586            };
9587            if pending_mutation_id != logical_mutation_id && pending_mutation_id != internal_mutation_id {
9588                continue;
9589            }
9590            if source_match.is_some() {
9591                return Err(decommission_capacity_blocked_error(format!(
9592                    "source pool {source_pool_index} has the same exact-delete capacity intent on multiple targets"
9593                )));
9594            }
9595            source_match = Some((target.pool_index, target.layout, target.pending_physical_bytes, pending_mutation_id));
9596        }
9597
9598        let Some((target_pool_index, target_layout, pending_physical_bytes, mutation_id)) = source_match else {
9599            continue;
9600        };
9601        if exact.version_id.is_none() && exact.mod_time.is_none() {
9602            return Err(decommission_capacity_blocked_error(
9603                "unversioned exact delete cannot identify pending capacity without a modification time",
9604            ));
9605        }
9606        let expected_data_bytes = if exact.delete_marker {
9607            0
9608        } else {
9609            usize::try_from(exact.size).map_err(|_| {
9610                decommission_capacity_blocked_error("exact delete cannot reconcile a negative or overflowing object size")
9611            })?
9612        };
9613        let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
9614        if pending_physical_bytes != expected_target_physical_bytes {
9615            return Err(decommission_capacity_blocked_error(format!(
9616                "source pool {source_pool_index} target pool {target_pool_index} pending capacity does not match the exact object size"
9617            )));
9618        }
9619        let remaining_target_physical_bytes = reservation
9620            .targets
9621            .iter()
9622            .find(|target| target.pool_index == target_pool_index)
9623            .map(|target| {
9624                target.remaining_reserved_physical_bytes(reservation.temporary_copies)
9625                    / 1usize.saturating_add(reservation.temporary_copies)
9626            })
9627            .unwrap_or_default();
9628        let remaining_total_physical_bytes = reservation
9629            .predicted_physical_bytes
9630            .saturating_sub(reservation.consumed_target_physical_bytes);
9631        let remaining_data_bytes = reservation
9632            .source_data_equivalent_bytes
9633            .saturating_sub(reservation.committed_data_bytes);
9634        if expected_target_physical_bytes > remaining_target_physical_bytes
9635            || expected_target_physical_bytes > remaining_total_physical_bytes
9636            || expected_data_bytes > remaining_data_bytes
9637        {
9638            return Err(decommission_capacity_blocked_error(format!(
9639                "source pool {source_pool_index} target pool {target_pool_index} lacks reservation capacity for the exact object"
9640            )));
9641        }
9642        matches.push(ExactDeleteCapacityReconciliation {
9643            source_pool_index,
9644            target_pool_index,
9645            mutation_id,
9646            expected_data_bytes,
9647            expected_target_physical_bytes,
9648        });
9649    }
9650
9651    Ok(matches)
9652}
9653
9654fn ensure_exact_delete_capacity_namespace_fences(opts: &ObjectOptions, bucket: &str, object: &str) -> Result<()> {
9655    let object_fence = opts.namespace_lock_fence.as_ref().ok_or_else(|| {
9656        decommission_capacity_blocked_error("exact delete capacity reconciliation requires an object namespace fence")
9657    })?;
9658    if object_fence.is_lock_lost() {
9659        return Err(StorageError::NamespaceLockQuorumUnavailable {
9660            mode: "exact_delete_capacity_reconciliation",
9661            bucket: bucket.to_string(),
9662            object: decode_dir_object(object),
9663            required: 1,
9664            achieved: 0,
9665        });
9666    }
9667    if opts
9668        .bucket_lifecycle_lock_fence
9669        .as_ref()
9670        .is_some_and(crate::object_api::NamespaceLockFence::is_lock_lost)
9671    {
9672        return Err(StorageError::NamespaceLockQuorumUnavailable {
9673            mode: "exact_delete_capacity_bucket_generation",
9674            bucket: bucket.to_string(),
9675            object: decode_dir_object(object),
9676            required: 1,
9677            achieved: 0,
9678        });
9679    }
9680    Ok(())
9681}
9682
9683pub(crate) fn ensure_decommission_capacity_mutation_id(bucket: &str, object: &str, opts: &mut ObjectOptions) {
9684    if opts
9685        .decommission_capacity
9686        .as_deref()
9687        .is_none_or(|capacity| capacity.mutation_id.is_some())
9688    {
9689        return;
9690    }
9691    let Some(owner) = DecommissionCapacityOwner::from_options(opts) else {
9692        return;
9693    };
9694    let mutation_id =
9695        decommission_capacity_mutation_id(owner, bucket, object, opts.version_id.as_deref(), opts.delete_marker, opts.mod_time);
9696    if let Some(capacity) = opts.decommission_capacity.as_mut() {
9697        capacity.mutation_id = Some(mutation_id);
9698    }
9699}
9700
9701impl DecommissionCapacityReservation {
9702    fn active(&self) -> bool {
9703        self.released_at.is_none()
9704    }
9705
9706    fn lease_active_at(&self, now: OffsetDateTime) -> bool {
9707        self.active() && self.expires_at > now
9708    }
9709
9710    fn admits_owner(&self, owner: DecommissionCapacityOwner, now: OffsetDateTime) -> bool {
9711        self.lease_active_at(now)
9712            && self.source_pool_index == owner.source_pool_index
9713            && self.operation_id == owner.operation_id
9714            && self.generation == owner.generation
9715            && self.owner_nonce == owner.owner_nonce
9716    }
9717
9718    fn admits_cleanup_owner(&self, owner: DecommissionCapacityOwner) -> bool {
9719        self.active()
9720            && self.source_pool_index == owner.source_pool_index
9721            && self.operation_id == owner.operation_id
9722            && self.generation == owner.generation
9723    }
9724
9725    fn remaining_peak_physical_bytes(&self) -> usize {
9726        self.predicted_physical_bytes
9727            .saturating_sub(self.consumed_target_physical_bytes)
9728            .saturating_mul(1usize.saturating_add(self.temporary_copies))
9729    }
9730}
9731
9732impl DecommissionCapacityTarget {
9733    fn remaining_reserved_physical_bytes(&self, temporary_copies: usize) -> usize {
9734        self.reserved_physical_bytes.saturating_sub(
9735            self.consumed_physical_bytes
9736                .saturating_mul(1usize.saturating_add(temporary_copies)),
9737        )
9738    }
9739}
9740
9741#[derive(Debug, Clone, Serialize, Deserialize, Default)]
9742pub struct PoolDecommissionInfo {
9743    #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
9744    pub start_time: Option<OffsetDateTime>,
9745    #[serde(rename = "startSize")]
9746    pub start_size: usize,
9747    #[serde(rename = "totalSize")]
9748    pub total_size: usize,
9749    #[serde(rename = "currentSize")]
9750    pub current_size: usize,
9751    #[serde(rename = "complete")]
9752    pub complete: bool,
9753    #[serde(rename = "failed")]
9754    pub failed: bool,
9755    #[serde(rename = "canceled")]
9756    pub canceled: bool,
9757    #[serde(skip)]
9758    pub queued: bool,
9759
9760    #[serde(skip)]
9761    pub queued_buckets: Vec<String>,
9762    #[serde(skip)]
9763    pub decommissioned_buckets: Vec<String>,
9764    #[serde(skip)]
9765    pub bucket: String,
9766    #[serde(skip)]
9767    pub prefix: String,
9768    #[serde(skip)]
9769    pub object: String,
9770    #[serde(skip)]
9771    pub stage: String,
9772
9773    #[serde(rename = "objectsDecommissioned")]
9774    pub items_decommissioned: usize,
9775    #[serde(rename = "objectsDecommissionedFailed")]
9776    pub items_decommission_failed: usize,
9777    #[serde(rename = "bytesDecommissioned")]
9778    pub bytes_done: usize,
9779    #[serde(rename = "bytesDecommissionedFailed")]
9780    pub bytes_failed: usize,
9781    #[serde(rename = "terminalReloadAttemptAt", with = "time::serde::rfc3339::option", default)]
9782    pub terminal_reload_attempt_at: Option<OffsetDateTime>,
9783    #[serde(rename = "terminalReloadFailures", default)]
9784    pub terminal_reload_failures: Vec<String>,
9785    #[serde(rename = "capacityReservation", default, skip_serializing_if = "Option::is_none")]
9786    pub capacity_reservation: Option<DecommissionCapacityReservation>,
9787    #[serde(rename = "capacityBlockedReason", default, skip_serializing_if = "Option::is_none")]
9788    pub capacity_blocked_reason: Option<String>,
9789    #[serde(skip)]
9790    pub unresolved_entries: Vec<DecommissionUnresolvedEntry>,
9791    #[serde(skip)]
9792    pub progress_save_item_baseline: usize,
9793    #[serde(skip)]
9794    pub progress_save_retry_after: Option<OffsetDateTime>,
9795    #[serde(skip)]
9796    pub progress_save_last_at: Option<OffsetDateTime>,
9797}
9798
9799#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9800struct DecommissionProgressCheckpoint {
9801    start_time: Option<OffsetDateTime>,
9802    queued: bool,
9803    counted_items: usize,
9804    checkpoint_at: OffsetDateTime,
9805    capacity_operation_id: Option<uuid::Uuid>,
9806    capacity_owner_nonce: Option<uuid::Uuid>,
9807    capacity_lease_expires_at: Option<OffsetDateTime>,
9808}
9809
9810impl PoolDecommissionInfo {
9811    pub fn has_decommission_state(&self) -> bool {
9812        self.complete
9813            || self.failed
9814            || self.canceled
9815            || self.queued
9816            || self.start_time.is_some()
9817            || self.start_size > 0
9818            || !self.queued_buckets.is_empty()
9819            || !self.decommissioned_buckets.is_empty()
9820            || !self.bucket.is_empty()
9821            || !self.prefix.is_empty()
9822            || !self.object.is_empty()
9823            || !self.stage.is_empty()
9824            || self.items_decommissioned > 0
9825            || self.items_decommission_failed > 0
9826            || self.bytes_done > 0
9827            || self.bytes_failed > 0
9828            || self.terminal_reload_attempt_at.is_some()
9829            || !self.terminal_reload_failures.is_empty()
9830            || self.capacity_reservation.is_some()
9831            || self.capacity_blocked_reason.is_some()
9832            || !self.unresolved_entries.is_empty()
9833    }
9834
9835    fn counted_items(&self) -> usize {
9836        self.items_decommissioned.saturating_add(self.items_decommission_failed)
9837    }
9838
9839    fn items_since_last_progress_save(&self) -> usize {
9840        self.counted_items().saturating_sub(self.progress_save_item_baseline)
9841    }
9842
9843    fn mark_progress_saved(&mut self) {
9844        self.progress_save_item_baseline = self.counted_items();
9845        self.progress_save_retry_after = None;
9846    }
9847
9848    pub fn bucket_push(&mut self, bucket: &DecomBucketInfo) {
9849        let bucket_key = bucket.to_string();
9850        if self.is_bucket_decommissioned(&bucket_key) {
9851            return;
9852        }
9853
9854        for b in self.queued_buckets.iter() {
9855            if b == &bucket_key {
9856                return;
9857            }
9858        }
9859
9860        self.queued_buckets.push(bucket_key);
9861
9862        self.bucket = bucket.name.clone();
9863        self.prefix = bucket.prefix.clone();
9864    }
9865    pub fn is_bucket_decommissioned(&self, bucket: &String) -> bool {
9866        for b in self.decommissioned_buckets.iter() {
9867            if b == bucket {
9868                return true;
9869            }
9870        }
9871        false
9872    }
9873    pub fn bucket_pop(&mut self, bucket: &String) -> bool {
9874        let mut found = None;
9875        for (i, b) in self.queued_buckets.iter().enumerate() {
9876            if b == bucket {
9877                found = Some(i);
9878                break;
9879            }
9880        }
9881
9882        if let Some(i) = found {
9883            if !self.is_bucket_decommissioned(bucket) {
9884                self.decommissioned_buckets.push(bucket.clone());
9885            }
9886            self.queued_buckets.remove(i);
9887            if &self.bucket == bucket {
9888                self.bucket = "".to_owned();
9889                self.prefix = "".to_owned();
9890                self.object = "".to_owned();
9891            }
9892
9893            return true;
9894        }
9895        false
9896    }
9897}
9898
9899#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9900pub struct PoolSpaceInfo {
9901    pub free: usize,
9902    pub total: usize,
9903    pub used: usize,
9904}
9905
9906#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9907pub(crate) struct DecommissionPoolCapacityInfo {
9908    pool_index: usize,
9909    space: PoolSpaceInfo,
9910    layout: DecommissionErasureLayout,
9911    physical_free: usize,
9912    physical_total: usize,
9913    physical_used: usize,
9914}
9915
9916impl DecommissionPoolCapacityInfo {
9917    #[cfg(any(test, feature = "test-util"))]
9918    pub(crate) fn for_test(
9919        pool_index: usize,
9920        layout: DecommissionErasureLayout,
9921        physical_free: usize,
9922        physical_total: usize,
9923        physical_used: usize,
9924    ) -> Self {
9925        Self {
9926            pool_index,
9927            space: PoolSpaceInfo {
9928                free: physical_free,
9929                total: physical_total,
9930                used: physical_used,
9931            },
9932            layout,
9933            physical_free,
9934            physical_total,
9935            physical_used,
9936        }
9937    }
9938}
9939
9940#[cfg(any(test, feature = "test-util"))]
9941type DecommissionCapacityInfoOverrides =
9942    std::sync::Mutex<HashMap<uuid::Uuid, std::collections::VecDeque<Vec<DecommissionPoolCapacityInfo>>>>;
9943
9944#[cfg(any(test, feature = "test-util"))]
9945static DECOMMISSION_CAPACITY_INFO_OVERRIDES: std::sync::OnceLock<DecommissionCapacityInfoOverrides> = std::sync::OnceLock::new();
9946
9947/// Queues capacity snapshots consumed in order by `get_decommission_all_pool_capacity_infos`;
9948/// the final snapshot is retained and replayed for every subsequent sample, so tests never
9949/// fall back to the host's real disk statistics once an override is installed.
9950#[cfg(any(test, feature = "test-util"))]
9951pub(crate) fn set_decommission_capacity_info_overrides_for_test(
9952    store_id: uuid::Uuid,
9953    snapshots: Vec<Vec<DecommissionPoolCapacityInfo>>,
9954) {
9955    DECOMMISSION_CAPACITY_INFO_OVERRIDES
9956        .get_or_init(|| std::sync::Mutex::new(HashMap::new()))
9957        .lock()
9958        .expect("decommission capacity info override should not be poisoned")
9959        .insert(store_id, snapshots.into());
9960}
9961
9962#[cfg(any(test, feature = "test-util"))]
9963fn take_decommission_capacity_info_override_for_test(store_id: uuid::Uuid) -> Option<Vec<DecommissionPoolCapacityInfo>> {
9964    let mut overrides = DECOMMISSION_CAPACITY_INFO_OVERRIDES
9965        .get_or_init(|| std::sync::Mutex::new(HashMap::new()))
9966        .lock()
9967        .expect("decommission capacity info override should not be poisoned");
9968    let queue = overrides.get_mut(&store_id)?;
9969    if queue.len() > 1 {
9970        queue.pop_front()
9971    } else {
9972        // The final snapshot is replayed forever: extra sampling points added to
9973        // the decommission paths must keep observing injected capacity instead of
9974        // silently falling back to the host's real statfs numbers (see #6989).
9975        queue.front().cloned()
9976    }
9977}
9978
9979#[cfg(test)]
9980struct DecommissionCapacityLockOrderBarrierState {
9981    owner_store_id: uuid::Uuid,
9982    external_store_id: uuid::Uuid,
9983    owner_arrived: tokio::sync::Notify,
9984    owner_release: tokio::sync::Notify,
9985    owner_pause_enabled: AtomicBool,
9986    external_capacity_released: tokio::sync::Notify,
9987    external_object_capacity_probe_acquired: tokio::sync::Notify,
9988    external_object_capacity_probe_release: tokio::sync::Notify,
9989    external_object_capacity_probe_paused: AtomicBool,
9990    external_object_commit_phase_started: tokio::sync::Notify,
9991    external_object_commit_phase_release: tokio::sync::Notify,
9992    external_object_commit_phase_paused: AtomicBool,
9993    external_heal_operation_started: tokio::sync::Notify,
9994    external_heal_target_lock_attempted: tokio::sync::Notify,
9995    target_gate_retry_entered: tokio::sync::Notify,
9996    target_gate_retry_entries: AtomicUsize,
9997    target_gate_inline_retry_entered: tokio::sync::Notify,
9998    target_gate_inline_retry_entries: AtomicUsize,
9999    target_gate_exact_reloads: AtomicUsize,
10000    target_gate_acquire_pause_target: AtomicUsize,
10001    target_gate_acquire_entered: tokio::sync::Notify,
10002    target_gate_acquire_release: tokio::sync::Notify,
10003    cancel_before_start_entered: tokio::sync::Notify,
10004    cancel_before_start_release: tokio::sync::Notify,
10005    cancel_before_start_paused: AtomicBool,
10006    cancel_target_timeout_entered: tokio::sync::Notify,
10007    cancel_target_timeout_release: tokio::sync::Notify,
10008    cancel_target_timeout_paused: AtomicBool,
10009    cancel_target_timeouts: AtomicUsize,
10010}
10011
10012#[cfg(test)]
10013pub(crate) struct DecommissionCapacityLockOrderBarrier {
10014    state: Arc<DecommissionCapacityLockOrderBarrierState>,
10015}
10016
10017#[cfg(test)]
10018static DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER: std::sync::OnceLock<
10019    std::sync::Mutex<Option<Arc<DecommissionCapacityLockOrderBarrierState>>>,
10020> = std::sync::OnceLock::new();
10021
10022#[cfg(test)]
10023impl DecommissionCapacityLockOrderBarrier {
10024    pub(crate) fn install(owner_store_id: uuid::Uuid, external_store_id: uuid::Uuid) -> Self {
10025        let state = Arc::new(DecommissionCapacityLockOrderBarrierState {
10026            owner_store_id,
10027            external_store_id,
10028            owner_arrived: tokio::sync::Notify::new(),
10029            owner_release: tokio::sync::Notify::new(),
10030            owner_pause_enabled: AtomicBool::new(true),
10031            external_capacity_released: tokio::sync::Notify::new(),
10032            external_object_capacity_probe_acquired: tokio::sync::Notify::new(),
10033            external_object_capacity_probe_release: tokio::sync::Notify::new(),
10034            external_object_capacity_probe_paused: AtomicBool::new(false),
10035            external_object_commit_phase_started: tokio::sync::Notify::new(),
10036            external_object_commit_phase_release: tokio::sync::Notify::new(),
10037            external_object_commit_phase_paused: AtomicBool::new(false),
10038            external_heal_operation_started: tokio::sync::Notify::new(),
10039            external_heal_target_lock_attempted: tokio::sync::Notify::new(),
10040            target_gate_retry_entered: tokio::sync::Notify::new(),
10041            target_gate_retry_entries: AtomicUsize::new(0),
10042            target_gate_inline_retry_entered: tokio::sync::Notify::new(),
10043            target_gate_inline_retry_entries: AtomicUsize::new(0),
10044            target_gate_exact_reloads: AtomicUsize::new(0),
10045            target_gate_acquire_pause_target: AtomicUsize::new(usize::MAX),
10046            target_gate_acquire_entered: tokio::sync::Notify::new(),
10047            target_gate_acquire_release: tokio::sync::Notify::new(),
10048            cancel_before_start_entered: tokio::sync::Notify::new(),
10049            cancel_before_start_release: tokio::sync::Notify::new(),
10050            cancel_before_start_paused: AtomicBool::new(false),
10051            cancel_target_timeout_entered: tokio::sync::Notify::new(),
10052            cancel_target_timeout_release: tokio::sync::Notify::new(),
10053            cancel_target_timeout_paused: AtomicBool::new(false),
10054            cancel_target_timeouts: AtomicUsize::new(0),
10055        });
10056        let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10057            .get_or_init(|| std::sync::Mutex::new(None))
10058            .lock()
10059            .expect("decommission capacity lock-order barrier should not be poisoned");
10060        assert!(slot.is_none(), "decommission capacity lock-order barrier must be unique");
10061        *slot = Some(Arc::clone(&state));
10062        Self { state }
10063    }
10064
10065    #[cfg(feature = "test-util")]
10066    pub(crate) async fn wait_until_owner_paused(&self) {
10067        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.owner_arrived.notified())
10068            .await
10069            .expect("owned capacity mutation should reach admission before its metadata write");
10070    }
10071
10072    pub(crate) async fn wait_until_external_capacity_released(&self) {
10073        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.external_capacity_released.notified())
10074            .await
10075            .expect("external mutation should release capacity before waiting for the object namespace");
10076    }
10077
10078    #[cfg(feature = "test-util")]
10079    pub(crate) async fn wait_until_external_object_capacity_probe_acquired(&self) {
10080        tokio::time::timeout(
10081            std::time::Duration::from_secs(30),
10082            self.state.external_object_capacity_probe_acquired.notified(),
10083        )
10084        .await
10085        .expect("external object mutation should acquire its no-active capacity probe");
10086    }
10087
10088    #[cfg(feature = "test-util")]
10089    pub(crate) async fn wait_until_external_object_commit_phase_started(&self) {
10090        tokio::time::timeout(
10091            std::time::Duration::from_secs(30),
10092            self.state.external_object_commit_phase_started.notified(),
10093        )
10094        .await
10095        .expect("external object mutation should reach its staged commit phase before waiting for the namespace");
10096    }
10097
10098    pub(crate) async fn wait_until_external_heal_operation_started(&self) {
10099        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.external_heal_operation_started.notified())
10100            .await
10101            .expect("external heal should reach the target operation after capacity admission");
10102    }
10103
10104    pub(crate) async fn wait_until_external_heal_target_lock_attempted(&self) {
10105        tokio::time::timeout(
10106            std::time::Duration::from_secs(30),
10107            self.state.external_heal_target_lock_attempted.notified(),
10108        )
10109        .await
10110        .expect("external heal should attempt the target namespace lock after capacity admission");
10111    }
10112
10113    #[cfg(feature = "test-util")]
10114    pub(crate) async fn wait_until_target_gate_retry(&self) {
10115        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.target_gate_retry_entered.notified())
10116            .await
10117            .expect("decommission entry should observe target gate contention");
10118    }
10119
10120    #[cfg(feature = "test-util")]
10121    pub(crate) async fn wait_until_target_gate_retries(&self, expected: usize) {
10122        tokio::time::timeout(std::time::Duration::from_secs(30), async {
10123            loop {
10124                let notified = self.state.target_gate_retry_entered.notified();
10125                if self.state.target_gate_retry_entries.load(Ordering::Acquire) >= expected {
10126                    return;
10127                }
10128                notified.await;
10129            }
10130        })
10131        .await
10132        .expect("decommission entries should observe target gate contention");
10133    }
10134
10135    #[cfg(feature = "test-util")]
10136    pub(crate) fn target_gate_inline_retries(&self) -> usize {
10137        self.state.target_gate_inline_retry_entries.load(Ordering::Acquire)
10138    }
10139
10140    #[cfg(feature = "test-util")]
10141    pub(crate) fn target_gate_exact_reloads(&self) -> usize {
10142        self.state.target_gate_exact_reloads.load(Ordering::Acquire)
10143    }
10144
10145    #[cfg(feature = "test-util")]
10146    pub(crate) fn pause_target_gate_acquire(&self, target_pool_index: usize) {
10147        self.state
10148            .target_gate_acquire_pause_target
10149            .store(target_pool_index, Ordering::Release);
10150    }
10151
10152    #[cfg(feature = "test-util")]
10153    pub(crate) async fn wait_until_target_gate_acquire_paused(&self) {
10154        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.target_gate_acquire_entered.notified())
10155            .await
10156            .expect("decommission capacity mutation should pause before acquiring its target gate");
10157    }
10158
10159    #[cfg(feature = "test-util")]
10160    pub(crate) fn release_target_gate_acquire(&self) {
10161        self.state
10162            .target_gate_acquire_pause_target
10163            .store(usize::MAX, Ordering::Release);
10164        self.state.target_gate_acquire_release.notify_one();
10165    }
10166
10167    pub(crate) fn pause_cancel_before_start(&self) {
10168        self.state.cancel_before_start_paused.store(true, Ordering::Release);
10169    }
10170
10171    pub(crate) async fn wait_until_cancel_before_start(&self) {
10172        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.cancel_before_start_entered.notified())
10173            .await
10174            .expect("decommission cancel should pause before acquiring the start gate");
10175    }
10176
10177    pub(crate) fn release_cancel_before_start(&self) {
10178        self.state.cancel_before_start_release.notify_one();
10179    }
10180
10181    fn pause_cancel_target_timeout(&self) {
10182        self.state.cancel_target_timeout_paused.store(true, Ordering::Release);
10183    }
10184
10185    async fn wait_until_cancel_target_timeout(&self) {
10186        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.cancel_target_timeout_entered.notified())
10187            .await
10188            .expect("cancel should observe contention on its target capacity fence");
10189    }
10190
10191    fn release_cancel_target_timeout(&self) {
10192        self.state.cancel_target_timeout_paused.store(false, Ordering::Release);
10193        self.state.cancel_target_timeout_release.notify_one();
10194    }
10195
10196    #[cfg(feature = "test-util")]
10197    pub(crate) fn release_owner(&self) {
10198        self.state.owner_release.notify_one();
10199        self.state.target_gate_acquire_release.notify_one();
10200    }
10201
10202    #[cfg(feature = "test-util")]
10203    pub(crate) fn disable_owner_pause(&self) {
10204        self.state.owner_pause_enabled.store(false, Ordering::Release);
10205        self.state.owner_release.notify_waiters();
10206    }
10207
10208    #[cfg(feature = "test-util")]
10209    pub(crate) fn pause_external_object_commit_phase(&self) {
10210        self.state.external_object_commit_phase_paused.store(true, Ordering::Release);
10211    }
10212
10213    #[cfg(feature = "test-util")]
10214    pub(crate) fn release_external_object_commit_phase(&self) {
10215        self.state.external_object_commit_phase_release.notify_one();
10216    }
10217
10218    #[cfg(feature = "test-util")]
10219    pub(crate) fn pause_external_object_capacity_probe(&self) {
10220        self.state
10221            .external_object_capacity_probe_paused
10222            .store(true, Ordering::Release);
10223    }
10224
10225    #[cfg(feature = "test-util")]
10226    pub(crate) fn release_external_object_capacity_probe(&self) {
10227        self.state.external_object_capacity_probe_release.notify_one();
10228    }
10229}
10230
10231#[cfg(test)]
10232impl Drop for DecommissionCapacityLockOrderBarrier {
10233    fn drop(&mut self) {
10234        self.state.owner_release.notify_one();
10235        self.state.external_object_capacity_probe_release.notify_one();
10236        self.state.external_object_commit_phase_release.notify_one();
10237        self.state.cancel_before_start_release.notify_one();
10238        self.state.cancel_target_timeout_release.notify_one();
10239        let mut slot = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10240            .get_or_init(|| std::sync::Mutex::new(None))
10241            .lock()
10242            .expect("decommission capacity lock-order barrier should not be poisoned");
10243        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
10244            *slot = None;
10245        }
10246    }
10247}
10248
10249#[cfg(test)]
10250async fn pause_decommission_capacity_before_owner_write(store_id: uuid::Uuid) {
10251    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10252        .get_or_init(|| std::sync::Mutex::new(None))
10253        .lock()
10254        .expect("decommission capacity lock-order barrier should not be poisoned")
10255        .as_ref()
10256        .filter(|state| state.owner_store_id == store_id && state.owner_pause_enabled.load(Ordering::Acquire))
10257        .cloned();
10258    if let Some(barrier) = barrier {
10259        barrier.owner_arrived.notify_one();
10260        barrier.owner_release.notified().await;
10261    }
10262}
10263
10264#[cfg(test)]
10265async fn pause_decommission_capacity_before_target_gate_acquire(store_id: uuid::Uuid, target_pool_index: usize) {
10266    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10267        .get_or_init(|| std::sync::Mutex::new(None))
10268        .lock()
10269        .expect("decommission capacity lock-order barrier should not be poisoned")
10270        .as_ref()
10271        .filter(|state| {
10272            state.owner_store_id == store_id
10273                && state.target_gate_acquire_pause_target.load(Ordering::Acquire) == target_pool_index
10274        })
10275        .cloned();
10276    if let Some(barrier) = barrier {
10277        barrier.target_gate_acquire_entered.notify_one();
10278        barrier.target_gate_acquire_release.notified().await;
10279    }
10280}
10281
10282#[cfg(test)]
10283async fn pause_decommission_cancel_before_start_gate(store_id: uuid::Uuid) {
10284    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10285        .get_or_init(|| std::sync::Mutex::new(None))
10286        .lock()
10287        .expect("decommission capacity lock-order barrier should not be poisoned")
10288        .as_ref()
10289        .filter(|state| state.owner_store_id == store_id && state.cancel_before_start_paused.load(Ordering::Acquire))
10290        .cloned();
10291    if let Some(barrier) = barrier {
10292        barrier.cancel_before_start_entered.notify_one();
10293        barrier.cancel_before_start_release.notified().await;
10294    }
10295}
10296
10297#[cfg(test)]
10298async fn pause_decommission_cancel_target_timeout(store_id: uuid::Uuid) {
10299    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10300        .get_or_init(|| std::sync::Mutex::new(None))
10301        .lock()
10302        .expect("decommission capacity lock-order barrier should not be poisoned")
10303        .as_ref()
10304        .filter(|state| state.owner_store_id == store_id)
10305        .cloned();
10306    if let Some(barrier) = barrier {
10307        barrier.cancel_target_timeouts.fetch_add(1, Ordering::AcqRel);
10308        barrier.cancel_target_timeout_entered.notify_one();
10309        if barrier.cancel_target_timeout_paused.load(Ordering::Acquire) {
10310            barrier.cancel_target_timeout_release.notified().await;
10311        }
10312    }
10313}
10314
10315#[cfg(test)]
10316fn notify_decommission_target_gate_retry(store_id: uuid::Uuid) {
10317    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10318        .get_or_init(|| std::sync::Mutex::new(None))
10319        .lock()
10320        .expect("decommission capacity lock-order barrier should not be poisoned")
10321        .as_ref()
10322        .filter(|state| state.owner_store_id == store_id)
10323        .cloned();
10324    if let Some(barrier) = barrier {
10325        barrier.target_gate_retry_entries.fetch_add(1, Ordering::AcqRel);
10326        barrier.target_gate_retry_entered.notify_one();
10327    }
10328}
10329
10330#[cfg(test)]
10331fn notify_decommission_target_gate_inline_retry(store_id: uuid::Uuid) {
10332    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10333        .get_or_init(|| std::sync::Mutex::new(None))
10334        .lock()
10335        .expect("decommission capacity lock-order barrier should not be poisoned")
10336        .as_ref()
10337        .filter(|state| state.owner_store_id == store_id)
10338        .cloned();
10339    if let Some(barrier) = barrier {
10340        barrier.target_gate_inline_retry_entries.fetch_add(1, Ordering::AcqRel);
10341        barrier.target_gate_inline_retry_entered.notify_one();
10342    }
10343}
10344
10345#[cfg(test)]
10346fn notify_decommission_target_gate_exact_reload(store_id: uuid::Uuid) {
10347    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10348        .get_or_init(|| std::sync::Mutex::new(None))
10349        .lock()
10350        .expect("decommission capacity lock-order barrier should not be poisoned")
10351        .as_ref()
10352        .filter(|state| state.owner_store_id == store_id)
10353        .cloned();
10354    if let Some(barrier) = barrier {
10355        barrier.target_gate_exact_reloads.fetch_add(1, Ordering::AcqRel);
10356    }
10357}
10358
10359#[cfg(test)]
10360pub(crate) fn notify_decommission_external_object_capacity_released(store_id: uuid::Uuid) {
10361    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10362        .get_or_init(|| std::sync::Mutex::new(None))
10363        .lock()
10364        .expect("decommission capacity lock-order barrier should not be poisoned")
10365        .as_ref()
10366        .filter(|state| state.external_store_id == store_id)
10367        .cloned();
10368    if let Some(barrier) = barrier {
10369        barrier.external_capacity_released.notify_one();
10370    }
10371}
10372
10373#[cfg(test)]
10374pub(crate) fn notify_decommission_external_object_capacity_probe_acquired(store_id: uuid::Uuid) {
10375    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10376        .get_or_init(|| std::sync::Mutex::new(None))
10377        .lock()
10378        .expect("decommission capacity lock-order barrier should not be poisoned")
10379        .as_ref()
10380        .filter(|state| state.external_store_id == store_id)
10381        .cloned();
10382    if let Some(barrier) = barrier {
10383        barrier.external_object_capacity_probe_acquired.notify_one();
10384    }
10385}
10386
10387#[cfg(test)]
10388pub(crate) async fn wait_for_decommission_external_object_capacity_probe_release(store_id: uuid::Uuid) {
10389    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10390        .get_or_init(|| std::sync::Mutex::new(None))
10391        .lock()
10392        .expect("decommission capacity lock-order barrier should not be poisoned")
10393        .as_ref()
10394        .filter(|state| state.external_store_id == store_id)
10395        .cloned();
10396    if let Some(barrier) = barrier
10397        && barrier.external_object_capacity_probe_paused.load(Ordering::Acquire)
10398    {
10399        barrier.external_object_capacity_probe_release.notified().await;
10400    }
10401}
10402
10403#[cfg(test)]
10404pub(crate) fn notify_decommission_external_object_commit_phase_started(store_id: uuid::Uuid) {
10405    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10406        .get_or_init(|| std::sync::Mutex::new(None))
10407        .lock()
10408        .expect("decommission capacity lock-order barrier should not be poisoned")
10409        .as_ref()
10410        .filter(|state| state.external_store_id == store_id)
10411        .cloned();
10412    if let Some(barrier) = barrier {
10413        barrier.external_object_commit_phase_started.notify_one();
10414    }
10415}
10416
10417#[cfg(test)]
10418pub(crate) async fn wait_for_decommission_external_object_commit_phase_release(store_id: uuid::Uuid) {
10419    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10420        .get_or_init(|| std::sync::Mutex::new(None))
10421        .lock()
10422        .expect("decommission capacity lock-order barrier should not be poisoned")
10423        .as_ref()
10424        .filter(|state| state.external_store_id == store_id)
10425        .cloned();
10426    if let Some(barrier) = barrier
10427        && barrier.external_object_commit_phase_paused.load(Ordering::Acquire)
10428    {
10429        barrier.external_object_commit_phase_release.notified().await;
10430    }
10431}
10432
10433#[cfg(test)]
10434pub(crate) fn notify_decommission_external_heal_operation_started(store_id: uuid::Uuid) {
10435    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10436        .get_or_init(|| std::sync::Mutex::new(None))
10437        .lock()
10438        .expect("decommission capacity lock-order barrier should not be poisoned")
10439        .as_ref()
10440        .filter(|state| state.external_store_id == store_id)
10441        .cloned();
10442    if let Some(barrier) = barrier {
10443        barrier.external_heal_operation_started.notify_one();
10444    }
10445}
10446
10447#[cfg(test)]
10448pub(crate) fn notify_decommission_external_heal_target_lock_attempted() {
10449    let barrier = DECOMMISSION_CAPACITY_LOCK_ORDER_BARRIER
10450        .get_or_init(|| std::sync::Mutex::new(None))
10451        .lock()
10452        .expect("decommission capacity lock-order barrier should not be poisoned")
10453        .as_ref()
10454        .cloned();
10455    if let Some(barrier) = barrier {
10456        barrier.external_heal_target_lock_attempted.notify_one();
10457    }
10458}
10459
10460#[derive(Debug, Default, Clone)]
10461pub struct DecomBucketInfo {
10462    pub name: String,
10463    pub prefix: String,
10464}
10465
10466impl Display for DecomBucketInfo {
10467    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
10468        write!(
10469            f,
10470            "{}",
10471            path_join(&[PathBuf::from(self.name.clone()), PathBuf::from(self.prefix.clone())]).to_string_lossy()
10472        )
10473    }
10474}
10475
10476#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10477enum DecommissionFinalState {
10478    Complete,
10479    Failed,
10480}
10481
10482fn determine_decommission_final_state(items_failed: usize, was_cancelled: bool) -> DecommissionFinalState {
10483    if items_failed > 0 || was_cancelled {
10484        DecommissionFinalState::Failed
10485    } else {
10486        DecommissionFinalState::Complete
10487    }
10488}
10489
10490fn decommission_remaining_version_count(versions: &[rustfs_filemeta::FileInfo], expired: usize) -> usize {
10491    versions
10492        .iter()
10493        .filter(|version| !version.tier_free_version())
10494        .count()
10495        .saturating_sub(expired)
10496}
10497
10498fn should_skip_decommission_delete_marker(
10499    version: &rustfs_filemeta::FileInfo,
10500    remaining_versions: usize,
10501    replication_configured: bool,
10502) -> bool {
10503    // Match MinIO decommission behavior: an empty delete marker is not moved to
10504    // another pool unless replication is configured and its marker state matters.
10505    version.deleted && remaining_versions == 1 && !replication_configured
10506}
10507
10508fn decommission_delete_marker_opts(
10509    version: &rustfs_filemeta::FileInfo,
10510    version_id: Option<String>,
10511    src_pool_idx: usize,
10512    expected_bucket_incarnation_id: Option<uuid::Uuid>,
10513) -> ObjectOptions {
10514    let version_suspended = version.version_id.is_none() && version_id.is_none();
10515    ObjectOptions {
10516        versioned: !version_suspended,
10517        version_suspended,
10518        version_id: version_id.or_else(|| version_suspended.then(|| uuid::Uuid::nil().to_string())),
10519        mod_time: version.mod_time,
10520        src_pool_idx,
10521        data_movement: true,
10522        delete_marker: true,
10523        skip_decommissioned: true,
10524        expected_bucket_incarnation_id,
10525        delete_replication: version
10526            .replication_state_internal
10527            .as_ref()
10528            .map(replication_state_from_filemeta),
10529        ..Default::default()
10530    }
10531}
10532
10533fn decommission_object_migration_read_opts(version_id: Option<String>) -> ObjectOptions {
10534    ObjectOptions {
10535        version_id,
10536        no_lock: true,
10537        data_movement: true,
10538        raw_data_movement_read: true,
10539        skip_decommissioned: true,
10540        skip_rebalancing: true,
10541        ..Default::default()
10542    }
10543}
10544
10545fn decommission_remote_tiered_opts(
10546    version: &rustfs_filemeta::FileInfo,
10547    version_id: Option<String>,
10548    src_pool_idx: usize,
10549    expected_bucket_incarnation_id: Option<uuid::Uuid>,
10550) -> ObjectOptions {
10551    ObjectOptions {
10552        versioned: version_id.is_some(),
10553        version_id,
10554        mod_time: version.mod_time,
10555        user_defined: version.metadata.clone(),
10556        src_pool_idx,
10557        data_movement: true,
10558        incl_free_versions: version.tier_free_version(),
10559        include_part_checksums: true,
10560        http_preconditions: Some(data_movement::data_movement_target_precondition()),
10561        expected_bucket_incarnation_id,
10562        ..Default::default()
10563    }
10564}
10565
10566fn decommission_capacity_version_mutation_id(
10567    owner: DecommissionCapacityOwner,
10568    bucket: &str,
10569    version: &rustfs_filemeta::FileInfo,
10570) -> uuid::Uuid {
10571    let version_id = if version.deleted && version.version_id.is_none() {
10572        Some(uuid::Uuid::nil().to_string())
10573    } else {
10574        version.version_id.map(|version_id| version_id.to_string())
10575    };
10576    decommission_capacity_mutation_id(owner, bucket, &version.name, version_id.as_deref(), version.deleted, version.mod_time)
10577}
10578
10579fn decommission_capacity_owned_opts(mut opts: ObjectOptions, capacity_owner: Option<DecommissionCapacityOwner>) -> ObjectOptions {
10580    if let Some(capacity_owner) = capacity_owner {
10581        capacity_owner.apply_to(&mut opts);
10582    }
10583    opts
10584}
10585
10586fn lifecycle_action_removes_data_movement_version(action: IlmAction) -> bool {
10587    matches!(
10588        action,
10589        IlmAction::DeleteVersionAction | IlmAction::DeleteAllVersionsAction | IlmAction::DelMarkerDeleteAllVersionsAction
10590    )
10591}
10592
10593fn lifecycle_action_skips_heal_version(action: IlmAction) -> bool {
10594    action.delete()
10595}
10596
10597#[cfg(test)]
10598struct LifecycleDataMovementMutationBarrierState {
10599    bucket: String,
10600    object: String,
10601    arrived: tokio::sync::Notify,
10602    release: tokio::sync::Notify,
10603}
10604
10605#[cfg(test)]
10606pub(crate) struct LifecycleDataMovementMutationBarrier {
10607    state: Arc<LifecycleDataMovementMutationBarrierState>,
10608}
10609
10610#[cfg(test)]
10611static LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER: std::sync::OnceLock<
10612    std::sync::Mutex<Option<Arc<LifecycleDataMovementMutationBarrierState>>>,
10613> = std::sync::OnceLock::new();
10614
10615#[cfg(test)]
10616impl LifecycleDataMovementMutationBarrier {
10617    pub(crate) fn install(bucket: &str, object: &str) -> Self {
10618        let state = Arc::new(LifecycleDataMovementMutationBarrierState {
10619            bucket: bucket.to_string(),
10620            object: object.to_string(),
10621            arrived: tokio::sync::Notify::new(),
10622            release: tokio::sync::Notify::new(),
10623        });
10624        let mut slot = LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER
10625            .get_or_init(|| std::sync::Mutex::new(None))
10626            .lock()
10627            .expect("lifecycle data movement mutation barrier should not be poisoned");
10628        assert!(slot.is_none(), "lifecycle data movement mutation barrier must be unique");
10629        *slot = Some(Arc::clone(&state));
10630        Self { state }
10631    }
10632
10633    pub(crate) async fn wait_until_paused(&self) {
10634        tokio::time::timeout(std::time::Duration::from_secs(30), self.state.arrived.notified())
10635            .await
10636            .expect("lifecycle data movement should reach its mutation boundary");
10637    }
10638
10639    pub(crate) fn release(&self) {
10640        self.state.release.notify_one();
10641    }
10642}
10643
10644#[cfg(test)]
10645impl Drop for LifecycleDataMovementMutationBarrier {
10646    fn drop(&mut self) {
10647        self.state.release.notify_one();
10648        let mut slot = LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER
10649            .get_or_init(|| std::sync::Mutex::new(None))
10650            .lock()
10651            .expect("lifecycle data movement mutation barrier should not be poisoned");
10652        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
10653            *slot = None;
10654        }
10655    }
10656}
10657
10658#[cfg(test)]
10659async fn pause_lifecycle_data_movement_mutation(bucket: &str, object: &str, has_run_fence_signal: bool) {
10660    if !has_run_fence_signal {
10661        return;
10662    }
10663    let barrier = LIFECYCLE_DATA_MOVEMENT_MUTATION_BARRIER
10664        .get_or_init(|| std::sync::Mutex::new(None))
10665        .lock()
10666        .expect("lifecycle data movement mutation barrier should not be poisoned")
10667        .as_ref()
10668        .filter(|barrier| barrier.bucket == bucket && barrier.object == object)
10669        .cloned();
10670    if let Some(barrier) = barrier {
10671        barrier.arrived.notify_one();
10672        barrier.release.notified().await;
10673    }
10674}
10675
10676fn resolve_data_movement_lifecycle_expiry_result(action: IlmAction, apply_actions: bool, applied: bool) -> Result<bool> {
10677    if !apply_actions || applied {
10678        return Ok(true);
10679    }
10680
10681    Err(Error::other(format!(
10682        "failed to apply lifecycle expiry action {action:?} during data movement"
10683    )))
10684}
10685
10686#[allow(clippy::too_many_arguments)]
10687pub(crate) async fn should_skip_lifecycle_for_data_movement(
10688    store: Arc<ECStore>,
10689    bucket: &str,
10690    version: &rustfs_filemeta::FileInfo,
10691    lifecycle_config: Option<&BucketLifecycleConfiguration>,
10692    object_lock_config: Option<&ObjectLockConfiguration>,
10693    apply_actions: bool,
10694    event_source: &LcEventSrc,
10695    lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
10696) -> Result<bool> {
10697    let Some(lifecycle_config) = lifecycle_config else {
10698        return Ok(false);
10699    };
10700
10701    let versioned = match BucketVersioningSys::get_in(&store.ctx, bucket).await {
10702        Ok(config) => config.prefix_enabled(&version.name),
10703        Err(err) => {
10704            warn!(
10705                event = EVENT_DECOMMISSION_ENTRY,
10706                component = LOG_COMPONENT_ECSTORE,
10707                subsystem = LOG_SUBSYSTEM_POOLS,
10708                state = "versioning_config_unavailable",
10709                bucket = %bucket,
10710                error = %err,
10711                "Decommission lifecycle versioning config unavailable; treating object as unversioned"
10712            );
10713            false
10714        }
10715    };
10716    let object_info = crate::object_api::ObjectInfo::from_file_info(version, bucket, &version.name, versioned);
10717    let event = eval_action_from_lifecycle(lifecycle_config, object_lock_config, &object_info).await;
10718
10719    match event.action {
10720        IlmAction::DeleteRestoredAction | IlmAction::DeleteRestoredVersionAction => {
10721            if apply_actions && object_info.is_remote() {
10722                let Ok(bucket_incarnation_id) = store.bucket_incarnation_id_from_disk(bucket).await else {
10723                    return Ok(false);
10724                };
10725                let _ = match lock_lost_signal {
10726                    Some(signal) => {
10727                        apply_expiry_rule_for_data_movement(store, &event, event_source, &object_info, Some(signal)).await
10728                    }
10729                    None => {
10730                        apply_expiry_on_transitioned_object(store, &object_info, &event, event_source, bucket_incarnation_id)
10731                            .await
10732                    }
10733                };
10734            }
10735            Ok(false)
10736        }
10737        action if lifecycle_action_removes_data_movement_version(action) => {
10738            #[cfg(test)]
10739            pause_lifecycle_data_movement_mutation(bucket, &version.name, lock_lost_signal.is_some()).await;
10740            if lifecycle_delete_all_versions_blocked_by_replication(store.clone(), bucket, &object_info.name, action).await? {
10741                return Ok(false);
10742            }
10743            let applied = !apply_actions
10744                || match lock_lost_signal {
10745                    Some(signal) => {
10746                        apply_expiry_rule_for_data_movement(store, &event, event_source, &object_info, Some(signal)).await
10747                    }
10748                    None => apply_expiry_rule_in(store, &event, event_source, &object_info).await,
10749                };
10750            resolve_data_movement_lifecycle_expiry_result(action, apply_actions, applied)
10751        }
10752        _ => Ok(false),
10753    }
10754}
10755
10756pub struct HealLifecycleExpiryContext {
10757    configs: LifecycleExpiryConfigs,
10758}
10759
10760impl ECStore {
10761    pub async fn load_heal_lifecycle_expiry_context(&self, bucket: &str) -> Result<Option<HealLifecycleExpiryContext>> {
10762        if bucket == RUSTFS_META_BUCKET {
10763            return Ok(None);
10764        }
10765
10766        let configs = get_expiry_configs(self, bucket).await?;
10767        if configs.lifecycle.is_none() {
10768            return Ok(None);
10769        }
10770
10771        Ok(Some(HealLifecycleExpiryContext { configs }))
10772    }
10773
10774    pub async fn enqueue_heal_lifecycle_expiry(
10775        self: &Arc<Self>,
10776        context: &HealLifecycleExpiryContext,
10777        bucket: &str,
10778        object: &str,
10779        version_id: Option<&str>,
10780        object_info: Option<&crate::object_api::ObjectInfo>,
10781    ) -> Result<bool> {
10782        let Some(lifecycle_config) = context.configs.lifecycle.as_ref() else {
10783            return Ok(false);
10784        };
10785
10786        let object_info = if let Some(object_info) = object_info {
10787            if object_info.bucket != bucket || object_info.name != object {
10788                return Ok(false);
10789            }
10790            let snapshot_version_id = object_info
10791                .version_id
10792                .filter(|version_id| !version_id.is_nil())
10793                .map(|version_id| version_id.to_string());
10794            if snapshot_version_id.as_deref() != version_id {
10795                return Ok(false);
10796            }
10797            object_info.clone()
10798        } else {
10799            match self
10800                .get_object_info(
10801                    bucket,
10802                    object,
10803                    &ObjectOptions {
10804                        version_id: version_id.map(str::to_string),
10805                        versioned: version_id.is_some(),
10806                        expected_bucket_incarnation_id: Some(context.configs.bucket_incarnation_id),
10807                        ..Default::default()
10808                    },
10809                )
10810                .await
10811            {
10812                Ok(object_info) => object_info,
10813                Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => return Ok(false),
10814                Err(err) => return Err(err),
10815            }
10816        };
10817
10818        let event = eval_action_from_lifecycle(lifecycle_config, context.configs.object_lock.as_deref(), &object_info).await;
10819        if !lifecycle_action_skips_heal_version(event.action) {
10820            return Ok(false);
10821        }
10822
10823        if lifecycle_delete_all_versions_blocked_by_replication(self.clone(), bucket, &object_info.name, event.action).await? {
10824            return Ok(false);
10825        }
10826
10827        Ok(apply_expiry_rule_in(self.clone(), &event, &LcEventSrc::Scanner, &object_info).await)
10828    }
10829
10830    async fn acquire_pool_meta_write_guard(
10831        &self,
10832        write_state: &mut PoolMetaWriteState,
10833        operation: &str,
10834    ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> {
10835        self.acquire_pool_meta_write_guard_with_lock_error(write_state, operation, activation_pool_meta_lock_error)
10836            .await
10837    }
10838
10839    async fn acquire_pool_meta_write_guard_with_lock_error<F>(
10840        &self,
10841        write_state: &mut PoolMetaWriteState,
10842        operation: &str,
10843        map_lock_error: F,
10844    ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)>
10845    where
10846        F: FnOnce(rustfs_lock::LockError) -> Error,
10847    {
10848        write_state.ensure_write_safe(operation)?;
10849        load_pool_meta_identity_observing(self.pools.clone(), write_state).await?;
10850        let pool = self.pools.first().cloned().ok_or_else(|| {
10851            Error::InvalidArgument(
10852                operation.to_string(),
10853                "storage-pools".to_string(),
10854                "no storage pools available".to_string(),
10855            )
10856        })?;
10857        let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
10858        let pool_meta_guard = pool_meta_lock
10859            .get_write_lock(get_lock_acquire_timeout())
10860            .await
10861            .map_err(map_lock_error)?;
10862        let selection = load_pool_meta_replicas_observing(self.pools.clone(), true, write_state).await?;
10863        write_state.observe_replicas(selection.replica_state);
10864        write_state.ensure_write_safe(operation)?;
10865        Ok((pool_meta_guard, selection.meta))
10866    }
10867
10868    async fn acquire_pool_meta_read_guard(
10869        &self,
10870        write_state: &PoolMetaWriteState,
10871        operation: &str,
10872    ) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> {
10873        write_state.ensure_write_safe(operation)?;
10874        let pool = self.pools.first().cloned().ok_or_else(|| {
10875            Error::InvalidArgument(
10876                operation.to_string(),
10877                "storage-pools".to_string(),
10878                "no storage pools available".to_string(),
10879            )
10880        })?;
10881        let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
10882        let pool_meta_guard = pool_meta_lock.get_read_lock(get_lock_acquire_timeout()).await?;
10883        let selection = load_pool_meta_replicas_for_read_probe(self.pools.clone(), true, write_state, operation).await?;
10884        Ok((pool_meta_guard, selection.meta))
10885    }
10886
10887    async fn acquire_decommission_capacity_target_guard(
10888        &self,
10889        target_pool_index: usize,
10890    ) -> Result<rustfs_lock::NamespaceLockGuard> {
10891        // Some reconciliation callers already hold an object lock, while a
10892        // target mutation acquires its object lock after this gate. A short,
10893        // retryable acquisition bounds that inverse-order overlap.
10894        let pool = self.pools.first().cloned().ok_or_else(|| {
10895            Error::InvalidArgument(
10896                "decommission-capacity".to_string(),
10897                "storage-pools".to_string(),
10898                "no storage pools available".to_string(),
10899            )
10900        })?;
10901        let object = format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}");
10902        let target_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
10903        #[cfg(test)]
10904        pause_decommission_capacity_before_target_gate_acquire(self.id, target_pool_index).await;
10905        match target_lock
10906            .get_write_lock_quiet(DECOMMISSION_CAPACITY_TARGET_LOCK_TIMEOUT)
10907            .await
10908        {
10909            Ok(guard) => Ok(guard),
10910            Err(rustfs_lock::LockError::Timeout { .. } | rustfs_lock::LockError::AlreadyLocked { .. }) => {
10911                Err(decommission_capacity_blocked_error(format!(
10912                    "{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}{target_pool_index}{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}"
10913                )))
10914            }
10915            Err(rustfs_lock::LockError::QuorumNotReached { required, achieved }) => Err(Error::NamespaceLockQuorumUnavailable {
10916                mode: "write",
10917                bucket: RUSTFS_META_BUCKET.to_string(),
10918                object,
10919                required,
10920                achieved,
10921            }),
10922            Err(err) => Err(Error::Lock(err)),
10923        }
10924    }
10925
10926    /// Acquire the target pool's capacity mutation gate, absorbing a transient
10927    /// gate-busy result with a small jittered inline retry.
10928    ///
10929    /// The target gate is acquired before the admitted mutation reads or writes
10930    /// any data, so a busy gate guarantees the mutation has not started and can
10931    /// be retried in place. A bounded retry here keeps short contention from
10932    /// escalating an otherwise healthy object migration to a durable pause.
10933    async fn acquire_decommission_capacity_target_guard_retried(
10934        &self,
10935        target_pool_index: usize,
10936    ) -> Result<rustfs_lock::NamespaceLockGuard> {
10937        let mut gate_attempt = 0;
10938        loop {
10939            match self.acquire_decommission_capacity_target_guard(target_pool_index).await {
10940                Ok(guard) => return Ok(guard),
10941                Err(err) if is_decommission_capacity_target_gate_busy(&err) => {
10942                    gate_attempt += 1;
10943                    if gate_attempt >= DECOMMISSION_MUTATION_GATE_MAX_INLINE_ATTEMPTS {
10944                        return Err(err);
10945                    }
10946                    #[cfg(test)]
10947                    notify_decommission_target_gate_inline_retry(self.id);
10948                    record_decommission_capacity_gate_retry(target_pool_index);
10949                    let retry_delay = decommission_capacity_gate_retry_delay(gate_attempt);
10950                    warn!(
10951                        event = EVENT_DECOMMISSION_STATE,
10952                        component = LOG_COMPONENT_ECSTORE,
10953                        subsystem = LOG_SUBSYSTEM_POOLS,
10954                        store_id = %self.id,
10955                        target_pool_index,
10956                        attempt = gate_attempt,
10957                        max_attempts = DECOMMISSION_MUTATION_GATE_MAX_INLINE_ATTEMPTS,
10958                        retry_delay_ms = retry_delay.as_millis(),
10959                        state = "capacity_gate_inline_retry",
10960                        error = %err,
10961                        "Decommission target capacity gate busy; retrying mutation inline"
10962                    );
10963                    tokio::time::sleep(retry_delay).await;
10964                }
10965                Err(err) => return Err(err),
10966            }
10967        }
10968    }
10969
10970    async fn acquire_decommission_capacity_owner_target_guard(
10971        &self,
10972        owner: DecommissionCapacityOwner,
10973        target_pool_index: usize,
10974    ) -> Result<Option<rustfs_lock::NamespaceLockGuard>> {
10975        let model_version = self
10976            .pool_meta
10977            .read()
10978            .await
10979            .pools
10980            .get(owner.source_pool_index)
10981            .and_then(|pool| pool.decommission.as_ref())
10982            .and_then(|info| info.capacity_reservation.as_ref())
10983            .filter(|reservation| {
10984                reservation.active()
10985                    && reservation.source_pool_index == owner.source_pool_index
10986                    && reservation.operation_id == owner.operation_id
10987                    && reservation.generation == owner.generation
10988            })
10989            .map(|reservation| reservation.model_version)
10990            .ok_or_else(|| decommission_capacity_blocked_error("decommission capacity owner is stale"))?;
10991        match model_version {
10992            DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION => Ok(None),
10993            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION => {
10994                discard_decommission_capacity_target_permit_except(self.id, owner, Some(target_pool_index));
10995                match take_decommission_capacity_target_permit(self.id, target_pool_index, owner) {
10996                    Some(guard) => Ok(Some(guard)),
10997                    None => self
10998                        .acquire_decommission_capacity_target_guard(target_pool_index)
10999                        .await
11000                        .map(Some),
11001                }
11002            }
11003            version => Err(Error::DecommissionCapacity(format!(
11004                "decommission capacity owner uses unsupported lock model {version}"
11005            ))),
11006        }
11007    }
11008
11009    async fn acquire_decommission_capacity_terminal_guards(
11010        &self,
11011        source_pool_index: usize,
11012        plan: Option<&DecommissionCapacityTerminalFencePlan>,
11013    ) -> Result<Vec<rustfs_lock::NamespaceLockGuard>> {
11014        let Some(plan) = plan else {
11015            return Ok(Vec::new());
11016        };
11017        match plan.model_version {
11018            DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION => return Ok(Vec::new()),
11019            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION => {}
11020            version => {
11021                return Err(Error::DecommissionCapacity(format!(
11022                    "decommission terminal transition found unsupported capacity lock model {version}"
11023                )));
11024            }
11025        }
11026        // Terminal transitions hold no object or pool metadata lock here, so
11027        // they can wait in target-index order without forming a lock cycle.
11028        let pool = self.pools.first().cloned().ok_or_else(|| {
11029            Error::InvalidArgument(
11030                "decommission-capacity".to_string(),
11031                "storage-pools".to_string(),
11032                "no storage pools available".to_string(),
11033            )
11034        })?;
11035        // Retry only target acquisition, never persistence. Keep the original
11036        // owner/cohort pinned so a remote Clear/start cannot retarget a cancel.
11037        let mut attempt = 1;
11038        'acquire_targets: loop {
11039            let mut guards = Vec::with_capacity(plan.target_pool_indices.len());
11040            for &target_pool_index in &plan.target_pool_indices {
11041                let object = format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/{target_pool_index}");
11042                let target_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
11043                let started = std::time::Instant::now();
11044                match target_lock.get_write_lock(get_lock_acquire_timeout()).await {
11045                    Ok(guard) => guards.push(guard),
11046                    Err(err @ rustfs_lock::LockError::Timeout { .. }) => {
11047                        // Release the entire partial cohort before backoff or
11048                        // metadata reads; workers need these gates to settle I/O.
11049                        drop(guards);
11050                        #[cfg(test)]
11051                        pause_decommission_cancel_target_timeout(self.id).await;
11052                        if attempt >= DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS {
11053                            return Err(Error::Lock(err));
11054                        }
11055                        warn!(
11056                            event = EVENT_DECOMMISSION_STATE,
11057                            component = LOG_COMPONENT_ECSTORE,
11058                            subsystem = LOG_SUBSYSTEM_POOLS,
11059                            state = "cancel_target_fence_retry",
11060                            pool_index = source_pool_index,
11061                            target_pool_index,
11062                            operation_id = %plan.operation_id,
11063                            generation = plan.generation,
11064                            owner_nonce = %plan.owner_nonce,
11065                            attempt,
11066                            max_attempts = DECOMMISSION_CANCEL_TARGET_LOCK_MAX_ATTEMPTS,
11067                            wait_ms = %started.elapsed().as_millis(),
11068                            error = %err,
11069                            "Decommission cancel will retry target capacity fencing"
11070                        );
11071                        tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await;
11072                        let save_guard = self.pool_meta_save_gate.lock().await;
11073                        let (_read_guard, snapshot) = self
11074                            .acquire_pool_meta_read_guard(&save_guard, "decommission cancel fence retry failed")
11075                            .await?;
11076                        if decommission_capacity_terminal_fence_plan(&snapshot, source_pool_index)?.as_ref() != Some(plan) {
11077                            return Err(decommission_capacity_blocked_error(
11078                                "decommission capacity owner or target cohort changed while retrying terminal fences",
11079                            ));
11080                        }
11081                        attempt += 1;
11082                        continue 'acquire_targets;
11083                    }
11084                    Err(rustfs_lock::LockError::QuorumNotReached { required, achieved }) => {
11085                        return Err(Error::NamespaceLockQuorumUnavailable {
11086                            mode: "write",
11087                            bucket: RUSTFS_META_BUCKET.to_string(),
11088                            object,
11089                            required,
11090                            achieved,
11091                        });
11092                    }
11093                    Err(err) => return Err(Error::Lock(err)),
11094                }
11095            }
11096            return Ok(guards);
11097        }
11098    }
11099
11100    pub(crate) async fn acquire_external_decommission_capacity_fence(
11101        &self,
11102        target_pool_indices: &[usize],
11103        admission: DecommissionCapacityAdmission,
11104    ) -> Result<rustfs_lock::NamespaceLockGuard> {
11105        Ok(self
11106            .acquire_external_decommission_capacity_fence_with_active_source(target_pool_indices, admission)
11107            .await?
11108            .0)
11109    }
11110
11111    pub(crate) async fn acquire_external_decommission_capacity_fence_with_active_source(
11112        &self,
11113        target_pool_indices: &[usize],
11114        admission: DecommissionCapacityAdmission,
11115    ) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
11116        let save_guard = self.pool_meta_save_gate.lock().await;
11117        let (pool_meta_guard, snapshot) = self
11118            .acquire_pool_meta_read_guard(&save_guard, "target capacity admission failed")
11119            .await?;
11120        for target_pool_index in target_pool_indices.iter().copied() {
11121            ensure_external_decommission_target_admission(&snapshot, target_pool_index, admission)?;
11122        }
11123        let has_active_source = pool_meta_has_active_decommission(&snapshot);
11124        drop(save_guard);
11125        Ok((pool_meta_guard, has_active_source))
11126    }
11127
11128    /// Fence healing of the pool metadata object itself without recursively
11129    /// acquiring its namespace lock through the ordinary capacity probe. The
11130    /// caller must retain the returned write guard through every admitted
11131    /// repair; admission results preserve the input target order.
11132    pub(crate) async fn acquire_pool_meta_object_heal_fence(
11133        &self,
11134        target_pool_indices: &[usize],
11135    ) -> Result<(rustfs_lock::NamespaceLockGuard, Vec<Result<()>>)> {
11136        let mut save_guard = self.pool_meta_save_gate.lock().await;
11137        let (pool_meta_guard, snapshot) = self
11138            .acquire_pool_meta_write_guard_with_lock_error(&mut save_guard, "pool metadata heal admission failed", Error::from)
11139            .await?;
11140        let admissions = target_pool_indices
11141            .iter()
11142            .copied()
11143            .map(|target_pool_index| {
11144                ensure_external_decommission_target_admission(&snapshot, target_pool_index, DecommissionCapacityAdmission::Heal)
11145            })
11146            .collect();
11147        drop(save_guard);
11148        Ok((pool_meta_guard, admissions))
11149    }
11150
11151    pub(crate) async fn acquire_decommission_capacity_release_fence_with_active_source(
11152        &self,
11153    ) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
11154        let save_guard = self.pool_meta_save_gate.lock().await;
11155        let (pool_meta_guard, snapshot) = self
11156            .acquire_pool_meta_read_guard(&save_guard, "capacity release fence failed")
11157            .await?;
11158        let has_active_source = pool_meta_has_active_decommission(&snapshot);
11159        drop(save_guard);
11160        Ok((pool_meta_guard, has_active_source))
11161    }
11162
11163    pub(crate) async fn run_decommission_capacity_admitted_mutation<T, F, Fut>(
11164        &self,
11165        target_pool_index: usize,
11166        capacity_owner: Option<DecommissionCapacityOwner>,
11167        expected_data_bytes: Option<usize>,
11168        operation: F,
11169    ) -> Result<T>
11170    where
11171        F: FnOnce() -> Fut,
11172        Fut: std::future::Future<Output = Result<T>>,
11173    {
11174        self.run_decommission_capacity_mutation(
11175            target_pool_index,
11176            capacity_owner,
11177            expected_data_bytes,
11178            DecommissionCapacityMutationMode::Durable,
11179            |_| false,
11180            |_| operation(),
11181        )
11182        .await
11183    }
11184
11185    pub(crate) async fn run_decommission_capacity_admitted_mutation_with_capacity_lease<T, F, Fut>(
11186        &self,
11187        target_pool_index: usize,
11188        capacity_owner: Option<DecommissionCapacityOwner>,
11189        expected_data_bytes: Option<usize>,
11190        operation: F,
11191    ) -> Result<T>
11192    where
11193        F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11194        Fut: std::future::Future<Output = Result<T>>,
11195    {
11196        self.run_decommission_capacity_mutation(
11197            target_pool_index,
11198            capacity_owner,
11199            expected_data_bytes,
11200            DecommissionCapacityMutationMode::Durable,
11201            |_| false,
11202            operation,
11203        )
11204        .await
11205    }
11206
11207    pub(crate) async fn reconcile_decommission_capacity_before_exact_delete(
11208        &self,
11209        bucket: &str,
11210        object: &str,
11211        opts: &ObjectOptions,
11212        exact: &ObjectInfo,
11213    ) -> Result<()> {
11214        if exact.bucket != bucket || exact.name != decode_dir_object(object) {
11215            return Err(decommission_capacity_blocked_error(
11216                "exact delete object identity changed before capacity reconciliation",
11217            ));
11218        }
11219
11220        let (reconciliations, model_version) = {
11221            let save_guard = self.pool_meta_save_gate.lock().await;
11222            let (_read_guard, snapshot) = self
11223                .acquire_pool_meta_read_guard(&save_guard, "exact delete capacity reconciliation failed")
11224                .await?;
11225            let reconciliations = plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)?;
11226            let model_version = active_decommission_capacity_model(&snapshot)?;
11227            (reconciliations, model_version)
11228        };
11229        if reconciliations.is_empty() {
11230            return Ok(());
11231        }
11232        ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
11233
11234        let mut target_guards = Vec::new();
11235        let mut legacy_write_fence = None;
11236        match model_version {
11237            Some(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION) => {
11238                let mut save_guard = self.pool_meta_save_gate.lock().await;
11239                let (write_guard, snapshot) = self
11240                    .acquire_pool_meta_write_guard(&mut save_guard, "exact delete capacity reconciliation failed")
11241                    .await?;
11242                if active_decommission_capacity_model(&snapshot)? != model_version
11243                    || plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)? != reconciliations
11244                {
11245                    return Err(decommission_capacity_blocked_error(
11246                        "pending capacity changed before exact target evidence could be verified",
11247                    ));
11248                }
11249                legacy_write_fence = Some((save_guard, write_guard, snapshot));
11250            }
11251            Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION) => {
11252                let mut target_pool_indices = reconciliations
11253                    .iter()
11254                    .map(|reconciliation| reconciliation.target_pool_index)
11255                    .collect::<Vec<_>>();
11256                target_pool_indices.sort_unstable();
11257                target_pool_indices.dedup();
11258                target_guards.reserve(target_pool_indices.len());
11259                for target_pool_index in target_pool_indices {
11260                    let guard = self.acquire_decommission_capacity_target_guard(target_pool_index).await?;
11261                    ensure_decommission_capacity_target_fence(&guard, target_pool_index, "exact delete evidence")?;
11262                    target_guards.push((target_pool_index, guard));
11263                }
11264            }
11265            Some(version) => {
11266                return Err(Error::DecommissionCapacity(format!(
11267                    "exact delete capacity reconciliation found unsupported lock model {version}"
11268                )));
11269            }
11270            None => {
11271                return Err(decommission_capacity_blocked_error(
11272                    "exact delete capacity reconciliation has no active reservation lock model",
11273                ));
11274            }
11275        }
11276
11277        let target_lookup_options = ObjectOptions {
11278            versioned: opts.versioned,
11279            version_suspended: opts.version_suspended,
11280            version_id: opts.version_id.clone(),
11281            metadata_chg: opts.version_id.is_some(),
11282            no_lock: true,
11283            ..Default::default()
11284        };
11285        for reconciliation in &reconciliations {
11286            let target_pool = self.pools.get(reconciliation.target_pool_index).ok_or_else(|| {
11287                decommission_capacity_blocked_error(format!(
11288                    "source pool {} exact-delete capacity target pool {} is out of range",
11289                    reconciliation.source_pool_index, reconciliation.target_pool_index
11290                ))
11291            })?;
11292            let target = target_pool
11293                .get_object_info(bucket, object, &target_lookup_options)
11294                .await
11295                .map_err(|err| {
11296                    decommission_capacity_blocked_error(format!(
11297                        "source pool {} target pool {} exact object evidence could not be read: {err}",
11298                        reconciliation.source_pool_index, reconciliation.target_pool_index
11299                    ))
11300                })?;
11301            if !Self::is_equivalent_decommission_capacity_target(exact, &target) {
11302                return Err(decommission_capacity_blocked_error(format!(
11303                    "source pool {} target pool {} does not contain an equivalent exact object for its pending capacity intent",
11304                    reconciliation.source_pool_index, reconciliation.target_pool_index
11305                )));
11306            }
11307        }
11308        ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
11309
11310        let (mut save_guard, write_guard, mut snapshot) = if let Some(fence) = legacy_write_fence {
11311            fence
11312        } else {
11313            let mut save_guard = self.pool_meta_save_gate.lock().await;
11314            let (write_guard, snapshot) = self
11315                .acquire_pool_meta_write_guard(&mut save_guard, "exact delete capacity reconciliation failed")
11316                .await?;
11317            (save_guard, write_guard, snapshot)
11318        };
11319        if active_decommission_capacity_model(&snapshot)? != model_version {
11320            return Err(decommission_capacity_blocked_error(
11321                "capacity lock model changed while exact target evidence was being verified",
11322            ));
11323        }
11324        let current_reconciliations = plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)?;
11325        if current_reconciliations != reconciliations {
11326            return Err(decommission_capacity_blocked_error(
11327                "pending capacity changed while exact target evidence was being verified",
11328            ));
11329        }
11330        ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
11331        for (target_pool_index, target_guard) in &target_guards {
11332            ensure_decommission_capacity_target_fence(target_guard, *target_pool_index, "exact delete capacity finalize")?;
11333        }
11334
11335        let now = OffsetDateTime::now_utc();
11336        let mut source_pool_indices = Vec::with_capacity(current_reconciliations.len());
11337        for reconciliation in current_reconciliations {
11338            resolve_decommission_target_pending(
11339                &mut snapshot,
11340                reconciliation.source_pool_index,
11341                reconciliation.target_pool_index,
11342                reconciliation.expected_target_physical_bytes,
11343                reconciliation.mutation_id,
11344            )?;
11345            record_decommission_target_consumption(
11346                &mut snapshot,
11347                reconciliation.source_pool_index,
11348                reconciliation.target_pool_index,
11349                DecommissionTargetConsumption {
11350                    committed_data_bytes: reconciliation.expected_data_bytes,
11351                    target_physical_bytes: reconciliation.expected_target_physical_bytes,
11352                    observed_physical_bytes: 0,
11353                },
11354                reconciliation.mutation_id,
11355                now,
11356            )?;
11357            source_pool_indices.push(reconciliation.source_pool_index);
11358        }
11359        source_pool_indices.sort_unstable();
11360        source_pool_indices.dedup();
11361        ensure_exact_delete_capacity_namespace_fences(opts, bucket, object)?;
11362
11363        let outcome = snapshot
11364            .save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &source_pool_indices)
11365            .await?;
11366        ensure_pool_meta_write_fence(&write_guard, "exact delete capacity reconciliation save failed")?;
11367        for (target_pool_index, target_guard) in &target_guards {
11368            ensure_decommission_capacity_target_fence(target_guard, *target_pool_index, "exact delete capacity save")?;
11369        }
11370        {
11371            let mut pool_meta = self.pool_meta.write().await;
11372            for source_pool_index in source_pool_indices {
11373                publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, source_pool_index)?;
11374            }
11375        }
11376        ensure_pool_meta_write_fence(&write_guard, "exact delete capacity reconciliation save failed")?;
11377        for (target_pool_index, target_guard) in &target_guards {
11378            ensure_decommission_capacity_target_fence(target_guard, *target_pool_index, "exact delete capacity publication")?;
11379        }
11380        outcome.disarm();
11381        Ok(())
11382    }
11383
11384    pub(crate) async fn reconcile_decommission_capacity_after_equivalent_target(
11385        &self,
11386        owner: DecommissionCapacityOwner,
11387        target_pool_index: usize,
11388        expected_data_bytes: usize,
11389    ) -> Result<()> {
11390        let target_guard = self
11391            .acquire_decommission_capacity_owner_target_guard(owner, target_pool_index)
11392            .await?;
11393        if let Some(target_guard) = target_guard.as_ref() {
11394            ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target reconciliation")?;
11395        }
11396        let mut save_guard = self.pool_meta_save_gate.lock().await;
11397        let (pool_meta_guard, mut snapshot) = self
11398            .acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent target reconciliation failed")
11399            .await?;
11400        let source_pool_index = owner.source_pool_index;
11401        let (target_layout, target_pending_physical_bytes, target_consumed_physical_bytes) = {
11402            let reservation = snapshot
11403                .pools
11404                .get(source_pool_index)
11405                .and_then(|pool| pool.decommission.as_ref())
11406                .and_then(|info| info.capacity_reservation.as_ref())
11407                .filter(|reservation| {
11408                    reservation.active()
11409                        && reservation.source_pool_index == owner.source_pool_index
11410                        && reservation.operation_id == owner.operation_id
11411                        && reservation.generation == owner.generation
11412                        && reservation.owner_nonce == owner.owner_nonce
11413                })
11414                .ok_or_else(|| decommission_capacity_blocked_error("equivalent target reconciliation owner is stale"))?;
11415            let target = reservation
11416                .targets
11417                .iter()
11418                .find(|target| target.pool_index == target_pool_index)
11419                .ok_or_else(|| decommission_capacity_blocked_error("equivalent target reconciliation allocation is missing"))?;
11420            (target.layout, target.pending_physical_bytes, target.consumed_physical_bytes)
11421        };
11422        let mutation_id = owner.mutation_id.ok_or_else(|| {
11423            decommission_capacity_blocked_error("equivalent target reconciliation mutation identity is missing")
11424        })?;
11425        let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
11426        if target_pending_physical_bytes == 0 {
11427            if target_consumed_physical_bytes >= expected_target_physical_bytes {
11428                let mut pool_meta = self.pool_meta.write().await;
11429                ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent target idempotent publication")?;
11430                publish_decommission_capacity_update(&mut pool_meta, &snapshot, source_pool_index)?;
11431                if let Some(target_guard) = target_guard.as_ref() {
11432                    ensure_decommission_capacity_target_fence(
11433                        target_guard,
11434                        target_pool_index,
11435                        "equivalent target idempotent publication",
11436                    )?;
11437                }
11438                return Ok(());
11439            }
11440            return Err(decommission_capacity_blocked_error(
11441                "equivalent target has no pending capacity intent to reconcile",
11442            ));
11443        }
11444        if target_pending_physical_bytes < expected_target_physical_bytes {
11445            return Err(decommission_capacity_blocked_error(
11446                "equivalent target pending capacity is smaller than the committed object",
11447            ));
11448        }
11449
11450        resolve_decommission_target_pending(
11451            &mut snapshot,
11452            source_pool_index,
11453            target_pool_index,
11454            expected_target_physical_bytes,
11455            mutation_id,
11456        )?;
11457        record_decommission_target_consumption(
11458            &mut snapshot,
11459            source_pool_index,
11460            target_pool_index,
11461            DecommissionTargetConsumption {
11462                committed_data_bytes: expected_data_bytes,
11463                target_physical_bytes: expected_target_physical_bytes,
11464                observed_physical_bytes: 0,
11465            },
11466            mutation_id,
11467            OffsetDateTime::now_utc(),
11468        )?;
11469        let outcome = snapshot
11470            .save_no_lock_armed(
11471                self.pools.clone(),
11472                &mut save_guard,
11473                pool_meta_guard.lock_lost_signal(),
11474                &[source_pool_index],
11475            )
11476            .await?;
11477        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission equivalent target reconciliation save failed")?;
11478        if let Some(target_guard) = target_guard.as_ref() {
11479            ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target capacity save")?;
11480        }
11481        {
11482            let mut pool_meta = self.pool_meta.write().await;
11483            publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, source_pool_index)?;
11484        }
11485        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission equivalent target reconciliation save failed")?;
11486        if let Some(target_guard) = target_guard.as_ref() {
11487            ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "equivalent target capacity publication")?;
11488        }
11489        outcome.disarm();
11490        Ok(())
11491    }
11492
11493    pub(crate) async fn run_decommission_capacity_temporary_mutation<T, F, Fut>(
11494        &self,
11495        target_pool_index: usize,
11496        capacity_owner: Option<DecommissionCapacityOwner>,
11497        expected_data_bytes: Option<usize>,
11498        operation: F,
11499    ) -> Result<T>
11500    where
11501        F: FnOnce() -> Fut,
11502        Fut: std::future::Future<Output = Result<T>>,
11503    {
11504        self.run_decommission_capacity_mutation(
11505            target_pool_index,
11506            capacity_owner,
11507            expected_data_bytes,
11508            DecommissionCapacityMutationMode::Temporary,
11509            |_| false,
11510            |_| operation(),
11511        )
11512        .await
11513    }
11514
11515    pub(crate) async fn run_decommission_capacity_temporary_mutation_with_capacity_lease<T, F, Fut>(
11516        &self,
11517        target_pool_index: usize,
11518        capacity_owner: Option<DecommissionCapacityOwner>,
11519        expected_data_bytes: Option<usize>,
11520        operation: F,
11521    ) -> Result<T>
11522    where
11523        F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11524        Fut: std::future::Future<Output = Result<T>>,
11525    {
11526        self.run_decommission_capacity_mutation(
11527            target_pool_index,
11528            capacity_owner,
11529            expected_data_bytes,
11530            DecommissionCapacityMutationMode::Temporary,
11531            |_| false,
11532            operation,
11533        )
11534        .await
11535    }
11536
11537    /// Run an identity-preserving replacement that the caller has already
11538    /// proven cannot grow the target object. A failed or ambiguous write keeps
11539    /// the ordinary temporary-mutation recovery marker, while a successful
11540    /// write resolves the capacity intent without retaining MPU cleanup state.
11541    pub(crate) async fn run_decommission_capacity_non_growing_replacement_with_capacity_lease<T, F, Fut>(
11542        &self,
11543        target_pool_index: usize,
11544        capacity_owner: Option<DecommissionCapacityOwner>,
11545        expected_data_bytes: Option<usize>,
11546        operation: F,
11547    ) -> Result<T>
11548    where
11549        F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11550        Fut: std::future::Future<Output = Result<T>>,
11551    {
11552        self.run_decommission_capacity_mutation(
11553            target_pool_index,
11554            capacity_owner,
11555            expected_data_bytes,
11556            DecommissionCapacityMutationMode::NonGrowingReplacement,
11557            |_| false,
11558            operation,
11559        )
11560        .await
11561    }
11562
11563    /// Finish the capacity transaction for an identity-preserving temporary
11564    /// replacement whose target bytes are already durably present. This is
11565    /// the crash-recovery half of a non-growing replacement: it never writes
11566    /// the target again and only settles state owned by the exact deterministic
11567    /// mutation id.
11568    pub(crate) async fn reconcile_decommission_capacity_after_equivalent_temporary_target(
11569        &self,
11570        owner: DecommissionCapacityOwner,
11571        target_pool_index: usize,
11572        expected_data_bytes: usize,
11573    ) -> Result<()> {
11574        let target_guard = self
11575            .acquire_decommission_capacity_owner_target_guard(owner, target_pool_index)
11576            .await?;
11577        if let Some(target_guard) = target_guard.as_ref() {
11578            ensure_decommission_capacity_target_fence(
11579                target_guard,
11580                target_pool_index,
11581                "equivalent temporary target reconciliation",
11582            )?;
11583        }
11584        let mut save_guard = self.pool_meta_save_gate.lock().await;
11585        let (pool_meta_guard, mut snapshot) = self
11586            .acquire_pool_meta_write_guard(&mut save_guard, "decommission equivalent temporary target reconciliation failed")
11587            .await?;
11588        let source_pool_index = owner.source_pool_index;
11589        let mutation_id = owner
11590            .mutation_id
11591            .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target mutation identity is missing"))?;
11592        let (target_layout, pending_physical_bytes, pending_mutation_id, has_temporary_mutation) = {
11593            let reservation = snapshot
11594                .pools
11595                .get(source_pool_index)
11596                .and_then(|pool| pool.decommission.as_ref())
11597                .and_then(|info| info.capacity_reservation.as_ref())
11598                .filter(|reservation| reservation.admits_cleanup_owner(owner))
11599                .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target owner is stale"))?;
11600            let target = reservation
11601                .targets
11602                .iter()
11603                .find(|target| target.pool_index == target_pool_index)
11604                .ok_or_else(|| decommission_capacity_blocked_error("equivalent temporary target allocation is missing"))?;
11605            (
11606                target.layout,
11607                target.pending_physical_bytes,
11608                target.pending_mutation_id,
11609                target
11610                    .temporary_mutations
11611                    .iter()
11612                    .any(|mutation| mutation.mutation_id == mutation_id),
11613            )
11614        };
11615        if pending_physical_bytes == 0 && !has_temporary_mutation {
11616            // A prior successful attempt already settled both durable halves.
11617            ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation fence failed")?;
11618            if let Some(target_guard) = target_guard.as_ref() {
11619                ensure_decommission_capacity_target_fence(
11620                    target_guard,
11621                    target_pool_index,
11622                    "equivalent temporary target idempotent reconciliation",
11623                )?;
11624            }
11625            return Ok(());
11626        }
11627        if pending_physical_bytes > 0 && pending_mutation_id != Some(mutation_id) {
11628            return Err(decommission_capacity_blocked_error(
11629                "equivalent temporary target pending intent belongs to another mutation",
11630            ));
11631        }
11632        let expected_target_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target_layout)?;
11633        if pending_physical_bytes > 0 && pending_physical_bytes != expected_target_physical_bytes {
11634            return Err(decommission_capacity_blocked_error(
11635                "equivalent temporary target pending capacity does not match the committed checkpoint",
11636            ));
11637        }
11638        if pending_physical_bytes > 0 {
11639            resolve_decommission_target_pending(
11640                &mut snapshot,
11641                source_pool_index,
11642                target_pool_index,
11643                expected_target_physical_bytes,
11644                mutation_id,
11645            )?;
11646        }
11647        // Exact byte equivalence proves that this identity-preserving,
11648        // byte-non-growing replacement committed. Settle both crash windows:
11649        // the pending intent before finalize and the temporary marker written
11650        // when the operation returned an error after committing its target.
11651        settle_decommission_target_non_growing_replacement(
11652            &mut snapshot,
11653            source_pool_index,
11654            target_pool_index,
11655            mutation_id,
11656            OffsetDateTime::now_utc(),
11657        )?;
11658        let outcome = snapshot
11659            .save_no_lock_armed(
11660                self.pools.clone(),
11661                &mut save_guard,
11662                pool_meta_guard.lock_lost_signal(),
11663                &[source_pool_index],
11664            )
11665            .await?;
11666        ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?;
11667        if let Some(target_guard) = target_guard.as_ref() {
11668            ensure_decommission_capacity_target_fence(
11669                target_guard,
11670                target_pool_index,
11671                "equivalent temporary target capacity save",
11672            )?;
11673        }
11674        {
11675            let mut pool_meta = self.pool_meta.write().await;
11676            publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, source_pool_index)?;
11677        }
11678        ensure_pool_meta_write_fence(&pool_meta_guard, "equivalent temporary target reconciliation save failed")?;
11679        if let Some(target_guard) = target_guard.as_ref() {
11680            ensure_decommission_capacity_target_fence(
11681                target_guard,
11682                target_pool_index,
11683                "equivalent temporary target capacity publication",
11684            )?;
11685        }
11686        outcome.disarm();
11687        Ok(())
11688    }
11689
11690    pub(crate) async fn has_decommission_capacity_temporary_mutation_state(
11691        &self,
11692        target_pool_index: usize,
11693        owner: DecommissionCapacityOwner,
11694    ) -> bool {
11695        let Some(mutation_id) = owner.mutation_id else {
11696            return false;
11697        };
11698        self.pool_meta
11699            .read()
11700            .await
11701            .pools
11702            .get(owner.source_pool_index)
11703            .and_then(|pool| pool.decommission.as_ref())
11704            .and_then(|info| info.capacity_reservation.as_ref())
11705            .filter(|reservation| reservation.admits_cleanup_owner(owner))
11706            .and_then(|reservation| {
11707                reservation
11708                    .targets
11709                    .iter()
11710                    .find(|target| target.pool_index == target_pool_index)
11711                    .map(|target| (reservation.model_version, target))
11712            })
11713            .is_some_and(|(model_version, target)| {
11714                if model_version == DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION {
11715                    // V1 cannot persist an exact zero-byte staging marker, so
11716                    // retain its compatibility scan for admitted cleanup targets.
11717                    true
11718                } else {
11719                    (target.pending_physical_bytes > 0 && target.pending_mutation_id == Some(mutation_id))
11720                        || target
11721                            .temporary_mutations
11722                            .iter()
11723                            .any(|mutation| mutation.mutation_id == mutation_id)
11724                }
11725            })
11726    }
11727
11728    pub(crate) async fn decommission_capacity_cleanup_target_indices(
11729        &self,
11730        owner: DecommissionCapacityOwner,
11731    ) -> Result<Vec<usize>> {
11732        self.pool_meta
11733            .read()
11734            .await
11735            .pools
11736            .get(owner.source_pool_index)
11737            .and_then(|pool| pool.decommission.as_ref())
11738            .and_then(|info| info.capacity_reservation.as_ref())
11739            .filter(|reservation| reservation.admits_cleanup_owner(owner))
11740            .map(|reservation| reservation.targets.iter().map(|target| target.pool_index).collect())
11741            .ok_or_else(|| decommission_capacity_blocked_error("decommission multipart cleanup reservation is stale"))
11742    }
11743
11744    pub(crate) async fn run_decommission_capacity_temporary_release_with_capacity_lease<T, F, Fut>(
11745        &self,
11746        target_pool_index: usize,
11747        capacity_owner: Option<DecommissionCapacityOwner>,
11748        operation: F,
11749    ) -> Result<T>
11750    where
11751        F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11752        Fut: std::future::Future<Output = Result<(T, bool)>>,
11753    {
11754        self.run_decommission_capacity_mutation(
11755            target_pool_index,
11756            capacity_owner,
11757            None,
11758            DecommissionCapacityMutationMode::TemporaryRelease,
11759            |result: &(T, bool)| result.1,
11760            operation,
11761        )
11762        .await
11763        .map(|(result, _)| result)
11764    }
11765
11766    async fn run_decommission_capacity_mutation<T, F, Fut, P>(
11767        &self,
11768        target_pool_index: usize,
11769        capacity_owner: Option<DecommissionCapacityOwner>,
11770        expected_data_bytes: Option<usize>,
11771        mode: DecommissionCapacityMutationMode,
11772        clear_pending_on_temporary_release: P,
11773        operation: F,
11774    ) -> Result<T>
11775    where
11776        F: FnOnce(Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> Fut,
11777        Fut: std::future::Future<Output = Result<T>>,
11778        P: Fn(&T) -> bool,
11779    {
11780        let temporary = matches!(
11781            mode,
11782            DecommissionCapacityMutationMode::Temporary | DecommissionCapacityMutationMode::NonGrowingReplacement
11783        );
11784        let non_growing_replacement = matches!(mode, DecommissionCapacityMutationMode::NonGrowingReplacement);
11785        let temporary_release = matches!(mode, DecommissionCapacityMutationMode::TemporaryRelease);
11786        let mut operation = Some(operation);
11787        let save_guard = self.pool_meta_save_gate.lock().await;
11788        let (read_guard, snapshot) = self
11789            .acquire_pool_meta_read_guard(&save_guard, "target capacity admission failed")
11790            .await?;
11791        let admission_now = OffsetDateTime::now_utc();
11792        let admitted_owner = capacity_owner.and_then(|owner| {
11793            snapshot
11794                .pools
11795                .get(owner.source_pool_index)
11796                .and_then(|pool| pool.decommission.as_ref())
11797                .and_then(|info| info.capacity_reservation.as_ref())
11798                .filter(|reservation| {
11799                    if temporary_release {
11800                        reservation.admits_cleanup_owner(owner)
11801                    } else {
11802                        reservation.admits_owner(owner, admission_now)
11803                    }
11804                })
11805                .filter(|reservation| {
11806                    !temporary_release
11807                        || reservation
11808                            .targets
11809                            .iter()
11810                            .any(|target| target.pool_index == target_pool_index)
11811                })
11812                .map(|reservation| (owner, reservation.model_version))
11813        });
11814        if capacity_owner.is_some() && admitted_owner.is_none() {
11815            return Err(decommission_capacity_blocked_error(
11816                "decommission target mutation reservation identity is stale",
11817            ));
11818        }
11819        let Some((owner, model_version)) = admitted_owner else {
11820            ensure_external_decommission_target_admission(&snapshot, target_pool_index, DecommissionCapacityAdmission::Mutation)?;
11821            drop(save_guard);
11822            let capacity_lease = read_guard.lock_lost_signal();
11823            return operation.take().expect("capacity-admitted operation should run once")(capacity_lease).await;
11824        };
11825
11826        // Per-target reservations always acquire target -> pool metadata.
11827        // Legacy reservations keep the pool metadata write guard through I/O.
11828        drop(read_guard);
11829        drop(save_guard);
11830        if model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION {
11831            discard_decommission_capacity_target_permit_except(self.id, owner, Some(target_pool_index));
11832        }
11833        let target_guard = match model_version {
11834            DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION => None,
11835            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION => {
11836                Some(match take_decommission_capacity_target_permit(self.id, target_pool_index, owner) {
11837                    Some(guard) => guard,
11838                    None => {
11839                        self.acquire_decommission_capacity_target_guard_retried(target_pool_index)
11840                            .await?
11841                    }
11842                })
11843            }
11844            version => {
11845                return Err(Error::DecommissionCapacity(format!(
11846                    "decommission target mutation found unsupported capacity lock model {version}"
11847                )));
11848            }
11849        };
11850
11851        #[cfg(test)]
11852        pause_decommission_capacity_before_owner_write(self.id).await;
11853        let mut save_guard = self.pool_meta_save_gate.lock().await;
11854        let (write_guard, mut snapshot) = self
11855            .acquire_pool_meta_write_guard(&mut save_guard, "decommission target capacity admission failed")
11856            .await?;
11857        let mutation_id = owner
11858            .mutation_id
11859            .ok_or_else(|| decommission_capacity_blocked_error("decommission mutation identity is missing"))?;
11860        let source_pool_index = owner.source_pool_index;
11861        let owner_current = snapshot
11862            .pools
11863            .get(source_pool_index)
11864            .and_then(|pool| pool.decommission.as_ref())
11865            .and_then(|info| info.capacity_reservation.as_ref())
11866            .filter(|reservation| {
11867                (if temporary_release {
11868                    reservation.admits_cleanup_owner(owner)
11869                } else {
11870                    reservation.admits_owner(owner, OffsetDateTime::now_utc())
11871                }) && reservation.model_version == model_version
11872                    && (!temporary_release
11873                        || reservation
11874                            .targets
11875                            .iter()
11876                            .any(|target| target.pool_index == target_pool_index))
11877            })
11878            .is_some();
11879        if !owner_current {
11880            return Err(decommission_capacity_blocked_error(
11881                "decommission target mutation reservation identity changed before commit",
11882            ));
11883        }
11884        let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
11885        let reservation_grew = if temporary_release {
11886            false
11887        } else {
11888            let data_bytes = expected_data_bytes.unwrap_or(1).max(1);
11889            let grew = grow_decommission_target_reservation(
11890                &mut snapshot,
11891                &capacity_infos,
11892                source_pool_index,
11893                target_pool_index,
11894                data_bytes,
11895                mutation_id,
11896                OffsetDateTime::now_utc(),
11897            )?;
11898            ensure_decommission_capacity_reservations_available(&snapshot, &capacity_infos, "mutation")?;
11899            grew
11900        };
11901        let target_layout = snapshot
11902            .pools
11903            .get(source_pool_index)
11904            .and_then(|pool| pool.decommission.as_ref())
11905            .and_then(|info| info.capacity_reservation.as_ref())
11906            .and_then(|reservation| {
11907                reservation
11908                    .targets
11909                    .iter()
11910                    .find(|target| target.pool_index == target_pool_index)
11911            })
11912            .map(|target| target.layout)
11913            .ok_or_else(|| Error::SlowDown)?;
11914        let expected_target_physical_bytes = if temporary_release {
11915            0
11916        } else {
11917            capacity_target_physical_bytes(expected_data_bytes.unwrap_or(1).max(1), target_layout)?
11918        };
11919        if temporary {
11920            let (remaining, inflight) = snapshot
11921                .pools
11922                .get(source_pool_index)
11923                .and_then(|pool| pool.decommission.as_ref())
11924                .and_then(|info| info.capacity_reservation.as_ref())
11925                .and_then(|reservation| {
11926                    reservation
11927                        .targets
11928                        .iter()
11929                        .find(|target| target.pool_index == target_pool_index)
11930                        .map(|target| {
11931                            (
11932                                target.remaining_reserved_physical_bytes(reservation.temporary_copies)
11933                                    / 1usize.saturating_add(reservation.temporary_copies),
11934                                target.inflight_physical_bytes,
11935                            )
11936                        })
11937                })
11938                .unwrap_or_default();
11939            let available = remaining.saturating_sub(inflight);
11940            if expected_target_physical_bytes > available {
11941                return Err(decommission_capacity_blocked_error(format!(
11942                    "source pool {source_pool_index} target pool {target_pool_index} temporary operation requires {expected_target_physical_bytes} physical bytes, but only {available} temporary-copy bytes remain"
11943                )));
11944            }
11945        } else {
11946            ensure_decommission_target_owner_admission(
11947                &snapshot,
11948                owner,
11949                target_pool_index,
11950                expected_target_physical_bytes,
11951                OffsetDateTime::now_utc(),
11952            )?;
11953        }
11954        let pending_added = if temporary_release {
11955            0
11956        } else {
11957            reserve_decommission_target_pending(
11958                &mut snapshot,
11959                source_pool_index,
11960                target_pool_index,
11961                expected_target_physical_bytes,
11962                mutation_id,
11963                OffsetDateTime::now_utc(),
11964            )?
11965        };
11966        if reservation_grew || pending_added > 0 {
11967            let outcome = snapshot
11968                .save_no_lock_armed(self.pools.clone(), &mut save_guard, write_guard.lock_lost_signal(), &[source_pool_index])
11969                .await?;
11970            ensure_pool_meta_write_fence(&write_guard, "decommission target capacity intent save failed")?;
11971            snapshot = outcome.committed.clone();
11972            {
11973                let mut pool_meta = self.pool_meta.write().await;
11974                publish_decommission_capacity_update(&mut pool_meta, &snapshot, source_pool_index)?;
11975            }
11976            ensure_pool_meta_write_fence(&write_guard, "decommission target capacity intent save failed")?;
11977            outcome.disarm();
11978        }
11979        let mut write_guard = Some(write_guard);
11980        let mut save_guard = Some(save_guard);
11981        let capacity_lease = if let Some(target_guard) = target_guard.as_ref() {
11982            ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity intent prepare")?;
11983            let capacity_lease = target_guard.lock_lost_signal();
11984            drop(write_guard.take());
11985            drop(save_guard.take());
11986            capacity_lease
11987        } else {
11988            write_guard
11989                .as_ref()
11990                .ok_or_else(|| Error::other("legacy decommission capacity fence disappeared before target mutation"))?
11991                .lock_lost_signal()
11992        };
11993
11994        let capacity_infos = if pending_added > 0 {
11995            self.get_decommission_all_pool_capacity_infos().await?
11996        } else {
11997            capacity_infos
11998        };
11999        let before_free = capacity_infos
12000            .iter()
12001            .find(|capacity| capacity.pool_index == target_pool_index)
12002            .map(|capacity| capacity.physical_free)
12003            .ok_or_else(|| decommission_capacity_blocked_error("target capacity snapshot is missing before mutation"))?;
12004
12005        let result = operation.take().expect("capacity-admitted operation should run once")(capacity_lease).await;
12006        let clear_pending_on_temporary_release = result.as_ref().ok().is_some_and(&clear_pending_on_temporary_release);
12007        if let Some(target_guard) = target_guard.as_ref() {
12008            ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "target mutation")?;
12009            let mut finalize_save_guard = self.pool_meta_save_gate.lock().await;
12010            let (finalize_write_guard, finalize_snapshot) = self
12011                .acquire_pool_meta_write_guard(&mut finalize_save_guard, "decommission target capacity finalize failed")
12012                .await?;
12013            ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity finalize")?;
12014            save_guard = Some(finalize_save_guard);
12015            write_guard = Some(finalize_write_guard);
12016            snapshot = finalize_snapshot;
12017        }
12018        let write_guard = write_guard
12019            .as_ref()
12020            .ok_or_else(|| Error::other("decommission capacity write fence disappeared before finalize"))?;
12021        let save_guard = save_guard
12022            .as_mut()
12023            .ok_or_else(|| Error::other("decommission capacity save fence disappeared before finalize"))?;
12024        ensure_pool_meta_write_fence(write_guard, "decommission target capacity finalize failed")?;
12025        ensure_decommission_capacity_mutation_intent_current(
12026            &snapshot,
12027            owner,
12028            target_pool_index,
12029            expected_target_physical_bytes,
12030            mutation_id,
12031            temporary_release,
12032            model_version,
12033        )?;
12034        let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
12035        let after_free = capacity_infos
12036            .iter()
12037            .find(|capacity| capacity.pool_index == target_pool_index)
12038            .map(|capacity| capacity.physical_free)
12039            .ok_or_else(|| decommission_capacity_blocked_error("target capacity snapshot is missing after mutation"))?;
12040        let observed_physical_bytes = before_free.saturating_sub(after_free);
12041        let released_physical_bytes = after_free.saturating_sub(before_free);
12042        let now = OffsetDateTime::now_utc();
12043        let progress_changed = if temporary_release {
12044            if result.is_ok() {
12045                release_decommission_target_inflight(
12046                    &mut snapshot,
12047                    source_pool_index,
12048                    target_pool_index,
12049                    released_physical_bytes,
12050                    mutation_id,
12051                    DecommissionCapacityReleaseProof::confirmed_absence(clear_pending_on_temporary_release),
12052                    now,
12053                )?
12054            } else {
12055                false
12056            }
12057        } else if result.is_ok() {
12058            resolve_decommission_target_pending(
12059                &mut snapshot,
12060                source_pool_index,
12061                target_pool_index,
12062                expected_target_physical_bytes,
12063                mutation_id,
12064            )?;
12065            if temporary {
12066                if non_growing_replacement {
12067                    settle_decommission_target_non_growing_replacement(
12068                        &mut snapshot,
12069                        source_pool_index,
12070                        target_pool_index,
12071                        mutation_id,
12072                        now,
12073                    )?;
12074                } else {
12075                    record_decommission_target_inflight(
12076                        &mut snapshot,
12077                        source_pool_index,
12078                        target_pool_index,
12079                        observed_physical_bytes,
12080                        mutation_id,
12081                        now,
12082                    )?;
12083                }
12084            } else {
12085                let consumed_physical_bytes = expected_data_bytes
12086                    .map(|_| expected_target_physical_bytes)
12087                    .unwrap_or(observed_physical_bytes.max(expected_target_physical_bytes));
12088                let committed_data_bytes =
12089                    expected_data_bytes.unwrap_or(capacity_source_data_equivalent(observed_physical_bytes, target_layout)?);
12090                record_decommission_target_consumption(
12091                    &mut snapshot,
12092                    source_pool_index,
12093                    target_pool_index,
12094                    DecommissionTargetConsumption {
12095                        committed_data_bytes,
12096                        target_physical_bytes: consumed_physical_bytes,
12097                        observed_physical_bytes,
12098                    },
12099                    mutation_id,
12100                    now,
12101                )?;
12102            }
12103            true
12104        } else if temporary {
12105            record_decommission_target_inflight(
12106                &mut snapshot,
12107                source_pool_index,
12108                target_pool_index,
12109                observed_physical_bytes,
12110                mutation_id,
12111                now,
12112            )?
12113        } else {
12114            record_decommission_target_observation(
12115                &mut snapshot,
12116                source_pool_index,
12117                target_pool_index,
12118                observed_physical_bytes,
12119                now,
12120            )?;
12121            observed_physical_bytes > 0
12122        };
12123        let capacity_result = if temporary_release {
12124            Ok(())
12125        } else {
12126            ensure_decommission_capacity_reservations_available(&snapshot, &capacity_infos, "mutation")
12127        };
12128        if let Err(err) = &capacity_result {
12129            snapshot.mark_decommission_capacity_blocked(source_pool_index, err.to_string(), now)?;
12130        }
12131        if temporary_release && !progress_changed {
12132            ensure_pool_meta_write_fence(write_guard, "decommission target capacity cleanup fence failed")?;
12133            if let Some(target_guard) = target_guard.as_ref() {
12134                ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity cleanup")?;
12135            }
12136            capacity_result?;
12137            return result;
12138        }
12139        let outcome = snapshot
12140            .save_no_lock_armed(self.pools.clone(), save_guard, write_guard.lock_lost_signal(), &[source_pool_index])
12141            .await?;
12142        ensure_pool_meta_write_fence(write_guard, "decommission target capacity progress save failed")?;
12143        if let Some(target_guard) = target_guard.as_ref() {
12144            ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity progress save")?;
12145        }
12146        {
12147            let mut pool_meta = self.pool_meta.write().await;
12148            publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, source_pool_index)?;
12149        }
12150        ensure_pool_meta_write_fence(write_guard, "decommission target capacity progress save failed")?;
12151        if let Some(target_guard) = target_guard.as_ref() {
12152            ensure_decommission_capacity_target_fence(target_guard, target_pool_index, "capacity progress publication")?;
12153        }
12154        outcome.disarm();
12155        capacity_result?;
12156        result
12157    }
12158
12159    pub(crate) async fn ensure_pool_meta_side_effects_safe(&self, operation: &str) -> Result<()> {
12160        self.pool_meta_save_gate.lock().await.ensure_write_safe(operation)
12161    }
12162
12163    /// Reports whether pool metadata side effects are currently writable.
12164    /// Read-only admission probes do not change this state; startup and real
12165    /// metadata transactions still latch it on unrecoverable conditions.
12166    pub async fn pool_meta_writes_ready(&self) -> bool {
12167        self.pool_meta_write_status().await.is_ok()
12168    }
12169
12170    /// A health probe must not wait behind a metadata transaction's disk I/O.
12171    pub async fn pool_meta_write_gate_status(&self) -> PoolMetaWriteGateStatus {
12172        let Ok(write_state) = tokio::time::timeout(std::time::Duration::from_millis(100), self.pool_meta_save_gate.lock()).await
12173        else {
12174            return PoolMetaWriteGateStatus {
12175                writes_ready: false,
12176                check_timed_out: true,
12177                ..PoolMetaWriteGateStatus::default()
12178            };
12179        };
12180        let transaction_aborted = write_state.aborted_transaction.load(Ordering::SeqCst);
12181        let writes_ready = !write_state.write_blocked && !transaction_aborted;
12182        let failure = if writes_ready {
12183            None
12184        } else {
12185            write_state.ensure_write_safe("pool metadata snapshot").err()
12186        };
12187        let context = failure.as_ref().and_then(Error::pool_metadata_failure);
12188        PoolMetaWriteGateStatus {
12189            writes_ready,
12190            check_timed_out: false,
12191            write_blocked: write_state.write_blocked,
12192            transaction_aborted,
12193            pool_meta_absent: write_state.pool_meta_absent,
12194            identity_initialized: write_state.identity_initialized,
12195            identity_needs_repair: write_state.identity_needs_repair,
12196            cluster_epoch: write_state.cluster_epoch,
12197            reason: context.map(|context| context.kind.as_str()),
12198            phase: context.map(|context| context.phase),
12199            since_unix_secs: context.map(|context| context.since.unix_timestamp()),
12200        }
12201    }
12202
12203    /// A health probe must not wait behind a metadata transaction's disk I/O.
12204    pub async fn pool_meta_write_status(&self) -> Result<()> {
12205        let write_state = tokio::time::timeout(std::time::Duration::from_millis(100), self.pool_meta_save_gate.lock())
12206            .await
12207            .map_err(|_| Error::Timeout)?;
12208        write_state.ensure_write_safe("pool metadata readiness")
12209    }
12210
12211    pub(crate) async fn recover_pool_meta_transaction(&self) -> Result<bool> {
12212        {
12213            let Ok(state) = self.pool_meta_save_gate.try_lock() else {
12214                return Ok(false);
12215            };
12216            if state.ensure_write_safe("pool metadata recovery").is_ok() {
12217                return Ok(false);
12218            }
12219            if state.write_blocked
12220                || state.active_transactions.load(Ordering::SeqCst) != 0
12221                || state
12222                    .transaction_failure
12223                    .lock()
12224                    .unwrap_or_else(std::sync::PoisonError::into_inner)
12225                    .is_none()
12226            {
12227                return Ok(false);
12228            }
12229        }
12230        let _start_guard = self.start_gate.lock().await;
12231        // Cancellation must precede the movement writer. Let the existing
12232        // supervisor join every worker and relinquish its operation ownership;
12233        // a new worker must never overlap an old terminal/progress callback.
12234        {
12235            let cancelers = self.decommission_cancelers.read().await;
12236            let mut draining = false;
12237            for canceler in cancelers.iter().flatten().filter(|canceler| canceler.is_active()) {
12238                canceler.cancel();
12239                draining = true;
12240            }
12241            if draining {
12242                return Err(Error::OperationCanceled);
12243            }
12244        }
12245        let movement_gate = self.ctx.data_movement_operation_gate();
12246        let _movement_guard = movement_gate.write().await;
12247        let mut state = self.pool_meta_save_gate.lock().await;
12248        // Outcomes can outlive the save-gate guard. Do not let an old Drop
12249        // relatch, or an older recovery clear a newly installed block.
12250        if state.write_blocked || state.active_transactions.load(Ordering::SeqCst) != 0 {
12251            return Ok(false);
12252        }
12253        let Some(blocked) = state
12254            .transaction_failure
12255            .lock()
12256            .unwrap_or_else(std::sync::PoisonError::into_inner)
12257            .clone()
12258        else {
12259            return Ok(false);
12260        };
12261        if self
12262            .rebalance_meta
12263            .read()
12264            .await
12265            .as_ref()
12266            .is_some_and(|meta| meta.cancel.is_some())
12267        {
12268            return Err(Error::other("pool metadata recovery requires rebalance workers to be quiescent"));
12269        }
12270        let pool = self
12271            .pools
12272            .first()
12273            .ok_or_else(|| Error::other("pool metadata recovery has no storage pools"))?;
12274        let lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
12275        let guard = lock.get_write_lock(get_lock_acquire_timeout()).await?;
12276        let fence = PoolMetaPersistenceFence::Distributed(guard.lock_lost_signal());
12277        let mut candidate = PoolMetaWriteState {
12278            aborted_transaction: Arc::new(AtomicBool::new(false)),
12279            transaction_failure: Arc::default(),
12280            active_transactions: Arc::default(),
12281            block_context: None,
12282            recovery_failure: None,
12283            write_blocked: false,
12284            ..state.clone()
12285        };
12286        let selection = load_pool_meta_for_transaction_recovery(self.pools.clone(), &mut candidate).await?;
12287        if selection.meta.pools.len() != self.pools.len()
12288            || selection
12289                .meta
12290                .pools
12291                .iter()
12292                .zip(&self.pools)
12293                .enumerate()
12294                .any(|(index, (meta, pool))| meta.id != index || meta.cmd_line != pool.endpoints.cmd_line)
12295        {
12296            return Err(Error::other(
12297                "pool metadata recovery requires operator reconciliation of the pool topology",
12298            ));
12299        }
12300        let outcome = repair_pool_meta_transaction(self.pools.clone(), &mut candidate, selection, &fence).await?;
12301        let mut pool_meta = self.pool_meta.write().await;
12302        fence.ensure_held()?;
12303        self.ctx.advance_data_movement_operation_epoch();
12304        *pool_meta = outcome.committed.clone();
12305        fence.ensure_held()?;
12306        outcome.disarm();
12307        // No await between final fence validation, publication, and clearing.
12308        candidate.aborted_transaction = Arc::clone(&state.aborted_transaction);
12309        candidate.transaction_failure = Arc::clone(&state.transaction_failure);
12310        candidate.active_transactions = Arc::clone(&state.active_transactions);
12311        candidate.block_started_at = Arc::clone(&state.block_started_at);
12312        *state
12313            .block_started_at
12314            .lock()
12315            .unwrap_or_else(std::sync::PoisonError::into_inner) = None;
12316        *state
12317            .transaction_failure
12318            .lock()
12319            .unwrap_or_else(std::sync::PoisonError::into_inner) = None;
12320        state.aborted_transaction.store(false, Ordering::SeqCst);
12321        *state = candidate;
12322        info!(
12323            event = EVENT_DECOMMISSION_STATE,
12324            component = LOG_COMPONENT_ECSTORE,
12325            subsystem = LOG_SUBSYSTEM_POOLS,
12326            state = "pool_metadata_recovered",
12327            phase = blocked.phase,
12328            blocked_since = %blocked.since,
12329            "Pool metadata writes recovered"
12330        );
12331        metrics::counter!("rustfs_pool_metadata_recoveries_total").increment(1);
12332        Ok(true)
12333    }
12334
12335    pub(crate) fn record_pool_meta_recovery_failure(&self, error: Error) {
12336        let Ok(mut state) = self.pool_meta_save_gate.try_lock() else {
12337            return;
12338        };
12339        if state.ensure_write_safe("pool metadata recovery").is_ok() {
12340            return;
12341        }
12342        let classify = |error: &Error| match error.pool_metadata_failure() {
12343            Some(context) => (context.kind.as_str(), context.phase),
12344            None => match error {
12345                Error::OperationCanceled => ("workers_draining", "recovery_quiesce"),
12346                Error::Timeout => ("timeout", "recovery"),
12347                Error::Lock(_) => ("fence_unavailable", "recovery"),
12348                _ => ("unavailable", "recovery"),
12349            },
12350        };
12351        let (reason, phase) = classify(&error);
12352        if state
12353            .recovery_failure
12354            .as_ref()
12355            .is_none_or(|previous| classify(previous) != (reason, phase))
12356        {
12357            warn!(
12358                event = EVENT_DECOMMISSION_STATE,
12359                component = LOG_COMPONENT_ECSTORE,
12360                subsystem = LOG_SUBSYSTEM_POOLS,
12361                state = "pool_metadata_recovery_pending",
12362                reason,
12363                phase,
12364                "Pool metadata recovery remains blocked"
12365            );
12366        }
12367        state.recovery_failure = Some(Arc::new(error));
12368    }
12369
12370    async fn load_runtime_pool_meta_observing(&self, write_state: &mut PoolMetaWriteState, operation: &str) -> Result<PoolMeta> {
12371        write_state.ensure_write_safe(operation)?;
12372        load_pool_meta_identity_observing(self.pools.clone(), write_state).await?;
12373        let mut pool_meta = PoolMeta::default();
12374        let replica_state = pool_meta
12375            .load_no_lock_from_replicas_observing(self.pools.clone(), write_state)
12376            .await?;
12377        write_state.observe_replicas(replica_state);
12378        write_state.ensure_missing_metadata_can_initialize()?;
12379        write_state.ensure_write_safe(operation)?;
12380        Ok(pool_meta)
12381    }
12382
12383    pub(crate) async fn load_runtime_pool_meta_under_activation_fence(
12384        &self,
12385        write_state: &mut PoolMetaWriteState,
12386        activation_fence: &PoolRebalanceActivationFence,
12387        operation: &str,
12388    ) -> Result<PoolMeta> {
12389        activation_fence.ensure_held()?;
12390        let pool_meta = self.load_runtime_pool_meta_observing(write_state, operation).await?;
12391        activation_fence.ensure_held()?;
12392        Ok(pool_meta)
12393    }
12394
12395    pub(crate) async fn load_runtime_pool_meta(&self, operation: &str) -> Result<PoolMeta> {
12396        let mut write_state = self.pool_meta_save_gate.lock().await;
12397        write_state.ensure_write_safe(operation)?;
12398        let pool = self
12399            .pools
12400            .first()
12401            .cloned()
12402            .ok_or_else(|| Error::other(format!("{operation}: no storage pools available")))?;
12403        let pool_meta_lock = pool.new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME).await?;
12404        let _pool_meta_guard = pool_meta_lock.get_read_lock(get_lock_acquire_timeout()).await?;
12405        self.load_runtime_pool_meta_observing(&mut write_state, operation).await
12406    }
12407
12408    async fn run_guarded_decommission_side_effect<T, E, F, Fut>(
12409        &self,
12410        rx: &CancellationToken,
12411        operation_gate: &Arc<tokio::sync::RwLock<()>>,
12412        operation: F,
12413    ) -> std::result::Result<T, E>
12414    where
12415        F: FnOnce() -> Fut,
12416        Fut: std::future::Future<Output = std::result::Result<T, E>>,
12417        E: From<Error>,
12418    {
12419        let _operation_guard = tokio::select! {
12420            biased;
12421            _ = rx.cancelled() => return Err(Error::OperationCanceled.into()),
12422            guard = operation_gate.read() => guard,
12423        };
12424
12425        if rx.is_cancelled() {
12426            return Err(Error::OperationCanceled.into());
12427        }
12428        self.ensure_pool_meta_side_effects_safe("decommission side effect blocked because pool metadata requires recovery")
12429            .await
12430            .map_err(E::from)?;
12431        if rx.is_cancelled() {
12432            return Err(Error::OperationCanceled.into());
12433        }
12434
12435        let result = operation().await;
12436        if rx.is_cancelled() {
12437            return Err(Error::OperationCanceled.into());
12438        }
12439        result
12440    }
12441
12442    async fn save_current_pool_meta(&self, indices: &[usize]) -> Result<()> {
12443        let mut save_guard = self.pool_meta_save_gate.lock().await;
12444        let (pool_meta_guard, mut snapshot) = self
12445            .acquire_pool_meta_write_guard(&mut save_guard, "pool metadata save failed")
12446            .await?;
12447        {
12448            let pool_meta = self.pool_meta.read().await;
12449            merge_pool_meta_updates_for_save(&mut snapshot, &pool_meta, indices, "pool metadata save failed")?;
12450        }
12451        let outcome = snapshot
12452            .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), indices)
12453            .await?;
12454        let mut pool_meta = self.pool_meta.write().await;
12455        ensure_pool_meta_write_fence(&pool_meta_guard, "pool metadata save failed")?;
12456        publish_pool_meta_updates(&mut pool_meta, &outcome.committed, indices);
12457        ensure_pool_meta_write_fence(&pool_meta_guard, "pool metadata save failed")?;
12458        drop(pool_meta);
12459        outcome.disarm();
12460        Ok(())
12461    }
12462
12463    #[cfg(test)]
12464    pub(crate) async fn save_current_pool_meta_for_test(&self, indices: &[usize]) -> Result<()> {
12465        self.save_current_pool_meta(indices).await
12466    }
12467
12468    async fn persist_decommission_unresolved_entry(
12469        &self,
12470        idx: usize,
12471        generation: OffsetDateTime,
12472        entry: DecommissionUnresolvedEntry,
12473    ) -> Result<()> {
12474        {
12475            let rebalance_meta = self.rebalance_meta.read().await.clone();
12476            let mut pool_meta = self.pool_meta.write().await;
12477            ensure_decommission_ledger_persistence_supported(&pool_meta)?;
12478            record_decommission_unresolved_entry(
12479                &mut pool_meta,
12480                idx,
12481                generation,
12482                entry,
12483                OffsetDateTime::now_utc(),
12484                rebalance_meta.as_ref(),
12485            )?;
12486        }
12487        self.save_current_pool_meta(&[idx])
12488            .await
12489            .map_err(|err| Error::other(format!("decommission unresolved entry ledger save failed: {err}")))
12490    }
12491
12492    async fn save_decommission_progress_checkpoint(&self, idx: usize, generation: OffsetDateTime) -> Result<bool> {
12493        self.save_decommission_progress_checkpoint_at(idx, generation, OffsetDateTime::now_utc())
12494            .await
12495    }
12496
12497    async fn save_decommission_progress_checkpoint_at(
12498        &self,
12499        idx: usize,
12500        generation: OffsetDateTime,
12501        now: OffsetDateTime,
12502    ) -> Result<bool> {
12503        // Lock order: save gate, rebalance metadata, then the short pool
12504        // metadata read/write sections. Peer reloads are intentionally
12505        // performed by the caller after both locks are released.
12506        let mut save_guard = self.pool_meta_save_gate.lock().await;
12507        let (pool_meta_guard, mut snapshot) = self
12508            .acquire_pool_meta_write_guard(&mut save_guard, "decommission progress save failed")
12509            .await?;
12510        let (snapshot, checkpoint) = {
12511            let rebalance_meta = self.rebalance_meta.read().await.clone();
12512            let pool_meta = self.pool_meta.read().await;
12513            ensure_decommission_generation(&pool_meta, idx, generation)?;
12514            let Some(checkpoint) = pool_meta.decommission_progress_checkpoint(
12515                idx,
12516                DECOMMISSION_PROGRESS_SAVE_INTERVAL,
12517                now,
12518                rebalance_meta.as_ref(),
12519            )?
12520            else {
12521                return Ok(false);
12522            };
12523
12524            let mut current = pool_meta.clone();
12525            let current_count = current.pools.len();
12526            let Some(pool) = current.pools.get_mut(idx) else {
12527                return Err(invalid_decommission_pool_index_error(current_count, idx));
12528            };
12529            pool.last_update = checkpoint.checkpoint_at;
12530            if let (Some(operation_id), Some(owner_nonce), Some(expires_at), Some(reservation)) = (
12531                checkpoint.capacity_operation_id,
12532                checkpoint.capacity_owner_nonce,
12533                checkpoint.capacity_lease_expires_at,
12534                pool.decommission.as_mut().and_then(|info| info.capacity_reservation.as_mut()),
12535            ) && reservation.operation_id == operation_id
12536                && reservation.owner_nonce == owner_nonce
12537                && reservation.active()
12538            {
12539                reservation.renewed_at = checkpoint.checkpoint_at;
12540                reservation.expires_at = expires_at;
12541            }
12542            merge_pool_meta_updates_for_save(&mut snapshot, &current, &[idx], "decommission progress save failed")?;
12543            (snapshot, checkpoint)
12544        };
12545
12546        let outcome = match snapshot
12547            .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx])
12548            .await
12549        {
12550            Ok(outcome) => outcome,
12551            Err(err) => {
12552                let retry_after = OffsetDateTime::now_utc() + DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
12553                let mut pool_meta = self.pool_meta.write().await;
12554                pool_meta.defer_decommission_progress_checkpoint(idx, checkpoint, retry_after);
12555                return Err(err);
12556            }
12557        };
12558
12559        let mut pool_meta = self.pool_meta.write().await;
12560        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission progress save failed")?;
12561        pool_meta.version = pool_meta.version.max(outcome.committed.version);
12562        let committed = pool_meta.commit_decommission_progress_checkpoint(idx, checkpoint);
12563        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission progress save failed")?;
12564        drop(pool_meta);
12565        outcome.disarm();
12566        Ok(committed)
12567    }
12568
12569    async fn mark_decommission_bucket_done_and_save(
12570        &self,
12571        idx: usize,
12572        generation: OffsetDateTime,
12573        bucket: &DecomBucketInfo,
12574    ) -> Result<bool> {
12575        // Lock order: save gate -> pool metadata namespace -> rebalance metadata
12576        // -> short local pool metadata sections. Publish only after the save.
12577        let mut save_guard = self.pool_meta_save_gate.lock().await;
12578        let (pool_meta_guard, mut snapshot) = self
12579            .acquire_pool_meta_write_guard(&mut save_guard, "decommission bucket completion save failed")
12580            .await?;
12581        ensure_decommission_generation(&snapshot, idx, generation)?;
12582        let rebalance_meta = self.rebalance_meta.read().await.clone();
12583        let checkpoint = {
12584            let pool_meta = self.pool_meta.read().await;
12585            ensure_decommission_generation(&pool_meta, idx, generation)?;
12586            let mut current = pool_meta.clone();
12587            publish_decommission_capacity_update(&mut current, &snapshot, idx)?;
12588            if !mark_decommission_bucket_done(&mut current, idx, bucket)? {
12589                return Ok(false);
12590            }
12591            let checkpoint_at = current.next_scanner_data_movement_update(OffsetDateTime::now_utc(), rebalance_meta.as_ref());
12592            current.pools[idx].last_update = checkpoint_at;
12593            let info = current.pools[idx]
12594                .decommission
12595                .as_ref()
12596                .ok_or_else(|| decommission_metadata_not_initialized_error("prepare decommission bucket completion"))?;
12597            let checkpoint = DecommissionProgressCheckpoint {
12598                start_time: Some(generation),
12599                queued: false,
12600                counted_items: info.counted_items(),
12601                checkpoint_at,
12602                capacity_operation_id: None,
12603                capacity_owner_nonce: None,
12604                capacity_lease_expires_at: None,
12605            };
12606            merge_pool_meta_updates_for_save(&mut snapshot, &current, &[idx], "decommission bucket completion save failed")?;
12607            checkpoint
12608        };
12609
12610        #[cfg(test)]
12611        decommission_test_wrap_result("bucket_completion_before_save", &bucket.name, &bucket.prefix, 0, Ok(()))?;
12612        let outcome = snapshot
12613            .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx])
12614            .await?;
12615        #[cfg(test)]
12616        decommission_test_wrap_result("bucket_completion_after_save", &bucket.name, &bucket.prefix, 0, Ok(()))?;
12617        let mut pool_meta = self.pool_meta.write().await;
12618        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission bucket completion save failed")?;
12619        ensure_decommission_generation(&pool_meta, idx, generation)?;
12620        publish_decommission_capacity_update(&mut pool_meta, &outcome.committed, idx)?;
12621        mark_decommission_bucket_done(&mut pool_meta, idx, bucket)?;
12622        pool_meta.commit_decommission_progress_checkpoint(idx, checkpoint);
12623        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission bucket completion save failed")?;
12624        drop(pool_meta);
12625        outcome.disarm();
12626        Ok(true)
12627    }
12628
12629    pub(crate) async fn save_current_pool_meta_for_decommission_start(
12630        &self,
12631        indices: &[usize],
12632        decom_buckets: Vec<DecomBucketInfo>,
12633    ) -> Result<PoolMeta> {
12634        let mut save_guard = self.pool_meta_save_gate.lock().await;
12635        save_guard.ensure_write_safe("decommission start failed")?;
12636        let rebalance_pool = self
12637            .pools
12638            .first()
12639            .cloned()
12640            .ok_or_else(|| Error::other("decommission start rebalance metadata load failed: no storage pools available"))?;
12641        #[cfg(test)]
12642        observe_pool_activation_start_attempt(PoolActivationStartKind::Decommission);
12643        let fleet_proof = acquire_pool_activation_fleet_proof(&self.ctx).await?;
12644        let mut activation_fence = acquire_pool_rebalance_activation_locks(rebalance_pool.clone(), fleet_proof).await?;
12645
12646        let mut rebalance_meta = RebalanceMeta::new();
12647        match rebalance_meta
12648            .load_with_opts(
12649                rebalance_pool.clone(),
12650                ObjectOptions {
12651                    no_lock: true,
12652                    ..Default::default()
12653                },
12654            )
12655            .await
12656        {
12657            Ok(()) => ensure_decommission_start_rebalance_meta_allowed(Some(&rebalance_meta))?,
12658            Err(Error::ConfigNotFound) => {}
12659            Err(err) => {
12660                return Err(Error::other(format!(
12661                    "rebalance metadata load before decommission start save failed: {err}"
12662                )));
12663            }
12664        }
12665
12666        let current_pool_meta = {
12667            let pool_meta = self.pool_meta.read().await;
12668            pool_meta.clone()
12669        };
12670        let mut latest_pool_meta = PoolMeta::default();
12671        let replica_state = latest_pool_meta
12672            .load_no_lock_from_replicas_observing(self.pools.clone(), &mut save_guard)
12673            .await?;
12674        save_guard.observe_replicas(replica_state);
12675        save_guard.ensure_write_safe("decommission start failed")?;
12676        if latest_pool_meta.pools.is_empty() {
12677            latest_pool_meta = current_pool_meta;
12678        }
12679
12680        ensure_decommission_start_pool_states(&latest_pool_meta, indices)?;
12681        ensure_decommission_ledger_persistence_supported(&latest_pool_meta)?;
12682        ensure_decommission_capacity_writer_supported(&latest_pool_meta)?;
12683        let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
12684        activation_fence.ensure_held()?;
12685        let target_fence_proof = crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof();
12686
12687        let previous_pool_meta = latest_pool_meta.clone();
12688        let capacity_generation = next_decommission_capacity_generation(&latest_pool_meta)?;
12689        let first_idx = indices.first().copied();
12690        let now = OffsetDateTime::now_utc();
12691        for idx in indices.iter().copied() {
12692            let capacity = capacity_infos
12693                .iter()
12694                .find(|capacity| capacity.pool_index == idx)
12695                .ok_or_else(|| Error::DecommissionCapacity(format!("decommission capacity snapshot is missing pool {idx}")))?;
12696            latest_pool_meta.set_decommission_state_at(
12697                idx,
12698                capacity.space,
12699                Some(idx) != first_idx,
12700                now,
12701                Some(&rebalance_meta),
12702            )?;
12703            latest_pool_meta.queue_buckets(idx, decom_buckets.clone());
12704        }
12705        let model_version = select_decommission_capacity_model(&latest_pool_meta, target_fence_proof.is_some())?;
12706        if model_version == DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION {
12707            activation_fence.set_decommission_target_fence_proof(target_fence_proof);
12708        }
12709        reserve_decommission_start_target_capacity(
12710            &mut latest_pool_meta,
12711            indices,
12712            &capacity_infos,
12713            uuid::Uuid::new_v4(),
12714            capacity_generation,
12715            now,
12716            model_version,
12717        )?;
12718
12719        activation_fence.ensure_held()?;
12720        let outcome = latest_pool_meta
12721            .save_no_lock_with_activation_fence(self.pools.clone(), &mut save_guard, &activation_fence, indices)
12722            .await?;
12723        activation_fence.ensure_held()?;
12724        {
12725            let mut pool_meta = self.pool_meta.write().await;
12726            publish_pool_meta_updates(&mut pool_meta, &outcome.committed, indices);
12727        }
12728        activation_fence.ensure_held()?;
12729        outcome.disarm();
12730
12731        Ok(previous_pool_meta)
12732    }
12733
12734    async fn rollback_decommission_start_after_reload_failure(
12735        &self,
12736        movement_gate: &Arc<tokio::sync::RwLock<()>>,
12737        previous_pool_meta: &PoolMeta,
12738        indices: &[usize],
12739    ) -> Result<()> {
12740        let _movement_guard = movement_gate.write().await;
12741        let mut save_guard = self.pool_meta_save_gate.lock().await;
12742        let (pool_meta_guard, mut snapshot) = self
12743            .acquire_pool_meta_write_guard(&mut save_guard, "decommission start rollback failed")
12744            .await?;
12745        rollback_start_decommission_pool_meta(&mut snapshot, previous_pool_meta, indices);
12746        let outcome = snapshot
12747            .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), indices)
12748            .await?;
12749        let mut pool_meta = self.pool_meta.write().await;
12750        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission start rollback failed")?;
12751        publish_pool_meta_updates(&mut pool_meta, &outcome.committed, indices);
12752        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission start rollback failed")?;
12753        drop(pool_meta);
12754        outcome.disarm();
12755        self.ctx.advance_data_movement_operation_epoch();
12756        Ok(())
12757    }
12758
12759    async fn ensure_decommission_rebalance_idle_after_refresh(&self) -> Result<()> {
12760        self.load_rebalance_meta().await?;
12761        ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)
12762    }
12763
12764    async fn ensure_decommission_rebalance_idle_after_refresh_under_start_gate(&self) -> Result<()> {
12765        self.load_rebalance_meta_under_start_gate().await?;
12766        ensure_decommission_not_rebalancing(self.is_rebalance_conflicting_with_decommission().await)
12767    }
12768
12769    pub async fn status(&self, idx: usize) -> Result<PoolStatus> {
12770        let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
12771        let space_info = capacity_infos
12772            .iter()
12773            .find(|capacity| capacity.pool_index == idx)
12774            .map(|capacity| capacity.space)
12775            .ok_or_else(|| invalid_decommission_pool_index_error(self.pools.len(), idx))?;
12776
12777        let pool_meta = self.pool_meta.read().await;
12778        let active_target_reservations = active_decommission_target_reservations(&pool_meta);
12779
12780        let mut pool_info = get_by_index(pool_meta.pools.as_slice(), idx, "fetch decommission status")?.clone();
12781        if let Some(info) = pool_info.decommission.as_mut() {
12782            observe_decommission_capacity_reservation(info, &capacity_infos, &active_target_reservations);
12783        }
12784        Ok(apply_decommission_status_space_info(pool_info, space_info))
12785    }
12786
12787    #[tracing::instrument(skip_all)]
12788    pub async fn refresh_pool_status_meta(&self) -> Result<()> {
12789        let movement_gate = self.ctx.data_movement_operation_gate();
12790        let _movement_guard = movement_gate.write().await;
12791        let persisted = self.load_runtime_pool_meta("refresh pool status metadata failed").await?;
12792
12793        let active_workers = {
12794            let cancelers = self.decommission_cancelers.read().await;
12795            cancelers
12796                .iter()
12797                .map(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
12798                .collect::<Vec<_>>()
12799        };
12800
12801        let mut pool_meta = self.pool_meta.write().await;
12802        if merge_pool_status_refresh(&mut pool_meta, persisted, &active_workers) {
12803            self.ctx.advance_data_movement_operation_epoch();
12804        }
12805        Ok(())
12806    }
12807
12808    async fn get_decommission_pool_capacity_info(
12809        &self,
12810        idx: usize,
12811        use_owned_usage: bool,
12812    ) -> Result<DecommissionPoolCapacityInfo> {
12813        if let Some(sets) = self.pools.get(idx) {
12814            let mut info = sets.storage_info_snapshot().await;
12815            info.backend = StorageAdminApi::backend_info(self).await;
12816
12817            let total = get_total_usable_capacity(&info.disks, &info);
12818            let free = get_total_usable_capacity_free(&info.disks, &info);
12819            let space = PoolSpaceInfo {
12820                free,
12821                total,
12822                used: total.saturating_sub(free),
12823            };
12824            let layout = DecommissionErasureLayout {
12825                data: info
12826                    .backend
12827                    .standard_sc_data
12828                    .get(idx)
12829                    .copied()
12830                    .unwrap_or_else(|| sets.set_drive_count.saturating_sub(sets.parity_count)),
12831                parity: info
12832                    .backend
12833                    .standard_sc_parities
12834                    .get(idx)
12835                    .copied()
12836                    .unwrap_or(sets.parity_count),
12837            };
12838            if !layout.is_valid() {
12839                return Err(Error::DecommissionCapacity(format!(
12840                    "failed to read decommission capacity for pool {idx}: invalid erasure layout data={} parity={}",
12841                    layout.data, layout.parity
12842                )));
12843            }
12844            let owned_physical_used = if use_owned_usage {
12845                Some(decommission_owned_physical_usage(sets, layout).await)
12846            } else {
12847                None
12848            };
12849            let (physical_total, physical_free, physical_used) =
12850                decommission_physical_pool_capacity(&info.disks, idx, layout, space, owned_physical_used.as_ref());
12851
12852            Ok(DecommissionPoolCapacityInfo {
12853                pool_index: idx,
12854                space,
12855                layout,
12856                physical_free,
12857                physical_total,
12858                physical_used,
12859            })
12860        } else {
12861            Err(invalid_decommission_pool_index_error(self.pools.len(), idx))
12862        }
12863    }
12864
12865    async fn get_decommission_all_pool_capacity_infos(&self) -> Result<Vec<DecommissionPoolCapacityInfo>> {
12866        #[cfg(any(test, feature = "test-util"))]
12867        if let Some(capacity_infos) = take_decommission_capacity_info_override_for_test(self.id) {
12868            return Ok(capacity_infos);
12869        }
12870
12871        let mut capacity_infos = Vec::with_capacity(self.pools.len());
12872        for idx in 0..self.pools.len() {
12873            capacity_infos.push(self.get_decommission_pool_capacity_info(idx, false).await?);
12874        }
12875        Ok(capacity_infos)
12876    }
12877
12878    async fn get_decommission_all_pool_capacity_infos_with_owned_usage(&self) -> Result<Vec<DecommissionPoolCapacityInfo>> {
12879        #[cfg(any(test, feature = "test-util"))]
12880        if let Some(capacity_infos) = take_decommission_capacity_info_override_for_test(self.id) {
12881            return Ok(capacity_infos);
12882        }
12883
12884        let mut capacity_infos = Vec::with_capacity(self.pools.len());
12885        for idx in 0..self.pools.len() {
12886            capacity_infos.push(self.get_decommission_pool_capacity_info(idx, true).await?);
12887        }
12888        Ok(capacity_infos)
12889    }
12890
12891    async fn ensure_decommission_runtime_capacity_available(&self, idx: usize, generation: OffsetDateTime) -> Result<()> {
12892        let capacity_infos = self.get_decommission_all_pool_capacity_infos().await?;
12893        let pool_meta = self.pool_meta.read().await;
12894        ensure_decommission_generation(&pool_meta, idx, generation)?;
12895        #[cfg(test)]
12896        if pool_meta
12897            .pools
12898            .get(idx)
12899            .and_then(|pool| pool.decommission.as_ref())
12900            .is_some_and(|info| info.capacity_reservation.is_none())
12901        {
12902            return Ok(());
12903        }
12904        ensure_decommission_capacity_reservations_available(&pool_meta, &capacity_infos, "migration")
12905    }
12906
12907    async fn decommission_capacity_owner_for_worker(
12908        &self,
12909        idx: usize,
12910        generation: OffsetDateTime,
12911    ) -> Result<Option<DecommissionCapacityOwner>> {
12912        let pool_meta = self.pool_meta.read().await;
12913        ensure_decommission_generation(&pool_meta, idx, generation)?;
12914        let Some(reservation) = pool_meta
12915            .pools
12916            .get(idx)
12917            .and_then(|pool| pool.decommission.as_ref())
12918            .and_then(|info| info.capacity_reservation.as_ref())
12919        else {
12920            return Ok(None);
12921        };
12922        if !reservation.lease_active_at(OffsetDateTime::now_utc()) {
12923            return Err(decommission_capacity_blocked_error(
12924                "decommission capacity reservation lease is not active",
12925            ));
12926        }
12927        Ok(Some(DecommissionCapacityOwner {
12928            source_pool_index: idx,
12929            operation_id: reservation.operation_id,
12930            generation: reservation.generation,
12931            owner_nonce: reservation.owner_nonce,
12932            mutation_id: None,
12933        }))
12934    }
12935
12936    pub(crate) async fn is_decommission_capacity_target_reserved(
12937        &self,
12938        owner: DecommissionCapacityOwner,
12939        target_pool_index: usize,
12940    ) -> Result<bool> {
12941        let pool_meta = self.pool_meta.read().await;
12942        let reservation = pool_meta
12943            .pools
12944            .get(owner.source_pool_index)
12945            .and_then(|pool| pool.decommission.as_ref())
12946            .and_then(|info| info.capacity_reservation.as_ref())
12947            .filter(|reservation| reservation.admits_owner(owner, OffsetDateTime::now_utc()))
12948            .ok_or_else(|| decommission_capacity_blocked_error("decommission target selection reservation is stale"))?;
12949        Ok(reservation
12950            .targets
12951            .iter()
12952            .any(|target| target.pool_index == target_pool_index))
12953    }
12954
12955    pub(crate) async fn select_decommission_capacity_target_pool(
12956        &self,
12957        owner: DecommissionCapacityOwner,
12958        expected_data_bytes: usize,
12959    ) -> Result<usize> {
12960        let pool_meta = self.pool_meta.read().await;
12961        let reservation = pool_meta
12962            .pools
12963            .get(owner.source_pool_index)
12964            .and_then(|pool| pool.decommission.as_ref())
12965            .and_then(|info| info.capacity_reservation.as_ref())
12966            .filter(|reservation| reservation.admits_owner(owner, OffsetDateTime::now_utc()))
12967            .ok_or_else(|| decommission_capacity_blocked_error("decommission target selection reservation is stale"))?;
12968        let candidate = |target: &DecommissionCapacityTarget| {
12969            let expected_physical_bytes = capacity_target_physical_bytes(expected_data_bytes.max(1), target.layout).ok()?;
12970            let required_peak = expected_physical_bytes.saturating_mul(1usize.saturating_add(reservation.temporary_copies));
12971            let remaining = target.remaining_reserved_physical_bytes(reservation.temporary_copies);
12972            (required_peak <= remaining).then_some((target.pool_index, remaining))
12973        };
12974        if let Some(permitted_target_pool_index) = decommission_capacity_target_permit_index(self.id, owner) {
12975            if let Some((pool_index, _)) = reservation
12976                .targets
12977                .iter()
12978                .find(|target| target.pool_index == permitted_target_pool_index)
12979                .and_then(candidate)
12980            {
12981                return Ok(pool_index);
12982            }
12983            // The holder that preceded this waiter may have consumed the
12984            // remaining allocation. Release that guard before selecting a
12985            // different target so one mutation never holds two target gates.
12986            discard_decommission_capacity_target_permit_except(self.id, owner, None);
12987        }
12988        if let Some((pool_index, _)) = reservation
12989            .targets
12990            .iter()
12991            .filter_map(candidate)
12992            .max_by_key(|(_, remaining)| *remaining)
12993        {
12994            return Ok(pool_index);
12995        }
12996        if reservation.source_data_equivalent_bytes > 0 {
12997            return Err(decommission_capacity_blocked_error(format!(
12998                "source pool {} has no target allocation for {expected_data_bytes} data bytes",
12999                owner.source_pool_index
13000            )));
13001        }
13002        let capacities = self.get_decommission_all_pool_capacity_infos().await?;
13003        let active_sources = active_decommission_source_indices(&pool_meta);
13004        capacities
13005            .into_iter()
13006            .filter(|capacity| {
13007                capacity.pool_index != owner.source_pool_index
13008                    && !active_sources.contains(&capacity.pool_index)
13009                    && pool_meta
13010                        .pools
13011                        .get(capacity.pool_index)
13012                        .is_some_and(is_decommission_start_active_pool)
13013            })
13014            .max_by_key(|capacity| capacity.physical_free)
13015            .map(|capacity| capacity.pool_index)
13016            .ok_or_else(|| {
13017                decommission_capacity_blocked_error(format!(
13018                    "source pool {} has no target pool for {expected_data_bytes} data bytes",
13019                    owner.source_pool_index
13020                ))
13021            })
13022    }
13023
13024    pub(crate) async fn next_scanner_data_movement_update(&self, now: OffsetDateTime) -> OffsetDateTime {
13025        let pool_meta = self.pool_meta.read().await;
13026        let rebalance_meta = self.rebalance_meta.read().await;
13027        pool_meta.next_scanner_data_movement_update(now, rebalance_meta.as_ref())
13028    }
13029
13030    #[tracing::instrument(skip(self))]
13031    pub async fn decommission_cancel(self: &Arc<Self>, idx: usize) -> Result<()> {
13032        self.decommission_cancel_with_owner(idx, None).await
13033    }
13034
13035    async fn decommission_cancel_for_operation(self: &Arc<Self>, idx: usize, owner: &DecommissionCanceler) -> Result<()> {
13036        self.decommission_cancel_with_owner(idx, Some(owner)).await
13037    }
13038
13039    #[cfg(test)]
13040    async fn decommission_cancel_with_owner_and_save<Save, SaveFuture>(
13041        self: &Arc<Self>,
13042        idx: usize,
13043        owner: Option<&DecommissionCanceler>,
13044        save_pool_meta: Save,
13045    ) -> Result<()>
13046    where
13047        Save: FnOnce(PoolMeta, Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> SaveFuture + Send + 'static,
13048        SaveFuture: Future<Output = Result<()>> + Send + 'static,
13049    {
13050        let store = self.clone();
13051        let owner = owner.cloned();
13052        // Dropping the RPC waiter detaches this task; the transaction retains
13053        // the store and exact owner until persistence is resolved.
13054        tokio::spawn(async move { store.decommission_cancel_transaction(idx, owner, false, save_pool_meta).await })
13055            .await
13056            .map_err(|err| Error::other(format!("decommission cancel transaction task join error: {err}")))?
13057    }
13058
13059    async fn decommission_cancel_transaction<Save, SaveFuture>(
13060        &self,
13061        idx: usize,
13062        owner: Option<DecommissionCanceler>,
13063        acquire_runtime_fence: bool,
13064        save_pool_meta: Save,
13065    ) -> Result<()>
13066    where
13067        Save: FnOnce(PoolMeta, Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>) -> SaveFuture,
13068        SaveFuture: Future<Output = Result<()>>,
13069    {
13070        let owner = owner.as_ref();
13071        ensure_decommission_terminal_operation_supported(self.single_pool(), "cancel decommission")?;
13072        #[cfg(test)]
13073        if acquire_runtime_fence {
13074            pause_decommission_cancel_before_start_gate(self.id).await;
13075        }
13076        let _start_guard = self.start_gate.lock().await;
13077        // Read the fence model from durable metadata without retaining the
13078        // global lock, then fence the exact target cohort before taking the
13079        // write lock used to publish the terminal transition.
13080        let terminal_fence_plan = if acquire_runtime_fence {
13081            let read_save_guard = self.pool_meta_save_gate.lock().await;
13082            let (read_guard, snapshot) = self
13083                .acquire_pool_meta_read_guard(&read_save_guard, "decommission cancel fence planning failed")
13084                .await?;
13085            let plan = decommission_capacity_terminal_fence_plan(&snapshot, idx)?;
13086            drop(read_guard);
13087            drop(read_save_guard);
13088            plan
13089        } else {
13090            None
13091        };
13092        let _capacity_target_guards = if acquire_runtime_fence {
13093            self.acquire_decommission_capacity_terminal_guards(idx, terminal_fence_plan.as_ref())
13094                .await?
13095        } else {
13096            Vec::new()
13097        };
13098        let mut save_guard = self.pool_meta_save_gate.lock().await;
13099        let (_pool_meta_guard, mut persisted_pool_meta) = if acquire_runtime_fence {
13100            let (guard, pool_meta) = self
13101                .acquire_pool_meta_write_guard(&mut save_guard, "decommission cancel failed")
13102                .await?;
13103            (Some(guard), Some(pool_meta))
13104        } else {
13105            save_guard.ensure_write_safe("decommission cancel failed")?;
13106            (None, None)
13107        };
13108        if let Some(persisted_pool_meta) = persisted_pool_meta.as_ref() {
13109            let committed_plan = decommission_capacity_terminal_fence_plan(persisted_pool_meta, idx)?;
13110            if committed_plan != terminal_fence_plan {
13111                return Err(decommission_capacity_blocked_error(
13112                    "decommission capacity owner or target cohort changed while acquiring terminal fences",
13113                ));
13114            }
13115        }
13116        let pool_meta_fence = _pool_meta_guard
13117            .as_ref()
13118            .and_then(rustfs_lock::NamespaceLockGuard::lock_lost_signal);
13119
13120        // Lock order: start gate, target gates, save gate, distributed pool
13121        // metadata fence, rebalance_meta, decommission_cancelers, then
13122        // pool_meta. The state guards stay held across persistence so the
13123        // active generation cannot change before the cancel is published.
13124        let rebalance_meta = self.rebalance_meta.read().await.clone();
13125        let terminal_at = OffsetDateTime::now_utc();
13126        let mut cancelers = self.decommission_cancelers.write().await;
13127        let mut pool_meta = self.pool_meta.write().await;
13128        if acquire_runtime_fence {
13129            let local_plan = decommission_capacity_terminal_fence_plan(&pool_meta, idx)?;
13130            if local_plan != terminal_fence_plan {
13131                return Err(decommission_capacity_blocked_error(
13132                    "local decommission capacity owner differs from the durable terminal fence plan",
13133                ));
13134            }
13135        }
13136        let (pending, should_reload_pool_meta, already_canceled, terminal_canceler, durable_movement_generation) = {
13137            let mut already_canceled = false;
13138            let (pool_present, decommission_present, terminal) = if let Some(pool) = pool_meta.pools.get(idx) {
13139                if let Some(info) = pool.decommission.as_ref() {
13140                    already_canceled = info.canceled;
13141                    (
13142                        true,
13143                        info.has_decommission_state(),
13144                        should_reject_decommission_cancel_as_terminal(info.complete, info.failed),
13145                    )
13146                } else {
13147                    (true, false, false)
13148                }
13149            } else {
13150                (false, false, false)
13151            };
13152
13153            ensure_decommission_cancel_allowed(pool_present, decommission_present, terminal)?;
13154            let previous_pool = pool_meta
13155                .pools
13156                .get(idx)
13157                .cloned()
13158                .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), idx))?;
13159            let previous_decommission = previous_pool
13160                .decommission
13161                .as_ref()
13162                .ok_or_else(|| decommission_metadata_not_initialized_error("cancel decommission"))?;
13163            let mut snapshot = pool_meta.clone();
13164            let Some(changed) = update_decommission_for_operation(cancelers.as_slice(), &mut snapshot, idx, owner, |pool_meta| {
13165                pool_meta.decommission_cancel_at(idx, terminal_at, rebalance_meta.as_ref())
13166            }) else {
13167                return Ok(());
13168            };
13169            let pending = if changed {
13170                let canceled_pool = snapshot
13171                    .pools
13172                    .get(idx)
13173                    .cloned()
13174                    .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), idx))?;
13175                if let Some(persisted) = persisted_pool_meta.as_mut() {
13176                    merge_pool_meta_updates_for_save(persisted, &snapshot, &[idx], "decommission cancel failed")?;
13177                    snapshot = persisted.clone();
13178                }
13179                Some((
13180                    snapshot,
13181                    DecommissionCancelCommit {
13182                        previous_start_time: previous_decommission.start_time,
13183                        previous_queued: previous_decommission.queued,
13184                        previous_last_update: previous_pool.last_update,
13185                        canceled_pool,
13186                    },
13187                ))
13188            } else {
13189                None
13190            };
13191            let terminal_canceler = if let Some(owner) = owner {
13192                Some(owner.clone())
13193            } else {
13194                cancelers.get(idx).and_then(Option::as_ref).cloned()
13195            };
13196            let durable_movement_generation = pending
13197                .as_ref()
13198                .map(|(_, commit)| crate::store::scanner_data_movement_timestamp_generation(commit.canceled_pool.last_update))
13199                .unwrap_or_default();
13200            (
13201                pending,
13202                should_retry_decommission_cancel_reload(changed, already_canceled),
13203                already_canceled,
13204                terminal_canceler,
13205                durable_movement_generation,
13206            )
13207        };
13208        let active_worker = terminal_canceler.as_ref().is_some_and(DecommissionCanceler::is_active);
13209        if !active_worker && !already_canceled {
13210            warn!(
13211                event = EVENT_DECOMMISSION_STATE,
13212                component = LOG_COMPONENT_ECSTORE,
13213                subsystem = LOG_SUBSYSTEM_POOLS,
13214                pool_index = idx,
13215                state = "cancel_skipped",
13216                reason = "no_active_canceler",
13217                "Decommission cancel skipped"
13218            );
13219        }
13220
13221        let changed = pending.is_some();
13222        let commit_result = if let Some((snapshot, commit)) = pending {
13223            if let Err(err) = save_pool_meta(snapshot, pool_meta_fence).await {
13224                save_guard.block_writes();
13225                return Err(err);
13226            }
13227            if let Some(pool_meta_guard) = _pool_meta_guard.as_ref()
13228                && let Err(err) = ensure_pool_meta_write_fence(pool_meta_guard, "decommission cancel failed")
13229            {
13230                save_guard.block_writes();
13231                return Err(err);
13232            }
13233            commit_decommission_cancel(&mut pool_meta, idx, commit)
13234        } else {
13235            Ok(())
13236        };
13237
13238        commit_result?;
13239        if let Some(pool_meta_guard) = _pool_meta_guard.as_ref()
13240            && let Err(err) = ensure_pool_meta_write_fence(pool_meta_guard, "decommission cancel failed")
13241        {
13242            save_guard.block_writes();
13243            return Err(err);
13244        }
13245        if let Some(canceler) = terminal_canceler.as_ref() {
13246            take_and_cancel_decommission_canceler_for_operation(cancelers.as_mut_slice(), idx, canceler);
13247        }
13248        drop(pool_meta);
13249        drop(cancelers);
13250        drop(_pool_meta_guard);
13251        drop(save_guard);
13252
13253        if changed {
13254            // Persistence must commit before the cancellation signal. Wait for
13255            // in-flight movement only after signaling so readers can release
13256            // the shared gate without deadlocking the terminal transition.
13257            let movement_gate = self.ctx.data_movement_operation_gate();
13258            let _movement_guard = movement_gate.write().await;
13259            self.ctx
13260                .advance_data_movement_operation_epoch_to_durable_generation(durable_movement_generation);
13261        }
13262
13263        if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
13264            let stage = format!("decommission_cancel for pool {idx}");
13265            if let Err(err) =
13266                resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
13267            {
13268                warn!(
13269                    event = EVENT_DECOMMISSION_STATE,
13270                    component = LOG_COMPONENT_ECSTORE,
13271                    subsystem = LOG_SUBSYSTEM_POOLS,
13272                    pool_index = idx,
13273                    state = "terminal_reload_failed",
13274                    error = %err,
13275                    "Decommission cancel saved locally but pool meta reload failed"
13276                );
13277            }
13278        }
13279
13280        Ok(())
13281    }
13282
13283    async fn release_decommission_canceler_slot(&self, idx: usize, owner: &DecommissionCanceler) {
13284        let mut cancelers = self.decommission_cancelers.write().await;
13285        take_and_cancel_decommission_canceler_for_operation(cancelers.as_mut_slice(), idx, owner);
13286    }
13287
13288    async fn decommission_terminal_retryable_for_operation(&self, idx: usize, owner: &DecommissionCanceler) -> bool {
13289        let _start_guard = self.start_gate.lock().await;
13290        let mut cancelers = self.decommission_cancelers.write().await;
13291        if !decommission_canceler_is_owned_by(cancelers.as_slice(), idx, owner) {
13292            owner.release();
13293            return false;
13294        }
13295
13296        let retryable = {
13297            let pool_meta = self.pool_meta.read().await;
13298            pool_meta
13299                .pools
13300                .get(idx)
13301                .and_then(|pool| pool.decommission.as_ref())
13302                .is_some_and(|info| info.has_decommission_state() && !info.complete && !info.failed && !info.canceled)
13303        };
13304        if !retryable {
13305            take_and_cancel_decommission_canceler_for_operation(cancelers.as_mut_slice(), idx, owner);
13306        }
13307        retryable
13308    }
13309
13310    async fn retry_decommission_cancel_for_operation(self: &Arc<Self>, idx: usize, owner: &DecommissionCanceler) {
13311        if !self.decommission_terminal_retryable_for_operation(idx, owner).await {
13312            return;
13313        }
13314
13315        let mut attempt = 0usize;
13316        loop {
13317            if self
13318                .ensure_pool_meta_side_effects_safe("decommission cancel retry paused because pool metadata requires recovery")
13319                .await
13320                .is_err()
13321            {
13322                self.release_decommission_canceler_slot(idx, owner).await;
13323                return;
13324            }
13325            let Err(err) = self.decommission_cancel_for_operation(idx, owner).await else {
13326                return;
13327            };
13328            if !self.decommission_terminal_retryable_for_operation(idx, owner).await {
13329                return;
13330            }
13331            attempt += 1;
13332            warn!(
13333                event = EVENT_DECOMMISSION_STATE,
13334                component = LOG_COMPONENT_ECSTORE,
13335                subsystem = LOG_SUBSYSTEM_POOLS,
13336                pool_index = idx,
13337                state = "terminal_save_retry",
13338                terminal = "canceled",
13339                attempt,
13340                error = %err,
13341                "Decommission terminal save will be retried"
13342            );
13343            tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await;
13344        }
13345    }
13346
13347    async fn retry_decommission_failed_for_operation(&self, idx: usize, owner: &DecommissionCanceler) {
13348        let mut attempt = 0usize;
13349        loop {
13350            if self
13351                .ensure_pool_meta_side_effects_safe("decommission failure retry paused because pool metadata requires recovery")
13352                .await
13353                .is_err()
13354            {
13355                self.release_decommission_canceler_slot(idx, owner).await;
13356                return;
13357            }
13358            let Err(err) = self.decommission_failed_for_operation(idx, owner).await else {
13359                return;
13360            };
13361            if !self.decommission_terminal_retryable_for_operation(idx, owner).await {
13362                return;
13363            }
13364            attempt += 1;
13365            warn!(
13366                event = EVENT_DECOMMISSION_STATE,
13367                component = LOG_COMPONENT_ECSTORE,
13368                subsystem = LOG_SUBSYSTEM_POOLS,
13369                pool_index = idx,
13370                state = "terminal_save_retry",
13371                terminal = "failed",
13372                attempt,
13373                error = %err,
13374                "Decommission terminal save will be retried"
13375            );
13376            tokio::time::sleep(DECOMMISSION_TERMINAL_RETRY_DELAY).await;
13377        }
13378    }
13379
13380    async fn decommission_cancel_with_owner(self: &Arc<Self>, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> {
13381        let pools = self.pools.clone();
13382        let store = self.clone();
13383        let owner = owner.cloned();
13384        tokio::spawn(async move {
13385            store
13386                .decommission_cancel_transaction(idx, owner, true, move |snapshot, pool_meta_fence| async move {
13387                    snapshot.save_no_lock_with_fence(pools, pool_meta_fence, &[idx]).await
13388                })
13389                .await
13390        })
13391        .await
13392        .map_err(|err| Error::other(format!("decommission cancel transaction task join error: {err}")))?
13393    }
13394
13395    #[cfg(test)]
13396    async fn clear_decommission_with_save<Save, SaveFuture>(self: &Arc<Self>, idx: usize, save_pool_meta: Save) -> Result<()>
13397    where
13398        Save: FnOnce() -> SaveFuture + Send + 'static,
13399        SaveFuture: Future<Output = Result<()>> + Send + 'static,
13400    {
13401        let store = self.clone();
13402        tokio::spawn(async move { store.clear_decommission_transaction(idx, save_pool_meta).await })
13403            .await
13404            .map_err(|err| Error::other(format!("clear decommission transaction task join error: {err}")))?
13405    }
13406
13407    #[tracing::instrument(skip(self))]
13408    pub async fn clear_decommission(self: &Arc<Self>, idx: usize) -> Result<()> {
13409        let store = self.clone();
13410        let save_store = store.clone();
13411        // Dropping the RPC waiter detaches this task; once in-memory state can
13412        // change, the transaction must persist or roll it back before ending.
13413        tokio::spawn(async move {
13414            store
13415                .clear_decommission_transaction(idx, move || async move { save_store.save_current_pool_meta(&[idx]).await })
13416                .await
13417        })
13418        .await
13419        .map_err(|err| Error::other(format!("clear decommission transaction task join error: {err}")))?
13420    }
13421
13422    async fn clear_decommission_transaction<Save, SaveFuture>(&self, idx: usize, save_pool_meta: Save) -> Result<()>
13423    where
13424        Save: FnOnce() -> SaveFuture,
13425        SaveFuture: Future<Output = Result<()>>,
13426    {
13427        ensure_decommission_terminal_operation_supported(self.single_pool(), "clear decommission")?;
13428        let _start_guard = self.start_gate.lock().await;
13429
13430        {
13431            let pool_meta = self.pool_meta.read().await;
13432            let pool_count = pool_meta.pools.len();
13433            ensure_valid_decommission_pool_index(pool_count, idx)?;
13434            let Some(pool) = pool_meta.pools.get(idx) else {
13435                return Err(invalid_decommission_pool_index_error(pool_count, idx));
13436            };
13437            let (decommission_present, complete, failed, canceled, unresolved_entries) = pool
13438                .decommission
13439                .as_ref()
13440                .map(|info| {
13441                    (
13442                        info.has_decommission_state(),
13443                        info.complete,
13444                        info.failed,
13445                        info.canceled,
13446                        info.unresolved_entries.len(),
13447                    )
13448                })
13449                .unwrap_or((false, false, false, false, 0));
13450            ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled, unresolved_entries)?;
13451        }
13452        // Cancel workers before waiting for the movement writer so active
13453        // object operations can observe the signal and release read guards.
13454        self.cancel_decommission_routines(&[idx]).await;
13455        let movement_gate = self.ctx.data_movement_operation_gate();
13456        let _movement_guard = movement_gate.write().await;
13457
13458        let (should_reload_pool_meta, previous_pool_meta) = {
13459            let rebalance_meta = self.rebalance_meta.read().await.clone();
13460            let terminal_at = OffsetDateTime::now_utc();
13461            let mut pool_meta = self.pool_meta.write().await;
13462            let previous_pool_meta = pool_meta.clone();
13463            let changed = pool_meta.clear_decommission_at(idx, terminal_at, rebalance_meta.as_ref())?;
13464            (changed, changed.then_some(previous_pool_meta))
13465        };
13466
13467        if should_reload_pool_meta && let Err(err) = save_pool_meta().await {
13468            if let Some(previous_pool_meta) = previous_pool_meta {
13469                let mut pool_meta = self.pool_meta.write().await;
13470                rollback_decommission_pool_meta(&mut pool_meta, &previous_pool_meta, &[idx]);
13471            }
13472            return Err(err);
13473        }
13474
13475        if should_reload_pool_meta {
13476            self.ctx.advance_data_movement_operation_epoch();
13477        }
13478        drop(_movement_guard);
13479
13480        if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
13481            let stage = format!("clear_decommission for pool {idx}");
13482            if let Err(err) =
13483                resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
13484            {
13485                warn!(
13486                    event = EVENT_DECOMMISSION_STATE,
13487                    component = LOG_COMPONENT_ECSTORE,
13488                    subsystem = LOG_SUBSYSTEM_POOLS,
13489                    pool_index = idx,
13490                    state = "terminal_reload_failed",
13491                    error = %err,
13492                    "Decommission clear saved locally but pool meta reload failed"
13493                );
13494            }
13495        }
13496
13497        Ok(())
13498    }
13499
13500    async fn promote_queued_decommission(&self, idx: usize, owner: &DecommissionCanceler) -> Result<OffsetDateTime> {
13501        // Serialize promotion and generation capture with clear/restart transitions.
13502        let (changed, generation, save_error) = {
13503            let _start_guard = self.start_gate.lock().await;
13504            let movement_gate = self.ctx.data_movement_operation_gate();
13505            let _movement_guard = movement_gate.write().await;
13506            let mut save_guard = self.pool_meta_save_gate.lock().await;
13507            let (pool_meta_guard, mut snapshot) = self
13508                .acquire_pool_meta_write_guard(&mut save_guard, "decommission promotion failed")
13509                .await?;
13510            let rebalance_meta = self.rebalance_meta.read().await.clone();
13511            let capacity_infos = if self.pools.is_empty() {
13512                Vec::new()
13513            } else {
13514                self.get_decommission_all_pool_capacity_infos_with_owned_usage().await?
13515            };
13516            let target_fence_proof = crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof();
13517            let mut pool_meta = self.pool_meta.write().await;
13518            if pool_meta.pools.get(idx).is_none() {
13519                return Err(Error::other("failed to start decommission: target pool was not found"));
13520            }
13521            let capacity_recovery_needed = active_decommission_source_indices(&pool_meta).into_iter().any(|source_idx| {
13522                pool_meta
13523                    .pools
13524                    .get(source_idx)
13525                    .and_then(|pool| pool.decommission.as_ref())
13526                    .and_then(|info| info.capacity_reservation.as_ref())
13527                    .is_none_or(|reservation| !reservation.active())
13528            });
13529            let capacity_indices = if capacity_infos.is_empty() {
13530                Vec::new()
13531            } else {
13532                recover_decommission_capacity_reservations(
13533                    &mut pool_meta,
13534                    &capacity_infos,
13535                    OffsetDateTime::now_utc(),
13536                    target_fence_proof.is_some(),
13537                )?
13538            };
13539            let target_fence_proof_required = capacity_recovery_needed
13540                && active_decommission_capacity_model(&pool_meta)? == Some(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION);
13541            ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?;
13542            let reconciled = reconcile_decommission_meta_buckets(&mut pool_meta, idx);
13543            let promoted = pool_meta.promote_queued_decommission_at(idx, OffsetDateTime::now_utc(), rebalance_meta.as_ref());
13544            let mut changed_indices = capacity_indices;
13545            let changed = !changed_indices.is_empty() || reconciled || promoted;
13546            if changed {
13547                if !changed_indices.contains(&idx) {
13548                    changed_indices.push(idx);
13549                }
13550                merge_pool_meta_updates_for_save(&mut snapshot, &pool_meta, &changed_indices, "decommission promotion failed")?;
13551            }
13552            drop(pool_meta);
13553
13554            let (save_outcome, save_error) = if changed {
13555                ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?;
13556                match snapshot
13557                    .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &changed_indices)
13558                    .await
13559                {
13560                    Ok(outcome) => (Some(outcome), None),
13561                    Err(err) => (None, Some(err)),
13562                }
13563            } else {
13564                (None, None)
13565            };
13566            let generation = self.active_decommission_generation(idx).await?;
13567            ensure_pool_meta_write_fence(&pool_meta_guard, "decommission promotion failed")?;
13568            ensure_decommission_target_fence_fleet_proof(target_fence_proof.as_ref(), target_fence_proof_required)?;
13569            if let Some(outcome) = save_outcome {
13570                let mut pool_meta = self.pool_meta.write().await;
13571                pool_meta.version = pool_meta.version.max(outcome.committed.version);
13572                drop(pool_meta);
13573                outcome.disarm();
13574            }
13575            (changed, generation, save_error)
13576        };
13577
13578        if let Some(err) = save_error {
13579            resolve_decommission_terminal_mark_after_error_result(
13580                self.decommission_failed_for_operation(idx, owner).await,
13581                idx,
13582                &err,
13583            )?;
13584            return Err(err);
13585        }
13586
13587        if changed {
13588            self.ctx.advance_data_movement_operation_epoch();
13589        }
13590        if changed && let Some(notification_sys) = runtime_sources::notification_sys() {
13591            let stage = format!("promote_queued_decommission for pool {idx}");
13592            if let Err(err) =
13593                resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str())
13594            {
13595                resolve_decommission_terminal_mark_after_error_result(
13596                    self.decommission_failed_for_operation(idx, owner).await,
13597                    idx,
13598                    &err,
13599                )?;
13600                return Err(err);
13601            }
13602        }
13603
13604        Ok(generation)
13605    }
13606
13607    #[cfg(test)]
13608    pub(crate) async fn promote_queued_decommission_for_test(&self, idx: usize) -> Result<()> {
13609        let owner = DecommissionCanceler::new(CancellationToken::new());
13610        self.promote_queued_decommission(idx, &owner).await?;
13611        let mut cancelers = self.decommission_cancelers.write().await;
13612        if let Some(slot) = cancelers.get_mut(idx)
13613            && let Some(previous) = slot.replace(owner)
13614        {
13615            previous.release();
13616        }
13617        Ok(())
13618    }
13619
13620    async fn record_decommission_terminal_reload_failure(&self, idx: usize, stage: &str, err: Error) -> Result<()> {
13621        let movement_gate = self.ctx.data_movement_operation_gate();
13622        let _movement_guard = movement_gate.write().await;
13623        let changed = {
13624            let rebalance_meta = self.rebalance_meta.read().await.clone();
13625            let mut pool_meta = self.pool_meta.write().await;
13626            pool_meta.record_decommission_terminal_reload_failure_at(
13627                idx,
13628                stage,
13629                err.to_string(),
13630                OffsetDateTime::now_utc(),
13631                rebalance_meta.as_ref(),
13632            )?
13633        };
13634
13635        if changed {
13636            self.save_current_pool_meta(&[idx]).await?;
13637        }
13638
13639        Ok(())
13640    }
13641
13642    async fn pause_decommission_for_capacity(&self, idx: usize, err: &Error) -> Result<()> {
13643        let mut save_guard = self.pool_meta_save_gate.lock().await;
13644        let (pool_meta_guard, mut snapshot) = self
13645            .acquire_pool_meta_write_guard(&mut save_guard, "decommission capacity pause failed")
13646            .await?;
13647        snapshot.mark_decommission_capacity_blocked(idx, err.to_string(), OffsetDateTime::now_utc())?;
13648        let outcome = snapshot
13649            .save_no_lock_armed(self.pools.clone(), &mut save_guard, pool_meta_guard.lock_lost_signal(), &[idx])
13650            .await?;
13651        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission capacity pause failed")?;
13652        {
13653            let mut pool_meta = self.pool_meta.write().await;
13654            publish_pool_meta_updates(&mut pool_meta, &outcome.committed, &[idx]);
13655        }
13656        ensure_pool_meta_write_fence(&pool_meta_guard, "decommission capacity pause failed")?;
13657        outcome.disarm();
13658        Ok(())
13659    }
13660
13661    pub async fn is_decommission_running(&self) -> bool {
13662        {
13663            let cancelers = self.decommission_cancelers.read().await;
13664            if has_active_decommission_canceler(cancelers.as_slice()) {
13665                return true;
13666            }
13667        }
13668
13669        let pool_meta = self.pool_meta.read().await;
13670        for pool in pool_meta.pools.iter() {
13671            if let Some(ref info) = pool.decommission
13672                && info.has_decommission_state()
13673                && !info.complete
13674                && !info.failed
13675                && !info.canceled
13676            {
13677                return true;
13678            }
13679        }
13680
13681        false
13682    }
13683
13684    async fn decommission_cancel_requested(&self, idx: usize, rx: &CancellationToken) -> bool {
13685        let pool_meta = self.pool_meta.read().await;
13686        is_decommission_cancel_requested(rx.is_cancelled(), pool_meta.pools.get(idx))
13687    }
13688
13689    #[cfg(test)]
13690    async fn cancel_decommission_routines_and_wait(&self, indices: &[usize]) {
13691        self.cancel_decommission_routines(indices).await;
13692        let movement_gate = self.ctx.data_movement_operation_gate();
13693        let _movement_guard = movement_gate.write().await;
13694    }
13695
13696    async fn cancel_decommission_routines(&self, indices: &[usize]) {
13697        {
13698            let mut cancelers = self.decommission_cancelers.write().await;
13699            for idx in indices {
13700                take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), *idx);
13701            }
13702        }
13703    }
13704
13705    async fn quiesce_decommission_worker_after_join_error(&self, canceler: &DecommissionCanceler) {
13706        canceler.cancel();
13707        let movement_gate = self.ctx.data_movement_operation_gate();
13708        let _movement_guard = movement_gate.write().await;
13709    }
13710
13711    async fn reserve_decommission_routines(
13712        &self,
13713        rx: &CancellationToken,
13714        indices: &[usize],
13715    ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
13716        let indices = dedup_indices(indices);
13717        if indices.is_empty() {
13718            return Ok(Vec::new());
13719        }
13720
13721        let _start_guard = self.start_gate.lock().await;
13722        let save_guard = self.pool_meta_save_gate.lock().await;
13723        save_guard.ensure_write_safe("decommission cannot be scheduled while pool metadata requires recovery")?;
13724        let indices = {
13725            let pool_meta = self.pool_meta.read().await;
13726            let indices = resumable_decommission_queue_indices(&pool_meta)
13727                .into_iter()
13728                .filter(|idx| indices.contains(idx))
13729                .collect::<Vec<_>>();
13730            if !indices.is_empty() {
13731                ensure_decommission_ledger_persistence_supported(&pool_meta)?;
13732            }
13733            indices
13734        };
13735        if indices.is_empty() {
13736            return Ok(Vec::new());
13737        }
13738
13739        let index_cancelers = {
13740            let mut cancelers = self.decommission_cancelers.write().await;
13741            let missing = missing_decommission_worker_prefix(indices.as_slice(), cancelers.as_slice());
13742            if missing.is_empty() {
13743                return Ok(Vec::new());
13744            }
13745            let bound = bind_missing_decommission_cancelers(missing.as_slice(), rx, cancelers.as_mut_slice());
13746            let guards = guard_decommission_cancelers(bound);
13747            ensure_decommission_routines_scheduled(guards.len(), missing.len())?;
13748            guards
13749        };
13750        Ok(index_cancelers)
13751    }
13752
13753    async fn reserve_missing_local_decommission_routines(
13754        &self,
13755        rx: &CancellationToken,
13756        endpoints: &EndpointServerPools,
13757    ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
13758        let indices = {
13759            let pool_meta = self.pool_meta.read().await;
13760            resumable_decommission_queue_indices(&pool_meta)
13761        };
13762        let indices = local_decommission_queue_prefix(endpoints, &indices)?;
13763        self.reserve_decommission_routines(rx, indices.as_slice()).await
13764    }
13765
13766    pub async fn spawn_missing_local_decommission_routines(self: &Arc<Self>) -> Result<()> {
13767        self.spawn_missing_local_decommission_routines_with_token(CancellationToken::new())
13768            .await
13769    }
13770
13771    pub(crate) async fn has_active_local_decommission_worker(&self) -> bool {
13772        let cancelers = self.decommission_cancelers.read().await;
13773        has_active_decommission_canceler(cancelers.as_slice())
13774    }
13775
13776    pub(crate) async fn spawn_missing_local_decommission_routines_with_token(
13777        self: &Arc<Self>,
13778        rx: CancellationToken,
13779    ) -> Result<()> {
13780        #[cfg(test)]
13781        let endpoints = self.instance_endpoints().unwrap_or_else(|| self.endpoints());
13782        #[cfg(not(test))]
13783        let endpoints = self.endpoints();
13784        let index_cancelers = self.reserve_missing_local_decommission_routines(&rx, &endpoints).await?;
13785        if index_cancelers.is_empty() {
13786            return Ok(());
13787        }
13788
13789        drop(spawn_decommission_index_cancelers(
13790            self.clone(),
13791            rx,
13792            index_cancelers,
13793            Arc::new(Semaphore::new(decommission_entry_concurrency_limit())),
13794        ));
13795        Ok(())
13796    }
13797
13798    #[tracing::instrument(skip(self, rx))]
13799    pub async fn decommission(&self, rx: CancellationToken, indices: Vec<usize>) -> Result<()> {
13800        let indices = dedup_indices(&indices);
13801
13802        info!(
13803            event = EVENT_DECOMMISSION_STATE,
13804            component = LOG_COMPONENT_ECSTORE,
13805            subsystem = LOG_SUBSYSTEM_POOLS,
13806            pool_indices = ?indices,
13807            state = "requested",
13808            "Decommission requested"
13809        );
13810        validate_start_decommission_request(&indices, self.single_pool())?;
13811
13812        self.ensure_decommission_rebalance_idle_after_refresh().await?;
13813
13814        let store = require_decommission_store(runtime_sources::object_store_handle(), "start decommission")?;
13815        let local_indices = local_decommission_queue_prefix(&self.endpoints(), &indices)?;
13816        let index_cancelers = self
13817            .start_decommission_with_routines(indices, &rx, local_indices.as_slice())
13818            .await?;
13819        drop(spawn_decommission_index_cancelers(
13820            store,
13821            rx,
13822            index_cancelers,
13823            Arc::new(Semaphore::new(decommission_entry_concurrency_limit())),
13824        ));
13825
13826        Ok(())
13827    }
13828
13829    async fn active_decommission_generation(&self, idx: usize) -> Result<OffsetDateTime> {
13830        let pool_meta = self.pool_meta.read().await;
13831        let Some(pool) = pool_meta.pools.get(idx) else {
13832            return Err(invalid_decommission_pool_index_error(pool_meta.pools.len(), idx));
13833        };
13834        let Some(info) = pool.decommission.as_ref() else {
13835            return Err(decommission_metadata_not_initialized_error("load decommission generation"));
13836        };
13837        let Some(generation) = info.start_time else {
13838            return Err(Error::OperationCanceled);
13839        };
13840        ensure_decommission_generation(&pool_meta, idx, generation)?;
13841        Ok(generation)
13842    }
13843
13844    async fn ensure_decommission_generation_current(&self, idx: usize, generation: OffsetDateTime) -> Result<()> {
13845        let pool_meta = self.pool_meta.read().await;
13846        ensure_decommission_generation(&pool_meta, idx, generation)
13847    }
13848
13849    #[allow(clippy::too_many_arguments)]
13850    async fn decommission_entry_worker(
13851        self: Arc<Self>,
13852        rx: CancellationToken,
13853        idx: usize,
13854        set_idx: usize,
13855        generation: OffsetDateTime,
13856        bucket: String,
13857        set: Arc<SetDisks>,
13858        lifecycle_config: Option<BucketLifecycleConfiguration>,
13859        object_lock_config: Option<ObjectLockConfiguration>,
13860        replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
13861        expected_bucket_incarnation_id: Option<uuid::Uuid>,
13862        source_changed_exhaustions: Arc<AtomicUsize>,
13863        entry_budget: Arc<Semaphore>,
13864        queue: Arc<tokio::sync::Mutex<mpsc::Receiver<QueuedDecommissionEntry>>>,
13865        deferred: Arc<tokio::sync::Mutex<Vec<DeferredDecommissionEntry>>>,
13866        entry_error: Arc<tokio::sync::Mutex<Option<Error>>>,
13867    ) {
13868        loop {
13869            let queued = tokio::select! {
13870                biased;
13871                _ = rx.cancelled() => return,
13872                item = async {
13873                    let mut queue = queue.lock().await;
13874                    queue.recv().await
13875                } => item,
13876            };
13877            let Some(QueuedDecommissionEntry { entry, queue_permit }) = queued else {
13878                return;
13879            };
13880            // Only listing entries retain metadata that distinguishes prefixes
13881            // from real directory objects. Deferred names are already objects.
13882            if entry.is_dir() {
13883                continue;
13884            }
13885            let object_name = entry.name.clone();
13886            let mut attempts = 0;
13887
13888            if entry_error.lock().await.is_some() {
13889                drop(queue_permit);
13890                continue;
13891            }
13892
13893            if let Err(err) = self.ensure_decommission_generation_current(idx, generation).await {
13894                if matches!(err, Error::OperationCanceled) {
13895                    rx.cancel();
13896                } else {
13897                    record_decommission_entry_error(&entry_error, &rx, err).await;
13898                }
13899                return;
13900            }
13901
13902            if let Err(err) = backpressure::wait_for_data_movement_admission(DataMovementOperation::Decommission, idx, &rx).await
13903            {
13904                if matches!(err, Error::OperationCanceled) {
13905                    return;
13906                }
13907                error!(
13908                    event = EVENT_DECOMMISSION_ENTRY,
13909                    component = LOG_COMPONENT_ECSTORE,
13910                    subsystem = LOG_SUBSYSTEM_POOLS,
13911                    pool_index = idx,
13912                    set_index = set_idx,
13913                    bucket = %bucket,
13914                    object = %object_name,
13915                    state = "entry_admission_failed",
13916                    error = %err,
13917                    "Decommission entry admission failed"
13918                );
13919                record_decommission_entry_error(&entry_error, &rx, err).await;
13920                return;
13921            }
13922
13923            let entry_budget_permit = match tokio::select! {
13924                biased;
13925                _ = rx.cancelled() => return,
13926                permit = entry_budget.clone().acquire_owned() => permit,
13927            } {
13928                Ok(permit) => permit,
13929                Err(err) => {
13930                    let err = decommission_entry_budget_acquire_error(err);
13931                    error!(
13932                        event = EVENT_DECOMMISSION_ENTRY,
13933                        component = LOG_COMPONENT_ECSTORE,
13934                        subsystem = LOG_SUBSYSTEM_POOLS,
13935                        pool_index = idx,
13936                        set_index = set_idx,
13937                        bucket = %bucket,
13938                        object = %object_name,
13939                        state = "entry_budget_acquire_failed",
13940                        error = %err,
13941                        "Decommission entry budget permit acquire failed"
13942                    );
13943                    record_decommission_entry_error(&entry_error, &rx, err).await;
13944                    return;
13945                }
13946            };
13947
13948            let result = self
13949                .decommission_entry(
13950                    rx.clone(),
13951                    idx,
13952                    generation,
13953                    entry.clone(),
13954                    bucket.clone(),
13955                    set.clone(),
13956                    lifecycle_config.clone(),
13957                    object_lock_config.clone(),
13958                    replication_config.clone(),
13959                    expected_bucket_incarnation_id,
13960                    Arc::clone(&source_changed_exhaustions),
13961                    &mut attempts,
13962                )
13963                .await;
13964            drop(entry_budget_permit);
13965
13966            match result {
13967                Ok(DecommissionEntryOutcome::Complete) => drop(queue_permit),
13968                Ok(DecommissionEntryOutcome::Deferred(err)) => {
13969                    // Release the outstanding permit before deferring: the
13970                    // listing drain waits for the full budget to return, and the
13971                    // listing has already finished for this round.
13972                    drop(queue_permit);
13973                    // Only the name and attempt count are retained. A contention streak
13974                    // would otherwise pin one metadata buffer per object, and the
13975                    // replay reloads the versions it needs from disk anyway.
13976                    deferred.lock().await.push(DeferredDecommissionEntry {
13977                        name: object_name,
13978                        attempts,
13979                    });
13980                    info!(
13981                        event = EVENT_DECOMMISSION_ENTRY,
13982                        component = LOG_COMPONENT_ECSTORE,
13983                        subsystem = LOG_SUBSYSTEM_POOLS,
13984                        pool_index = idx,
13985                        set_index = set_idx,
13986                        bucket = %bucket,
13987                        object = %entry.name,
13988                        state = "entry_deferred_to_round_end",
13989                        error = %err,
13990                        "Decommission entry deferred to the end of the current round"
13991                    );
13992                }
13993                Err(err) => {
13994                    drop(queue_permit);
13995                    error!(
13996                        event = EVENT_DECOMMISSION_ENTRY,
13997                        component = LOG_COMPONENT_ECSTORE,
13998                        subsystem = LOG_SUBSYSTEM_POOLS,
13999                        pool_index = idx,
14000                        set_index = set_idx,
14001                        bucket = %bucket,
14002                        object = %object_name,
14003                        state = "entry_failed",
14004                        error = %err,
14005                        "Decommission entry failed"
14006                    );
14007                    record_decommission_entry_error(&entry_error, &rx, err).await;
14008                    return;
14009                }
14010            }
14011        }
14012    }
14013
14014    /// Run one deferred entry after the round's listing drained.
14015    ///
14016    /// The worker loop owns deferral, so this path must never defer again: any
14017    /// remaining contention is terminal for the round and fails the bucket.
14018    #[allow(clippy::too_many_arguments)]
14019    async fn replay_deferred_decommission_entry(
14020        self: Arc<Self>,
14021        rx: CancellationToken,
14022        idx: usize,
14023        set_idx: usize,
14024        generation: OffsetDateTime,
14025        mut deferred_entry: DeferredDecommissionEntry,
14026        bucket: String,
14027        set: Arc<SetDisks>,
14028        lifecycle_config: Option<BucketLifecycleConfiguration>,
14029        object_lock_config: Option<ObjectLockConfiguration>,
14030        replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
14031        expected_bucket_incarnation_id: Option<uuid::Uuid>,
14032        source_changed_exhaustions: Arc<AtomicUsize>,
14033        entry_budget: Arc<Semaphore>,
14034        entry_error: Arc<tokio::sync::Mutex<Option<Error>>>,
14035    ) -> Result<DecommissionEntryOutcome> {
14036        let object_name = deferred_entry.name;
14037        let entry = MetaCacheEntry {
14038            name: object_name.clone(),
14039            ..Default::default()
14040        };
14041
14042        if entry_error.lock().await.is_some() || rx.is_cancelled() {
14043            return Ok(DecommissionEntryOutcome::Complete);
14044        }
14045
14046        if let Err(err) = self.ensure_decommission_generation_current(idx, generation).await {
14047            if matches!(err, Error::OperationCanceled) {
14048                rx.cancel();
14049            } else {
14050                record_decommission_entry_error(&entry_error, &rx, err).await;
14051            }
14052            return Ok(DecommissionEntryOutcome::Complete);
14053        }
14054
14055        if let Err(err) = backpressure::wait_for_data_movement_admission(DataMovementOperation::Decommission, idx, &rx).await {
14056            if matches!(err, Error::OperationCanceled) {
14057                return Ok(DecommissionEntryOutcome::Complete);
14058            }
14059            error!(
14060                event = EVENT_DECOMMISSION_ENTRY,
14061                component = LOG_COMPONENT_ECSTORE,
14062                subsystem = LOG_SUBSYSTEM_POOLS,
14063                pool_index = idx,
14064                set_index = set_idx,
14065                bucket = %bucket,
14066                object = %object_name,
14067                state = "entry_admission_failed",
14068                error = %err,
14069                "Decommission entry admission failed"
14070            );
14071            record_decommission_entry_error(&entry_error, &rx, err).await;
14072            return Ok(DecommissionEntryOutcome::Complete);
14073        }
14074
14075        let entry_budget_permit = match tokio::select! {
14076            biased;
14077            _ = rx.cancelled() => return Ok(DecommissionEntryOutcome::Complete),
14078            permit = entry_budget.clone().acquire_owned() => permit,
14079        } {
14080            Ok(permit) => permit,
14081            Err(err) => {
14082                let err = decommission_entry_budget_acquire_error(err);
14083                record_decommission_entry_error(&entry_error, &rx, err).await;
14084                return Ok(DecommissionEntryOutcome::Complete);
14085            }
14086        };
14087
14088        let result = self
14089            .decommission_entry(
14090                rx.clone(),
14091                idx,
14092                generation,
14093                entry,
14094                bucket.clone(),
14095                set,
14096                lifecycle_config,
14097                object_lock_config,
14098                replication_config,
14099                expected_bucket_incarnation_id,
14100                source_changed_exhaustions,
14101                &mut deferred_entry.attempts,
14102            )
14103            .await;
14104        drop(entry_budget_permit);
14105
14106        let err = match result {
14107            Ok(DecommissionEntryOutcome::Complete) => return Ok(DecommissionEntryOutcome::Complete),
14108            Ok(outcome @ DecommissionEntryOutcome::Deferred(_)) => return Ok(outcome),
14109            Err(err) => err,
14110        };
14111        error!(
14112            event = EVENT_DECOMMISSION_ENTRY,
14113            component = LOG_COMPONENT_ECSTORE,
14114            subsystem = LOG_SUBSYSTEM_POOLS,
14115            pool_index = idx,
14116            set_index = set_idx,
14117            bucket = %bucket,
14118            object = %object_name,
14119            state = "entry_failed",
14120            error = %err,
14121            "Decommission entry failed"
14122        );
14123        record_decommission_entry_error(&entry_error, &rx, err).await;
14124        Ok(DecommissionEntryOutcome::Complete)
14125    }
14126
14127    #[allow(clippy::too_many_arguments)]
14128    async fn decommission_set(
14129        self: Arc<Self>,
14130        rx: CancellationToken,
14131        idx: usize,
14132        set_idx: usize,
14133        generation: OffsetDateTime,
14134        set: Arc<SetDisks>,
14135        bi: DecomBucketInfo,
14136        lifecycle_config: Option<BucketLifecycleConfiguration>,
14137        object_lock_config: Option<ObjectLockConfiguration>,
14138        replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
14139        expected_bucket_incarnation_id: Option<uuid::Uuid>,
14140        source_changed_exhaustions: Arc<AtomicUsize>,
14141        entry_budget: Arc<Semaphore>,
14142        entry_error: Arc<tokio::sync::Mutex<Option<Error>>>,
14143    ) -> Result<()> {
14144        let worker_count = DECOMMISSION_ENTRY_WORKERS_PER_SET;
14145        let queue_capacity = decommission_entry_queue_capacity(worker_count);
14146        let outstanding_capacity = queue_capacity.saturating_add(worker_count);
14147        let outstanding = Arc::new(Semaphore::new(outstanding_capacity));
14148        let (tx, rx_queue) = mpsc::channel(queue_capacity);
14149        let queue = Arc::new(tokio::sync::Mutex::new(rx_queue));
14150        let deferred: Arc<tokio::sync::Mutex<Vec<DeferredDecommissionEntry>>> = Arc::new(tokio::sync::Mutex::new(Vec::new()));
14151
14152        let mut entry_workers = tokio::task::JoinSet::new();
14153        for _ in 0..worker_count {
14154            let this = self.clone();
14155            let rx = rx.clone();
14156            let bucket = bi.name.clone();
14157            let set = set.clone();
14158            let lifecycle_config = lifecycle_config.clone();
14159            let object_lock_config = object_lock_config.clone();
14160            let replication_config = replication_config.clone();
14161            let source_changed_exhaustions = Arc::clone(&source_changed_exhaustions);
14162            let queue = queue.clone();
14163            let entry_budget = entry_budget.clone();
14164            let deferred = deferred.clone();
14165            let entry_error = entry_error.clone();
14166            entry_workers.spawn(async move {
14167                this.decommission_entry_worker(
14168                    rx,
14169                    idx,
14170                    set_idx,
14171                    generation,
14172                    bucket,
14173                    set,
14174                    lifecycle_config,
14175                    object_lock_config,
14176                    replication_config,
14177                    expected_bucket_incarnation_id,
14178                    source_changed_exhaustions,
14179                    entry_budget,
14180                    queue,
14181                    deferred,
14182                    entry_error,
14183                )
14184                .await;
14185            });
14186        }
14187
14188        let callback: ListCallback = Arc::new({
14189            let tx = tx.clone();
14190            let outstanding = outstanding.clone();
14191            let callback_rx = rx.clone();
14192            let entry_error = entry_error.clone();
14193            let bucket = bi.name.clone();
14194            move |entry: MetaCacheEntry| {
14195                let tx = tx.clone();
14196                let outstanding = outstanding.clone();
14197                let callback_rx = callback_rx.clone();
14198                let entry_error = entry_error.clone();
14199                let bucket = bucket.clone();
14200                Box::pin(async move {
14201                    if callback_rx.is_cancelled() || entry_error.lock().await.is_some() {
14202                        return;
14203                    }
14204
14205                    if matches!(
14206                        enqueue_decommission_entry(&callback_rx, &outstanding, &tx, entry).await,
14207                        DecommissionEntryEnqueueResult::Closed
14208                    ) {
14209                        let err = Error::other("decommission entry queue closed");
14210                        error!(
14211                            event = EVENT_DECOMMISSION_ENTRY,
14212                            component = LOG_COMPONENT_ECSTORE,
14213                            subsystem = LOG_SUBSYSTEM_POOLS,
14214                            pool_index = idx,
14215                            set_index = set_idx,
14216                            bucket = %bucket,
14217                            state = "entry_queue_closed",
14218                            error = %err,
14219                            "Decommission entry queue closed"
14220                        );
14221                        record_decommission_entry_error(&entry_error, &callback_rx, err).await;
14222                    }
14223                })
14224            }
14225        });
14226
14227        let list_set = set.clone();
14228        let list_rx = rx.clone();
14229        let list_rx_for_list = list_rx.clone();
14230        let list_rx_for_drain = list_rx.clone();
14231        let list_bi = bi.clone();
14232        let list_outstanding = outstanding.clone();
14233        let list_entry_error = entry_error.clone();
14234        let list_store = self.clone();
14235        let mut listing = tokio::spawn(async move {
14236            run_decommission_listing_with_retry_and_drain(
14237                list_rx.clone(),
14238                list_bi.name.clone(),
14239                callback,
14240                idx,
14241                set_idx,
14242                DECOMMISSION_LISTING_MAX_ATTEMPTS,
14243                move |callback| {
14244                    let set = list_set.clone();
14245                    let rx = list_rx_for_list.clone();
14246                    let bucket = list_bi.clone();
14247                    let entry_error = list_entry_error.clone();
14248                    let store = list_store.clone();
14249                    async move {
14250                        set.list_objects_to_decommission(
14251                            store,
14252                            rx,
14253                            bucket,
14254                            callback,
14255                            entry_error,
14256                            idx,
14257                            set_idx,
14258                            generation,
14259                            false,
14260                        )
14261                        .await
14262                    }
14263                },
14264                move || {
14265                    let rx = list_rx_for_drain.clone();
14266                    let outstanding = list_outstanding.clone();
14267                    async move { drain_decommission_entry_queue(&rx, &outstanding, outstanding_capacity).await }
14268                },
14269            )
14270            .await
14271        });
14272
14273        let mut listing_result = None;
14274        let mut workers_left = worker_count;
14275        let mut sender = Some(tx);
14276        while listing_result.is_none() || workers_left > 0 {
14277            tokio::select! {
14278                biased;
14279                result = &mut listing, if listing_result.is_none() => {
14280                    let result = resolve_decommission_listing_worker_result(set_idx, result);
14281                    if result.is_err() {
14282                        rx.cancel();
14283                    }
14284                    listing_result = Some(result);
14285                    drop(sender.take());
14286                }
14287                worker_result = entry_workers.join_next(), if workers_left > 0 => {
14288                    workers_left -= 1;
14289                    if let Some(Err(err)) = worker_result {
14290                        let err = Error::other(format!("decommission entry worker {set_idx} task join error: {err}"));
14291                        error!(
14292                            event = EVENT_DECOMMISSION_ENTRY,
14293                            component = LOG_COMPONENT_ECSTORE,
14294                            subsystem = LOG_SUBSYSTEM_POOLS,
14295                            pool_index = idx,
14296                            set_index = set_idx,
14297                            bucket = %bi.name,
14298                            state = "entry_worker_join_failed",
14299                            error = %err,
14300                            "Decommission entry worker task failed"
14301                        );
14302                        record_decommission_entry_error(&entry_error, &rx, err).await;
14303                    }
14304                }
14305            }
14306        }
14307
14308        let listing_result = listing_result.unwrap_or_else(|| Err(Error::other("decommission listing task did not complete")));
14309        if let Some(err) = entry_error.lock().await.clone() {
14310            return Err(err);
14311        }
14312        listing_result?;
14313
14314        // The listing drained, so every entry either completed or was deferred.
14315        // Replaying the deferred set is the last chance for this round; a
14316        // bucket must not be reported complete while a deferred entry remains.
14317        let deferred_entries = std::mem::take(&mut *deferred.lock().await);
14318        let deferred_remaining = Arc::new(AtomicUsize::new(0));
14319        if !deferred_entries.is_empty() {
14320            // The replay keeps the set's own worker bound: a whole-bucket
14321            // deferral must not spawn one task per object.
14322            futures::stream::iter(deferred_entries)
14323                .for_each_concurrent(Some(worker_count), |deferred_entry| {
14324                    let this = self.clone();
14325                    let rx = rx.clone();
14326                    let bucket = bi.name.clone();
14327                    let set = set.clone();
14328                    let lifecycle_config = lifecycle_config.clone();
14329                    let object_lock_config = object_lock_config.clone();
14330                    let replication_config = replication_config.clone();
14331                    let source_changed_exhaustions = Arc::clone(&source_changed_exhaustions);
14332                    let entry_budget = entry_budget.clone();
14333                    let entry_error = entry_error.clone();
14334                    let deferred_remaining = Arc::clone(&deferred_remaining);
14335                    async move {
14336                        let outcome = this
14337                            .replay_deferred_decommission_entry(
14338                                rx,
14339                                idx,
14340                                set_idx,
14341                                generation,
14342                                deferred_entry,
14343                                bucket,
14344                                set,
14345                                lifecycle_config,
14346                                object_lock_config,
14347                                replication_config,
14348                                expected_bucket_incarnation_id,
14349                                source_changed_exhaustions,
14350                                entry_budget,
14351                                entry_error,
14352                            )
14353                            .await;
14354                        if matches!(outcome, Ok(DecommissionEntryOutcome::Deferred(_))) {
14355                            deferred_remaining.fetch_add(1, Ordering::Relaxed);
14356                        }
14357                    }
14358                })
14359                .await;
14360            if let Some(err) = entry_error.lock().await.clone() {
14361                return Err(err);
14362            }
14363        }
14364        let deferred_remaining = deferred_remaining.load(Ordering::Relaxed);
14365        // A set must not report success while an entry is still deferred: the
14366        // bucket would be marked done and the object silently dropped.
14367        ensure_decommission_deferred_drained(bi.name.as_str(), set_idx, deferred_remaining)
14368    }
14369
14370    async fn track_decommission_entry_progress_stage(
14371        &self,
14372        idx: usize,
14373        generation: OffsetDateTime,
14374        bucket: &str,
14375        object: &str,
14376        stage: &'static str,
14377    ) -> Result<()> {
14378        {
14379            let mut pool_meta = self.pool_meta.write().await;
14380            ensure_decommission_generation(&pool_meta, idx, generation)?;
14381            track_decommission_current_object_stage(&mut pool_meta, idx, bucket, object, stage)
14382                .map_err(|err| with_decommission_entry_context(stage, bucket, object, err))?;
14383        }
14384
14385        Ok(())
14386    }
14387
14388    #[allow(clippy::too_many_arguments)]
14389    async fn wait_decommission_target_gate_retry(
14390        &self,
14391        rx: &CancellationToken,
14392        idx: usize,
14393        generation: OffsetDateTime,
14394        target_pool_index: usize,
14395        capacity_owner: Option<DecommissionCapacityOwner>,
14396        set: &SetDisks,
14397        entry: &MetaCacheEntry,
14398        bucket: &str,
14399        expected_version: &FileInfo,
14400        target_busy_attempt: usize,
14401    ) -> Result<Option<DecommissionCapacityTargetPermit>> {
14402        #[cfg(test)]
14403        notify_decommission_target_gate_retry(self.id);
14404        let mut wait_attempt = target_busy_attempt;
14405        let target_guard = loop {
14406            let retry_delay =
14407                decommission_capacity_gate_retry_delay(wait_attempt.min(DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS));
14408            if wait_decommission_retry_backoff(rx, retry_delay).await {
14409                decommission_cancel_signal_result(rx.is_cancelled())?;
14410            }
14411            if self.decommission_cancel_requested(idx, rx).await {
14412                rx.cancel();
14413            }
14414            decommission_cancel_signal_result(rx.is_cancelled())?;
14415            self.ensure_decommission_generation_current(idx, generation).await?;
14416
14417            match self.acquire_decommission_capacity_target_guard(target_pool_index).await {
14418                Ok(guard) => break guard,
14419                Err(err) if is_decommission_capacity_target_gate_busy(&err) => {
14420                    if decommission_capacity_target_gate_retry_exhausted(wait_attempt) {
14421                        warn!(
14422                            event = EVENT_DECOMMISSION_STATE,
14423                            component = LOG_COMPONENT_ECSTORE,
14424                            subsystem = LOG_SUBSYSTEM_POOLS,
14425                            pool_index = idx,
14426                            target_pool_index,
14427                            attempts = wait_attempt.saturating_add(1),
14428                            state = "capacity_gate_retry_exhausted",
14429                            error = %err,
14430                            "Decommission target capacity gate remained busy; pausing for supervised retry"
14431                        );
14432                        return Err(err);
14433                    }
14434                    wait_attempt = wait_attempt.saturating_add(1);
14435                }
14436                Err(err) => return Err(err),
14437            }
14438        };
14439
14440        #[cfg(test)]
14441        notify_decommission_target_gate_exact_reload(self.id);
14442        let current = load_decommission_entry_exact_versions(set, entry, bucket, "target_gate_retry").await?;
14443        ensure_decommission_capacity_target_fence(&target_guard, target_pool_index, "source identity revalidation")?;
14444        let expected_identity = data_movement::source_cleanup_version_identity(expected_version);
14445        let identity_current = current
14446            .versions
14447            .iter()
14448            .any(|version| data_movement::source_cleanup_version_identity(version) == expected_identity);
14449        if !identity_current {
14450            return Ok(None);
14451        }
14452        let owner = capacity_owner.ok_or_else(|| Error::other("target-gate retry is missing its decommission capacity owner"))?;
14453        let owner = owner.with_mutation_id(decommission_capacity_version_mutation_id(owner, bucket, expected_version));
14454        install_decommission_capacity_target_permit(self.id, target_pool_index, owner, target_guard).map(Some)
14455    }
14456
14457    #[cfg(feature = "test-util")]
14458    pub async fn prepare_scanner_pause_backlog_retirement_for_test(
14459        &self,
14460        source_pool_index: usize,
14461        source_bytes: usize,
14462    ) -> Result<()> {
14463        let source_bytes = source_bytes.max(1);
14464        let total = source_bytes.saturating_mul(8).saturating_add(64 * 1024);
14465        let capacities = (0..self.pools.len())
14466            .map(|pool_index| {
14467                DecommissionPoolCapacityInfo::for_test(
14468                    pool_index,
14469                    DecommissionErasureLayout { data: 1, parity: 0 },
14470                    if pool_index == source_pool_index { 0 } else { total },
14471                    total,
14472                    if pool_index == source_pool_index { source_bytes } else { 0 },
14473                )
14474            })
14475            .collect();
14476        set_decommission_capacity_info_overrides_for_test(self.id, vec![capacities]);
14477        self.save_current_pool_meta_for_decommission_start(&[source_pool_index], Vec::new())
14478            .await
14479            .map(|_| ())
14480    }
14481
14482    #[cfg(feature = "test-util")]
14483    pub async fn retire_scanner_pause_backlog_for_test(
14484        self: &Arc<Self>,
14485        source_pool_index: usize,
14486        source_set_index: usize,
14487    ) -> Result<()> {
14488        let set = self
14489            .pools
14490            .get(source_pool_index)
14491            .and_then(|pool| pool.disk_set.get(source_set_index))
14492            .cloned()
14493            .ok_or_else(|| Error::other("scanner retirement test requested an unknown set"))?;
14494        let generation = self.active_decommission_generation(source_pool_index).await?;
14495        let owner = self
14496            .decommission_capacity_owner_for_worker(source_pool_index, generation)
14497            .await?;
14498        let expected = set
14499            .load_file_info_versions_exact(RUSTFS_META_BUCKET, data_movement::scanner_backlog::SCANNER_PAUSE_BACKLOG_PATH)
14500            .await?
14501            .unwrap_or_default();
14502        match self
14503            .retire_scanner_pause_backlog_entry(CancellationToken::new(), source_pool_index, generation, set, expected, owner)
14504            .await?
14505        {
14506            DecommissionEntryAttemptOutcome::Complete => Ok(()),
14507            DecommissionEntryAttemptOutcome::SourceChanged => Err(Error::other("scanner retirement source changed")),
14508        }
14509    }
14510
14511    #[cfg(feature = "test-util")]
14512    pub async fn stage_scanner_pause_backlog_retirement_intent_for_test(
14513        &self,
14514        source_pool_index: usize,
14515        source_set_index: usize,
14516    ) -> Result<()> {
14517        let generation = self.active_decommission_generation(source_pool_index).await?;
14518        let owner = self
14519            .decommission_capacity_owner_for_worker(source_pool_index, generation)
14520            .await?
14521            .ok_or_else(|| Error::other("scanner retirement test has no capacity owner"))?;
14522        let versions = self.pools[source_pool_index].disk_set[source_set_index]
14523            .load_file_info_versions_exact(RUSTFS_META_BUCKET, data_movement::scanner_backlog::SCANNER_PAUSE_BACKLOG_PATH)
14524            .await?
14525            .ok_or_else(|| Error::other("scanner retirement test source is missing"))?;
14526        let version = versions
14527            .versions
14528            .first()
14529            .ok_or_else(|| Error::other("scanner retirement test source is empty"))?;
14530        let owner = owner.with_mutation_id(decommission_capacity_version_mutation_id(owner, RUSTFS_META_BUCKET, version));
14531        let size = usize::try_from(version.size).map_err(|_| Error::other("scanner retirement test source size is invalid"))?;
14532        let target = self.select_decommission_capacity_target_pool(owner, size).await?;
14533        let failed: Result<()> = self
14534            .run_decommission_capacity_admitted_mutation(target, Some(owner), Some(size), || async {
14535                Err(Error::other("injected scanner retirement target failure"))
14536            })
14537            .await;
14538        match failed {
14539            Err(err) if err.to_string().contains("injected scanner retirement target failure") => Ok(()),
14540            Err(err) => Err(err),
14541            Ok(()) => Err(Error::other("scanner retirement test did not retain its target intent")),
14542        }
14543    }
14544
14545    #[allow(clippy::too_many_arguments)]
14546    async fn retire_scanner_pause_backlog_entry(
14547        self: &Arc<Self>,
14548        rx: CancellationToken,
14549        idx: usize,
14550        generation: OffsetDateTime,
14551        set: Arc<SetDisks>,
14552        expected: FileInfoVersions,
14553        capacity_owner: Option<DecommissionCapacityOwner>,
14554    ) -> Result<DecommissionEntryAttemptOutcome> {
14555        let store = Arc::clone(self);
14556        // The disk layer may own a rename after its waiter is canceled. Keep
14557        // the topology and object fences in that operation's owning task.
14558        tokio::spawn(async move {
14559            let operation_gate = store.ctx.data_movement_operation_gate();
14560            store
14561                .run_guarded_decommission_side_effect(&rx, &operation_gate, || {
14562                    store.retire_scanner_pause_backlog_entry_inner(idx, generation, set, &expected, capacity_owner)
14563                })
14564                .await
14565        })
14566        .await
14567        .map_err(Error::from)?
14568    }
14569
14570    async fn retire_scanner_pause_backlog_entry_inner(
14571        &self,
14572        idx: usize,
14573        generation: OffsetDateTime,
14574        set: Arc<SetDisks>,
14575        expected: &FileInfoVersions,
14576        capacity_owner: Option<DecommissionCapacityOwner>,
14577    ) -> Result<DecommissionEntryAttemptOutcome> {
14578        use data_movement::scanner_backlog::{
14579            SCANNER_PAUSE_BACKLOG_PATH, persist_native_scanner_pause_backlog_replica, plan_scanner_pause_backlog_retirement,
14580            read_scanner_pause_backlog_retirement_replicas,
14581        };
14582
14583        if expected.versions.is_empty() && expected.free_versions.is_empty() {
14584            // A canceled entry waiter may resume after the owned cleanup
14585            // finished deleting its source. There is no remaining record to move.
14586            return Ok(DecommissionEntryAttemptOutcome::Complete);
14587        }
14588        let [version] = expected.versions.as_slice() else {
14589            return Err(Error::other("scanner pause backlog retirement requires exactly one source version"));
14590        };
14591        if version.version_id.is_some_and(|version| !version.is_nil())
14592            || version.deleted
14593            || version.tier_free_version()
14594            || version.is_remote()
14595            || !expected.free_versions.is_empty()
14596        {
14597            return Err(Error::other(
14598                "scanner pause backlog retirement requires an unversioned local source record",
14599            ));
14600        }
14601        let object_fence = self
14602            .acquire_decommission_source_cleanup_fence(RUSTFS_META_BUCKET, SCANNER_PAUSE_BACKLOG_PATH, set.as_ref())
14603            .await?;
14604        // Native replica writers acquire this same fixed object domain before
14605        // durable pool metadata. Keep both fences through physical source deletion.
14606        let save_guard = self.pool_meta_save_gate.lock().await;
14607        let (pool_meta_guard, snapshot) = self
14608            .acquire_pool_meta_read_guard(&save_guard, "scanner pause backlog retirement admission failed")
14609            .await?;
14610        ensure_decommission_generation(&snapshot, idx, generation)?;
14611        let owner = capacity_owner.ok_or_else(|| {
14612            decommission_capacity_blocked_error("scanner pause backlog retirement has no active capacity owner")
14613        })?;
14614        let reservation = snapshot.pools[idx]
14615            .decommission
14616            .as_ref()
14617            .and_then(|info| info.capacity_reservation.as_ref())
14618            .filter(|reservation| reservation.admits_owner(owner, OffsetDateTime::now_utc()))
14619            .ok_or_else(|| decommission_capacity_blocked_error("scanner pause backlog retirement reservation is stale"))?;
14620        let mutation_id = decommission_capacity_version_mutation_id(owner, RUSTFS_META_BUCKET, version);
14621        if reservation.targets.iter().any(|target| {
14622            target.pending_mutation_id == Some(mutation_id)
14623                || target
14624                    .temporary_mutations
14625                    .iter()
14626                    .any(|mutation| mutation.mutation_id == mutation_id)
14627        }) {
14628            return Err(decommission_capacity_blocked_error(
14629                "scanner pause backlog retirement has an unresolved target capacity intent",
14630            ));
14631        }
14632        if snapshot.scanner_pause_backlog_pool_writable(idx) {
14633            return Err(Error::other("scanner pause backlog source still belongs to writable membership"));
14634        }
14635        let sets: Vec<_> = self
14636            .pools
14637            .iter()
14638            .enumerate()
14639            .filter(|(pool_index, _)| *pool_index == idx || snapshot.scanner_pause_backlog_pool_writable(*pool_index))
14640            .flat_map(|(_, pool)| pool.disk_set.iter().cloned())
14641            .collect();
14642        let mut replicas = read_scanner_pause_backlog_retirement_replicas(idx, set.set_index, sets.clone()).await?;
14643        if let Some(plan) = plan_scanner_pause_backlog_retirement(idx, &replicas)? {
14644            let expected_stable_record = plan.stable_record.clone();
14645            let mut phases = Vec::with_capacity(3);
14646            if let Some(record) = plan.seed_record {
14647                phases.push(("seed", record));
14648            }
14649            phases.push(("commit", plan.commit_record));
14650            phases.push(("stabilize", plan.stable_record));
14651            for (phase, record) in phases {
14652                if object_fence.is_lock_lost()
14653                    || pool_meta_guard.is_lock_lost()
14654                    || !reservation.admits_owner(owner, OffsetDateTime::now_utc())
14655                {
14656                    return Err(decommission_capacity_blocked_error(
14657                        "scanner pause backlog retirement fence expired during native repair",
14658                    ));
14659                }
14660                for read in replicas.iter().filter(|read| read.replica.pool_index != idx) {
14661                    ensure_external_decommission_target_admission(
14662                        &snapshot,
14663                        read.replica.pool_index,
14664                        DecommissionCapacityAdmission::ScannerBacklog,
14665                    )?;
14666                }
14667                let results = join_all(replicas.iter().filter(|read| read.replica.pool_index != idx).map(|read| {
14668                    let target = Arc::clone(&self.pools[read.replica.pool_index].disk_set[read.replica.set_index]);
14669                    let record = record.clone();
14670                    let object_fence = &object_fence;
14671                    let pool_meta_guard = &pool_meta_guard;
14672                    async move {
14673                        let mut opts = ObjectOptions {
14674                            no_lock: self.pools[0].disk_set[0].shares_namespace_lock_domain(&target).await,
14675                            ..Default::default()
14676                        };
14677                        object_fence.add_namespace_lock_fence(&mut opts);
14678                        opts.add_namespace_lock_guard(pool_meta_guard);
14679                        persist_native_scanner_pause_backlog_replica(target, record, read.preconditions(), opts, phase).await
14680                    }
14681                }))
14682                .await;
14683                // Every dispatched native CAS has finished before an error can
14684                // release the owned task's object and durable membership fences.
14685                for result in results {
14686                    result?;
14687                }
14688                replicas = read_scanner_pause_backlog_retirement_replicas(idx, set.set_index, sets.clone()).await?;
14689                if replicas
14690                    .iter()
14691                    .filter(|read| read.replica.pool_index != idx)
14692                    .any(|read| read.replica.data.as_deref() != Some(record.as_slice()))
14693                {
14694                    return Err(Error::other(
14695                        "scanner pause backlog native repair did not persist every surviving replica",
14696                    ));
14697                }
14698                if let Some(next) = plan_scanner_pause_backlog_retirement(idx, &replicas)?
14699                    && next.stable_record != expected_stable_record
14700                {
14701                    return Err(Error::other("scanner pause backlog native authority changed during repair"));
14702                }
14703            }
14704            if plan_scanner_pause_backlog_retirement(idx, &replicas)?.is_some() {
14705                return Err(Error::other("scanner pause backlog native repair is not fully committed and stable"));
14706            }
14707        }
14708        if object_fence.is_lock_lost()
14709            || pool_meta_guard.is_lock_lost()
14710            || !reservation.admits_owner(owner, OffsetDateTime::now_utc())
14711        {
14712            return Err(decommission_capacity_blocked_error(
14713                "scanner pause backlog retirement fence expired before cleanup",
14714            ));
14715        }
14716        let result = data_movement::cleanup_source_entry_if_unchanged(
14717            set,
14718            RUSTFS_META_BUCKET,
14719            SCANNER_PAUSE_BACKLOG_PATH,
14720            expected,
14721            &[],
14722            data_movement::SourceCleanupBucketFence {
14723                expected_incarnation_id: None,
14724                lifecycle_guard: None,
14725                namespace_lock_lost_signal: pool_meta_guard.lock_lost_signal(),
14726                object_mutation_fence: Some(&object_fence),
14727            },
14728            "scanner pause backlog retirement",
14729        )
14730        .await;
14731        match result {
14732            Ok(_) => Ok(DecommissionEntryAttemptOutcome::Complete),
14733            Err(data_movement::SourceCleanupError::SourceChanged) => Ok(DecommissionEntryAttemptOutcome::SourceChanged),
14734            Err(data_movement::SourceCleanupError::Storage(err)) => Err(err),
14735        }
14736    }
14737
14738    #[allow(clippy::too_many_arguments)]
14739    #[tracing::instrument(
14740        level = "trace",
14741        skip_all,
14742        fields(
14743            event = EVENT_DECOMMISSION_ENTRY,
14744            component = LOG_COMPONENT_ECSTORE,
14745            subsystem = LOG_SUBSYSTEM_POOLS,
14746            state = "processing",
14747            pool_index = idx,
14748            bucket = %bucket,
14749            object = %entry.name,
14750            generation = %generation,
14751        )
14752    )]
14753    async fn decommission_entry(
14754        self: &Arc<Self>,
14755        rx: CancellationToken,
14756        idx: usize,
14757        generation: OffsetDateTime,
14758        entry: MetaCacheEntry,
14759        bucket: String,
14760        set: Arc<SetDisks>,
14761        lifecycle_config: Option<BucketLifecycleConfiguration>,
14762        object_lock_config: Option<ObjectLockConfiguration>,
14763        replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
14764        expected_bucket_incarnation_id: Option<uuid::Uuid>,
14765        source_changed_exhaustions: Arc<AtomicUsize>,
14766        attempts: &mut usize,
14767    ) -> Result<DecommissionEntryOutcome> {
14768        let mut counted_versions = HashSet::new();
14769
14770        for entry_attempt in 1..=DECOMMISSION_ENTRY_MAX_ATTEMPTS {
14771            let attempt_result = {
14772                let mut conflict_attempt = 0;
14773                let mut scanner_backlog_handoff_attempt = 0;
14774                loop {
14775                    let result = self
14776                        .decommission_entry_attempt(
14777                            rx.clone(),
14778                            idx,
14779                            generation,
14780                            entry.clone(),
14781                            bucket.clone(),
14782                            Arc::clone(&set),
14783                            lifecycle_config.clone(),
14784                            object_lock_config.clone(),
14785                            replication_config.clone(),
14786                            expected_bucket_incarnation_id,
14787                            entry_attempt,
14788                            scanner_backlog_handoff_attempt,
14789                            source_changed_exhaustions.as_ref(),
14790                            &mut counted_versions,
14791                            attempts,
14792                        )
14793                        .await;
14794                    let retry = result
14795                        .as_ref()
14796                        .err()
14797                        .and_then(|err| decommission_capacity_retry_kind(err, conflict_attempt));
14798                    if let Some(DecommissionCapacityRetryKind::IntentConflict) = retry {
14799                        conflict_attempt += 1;
14800                        let retry_delay =
14801                            decommission_retry_backoff_delay(DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, conflict_attempt);
14802                        if wait_decommission_retry_backoff(&rx, retry_delay).await {
14803                            decommission_cancel_signal_result(rx.is_cancelled())?;
14804                        }
14805                        continue;
14806                    }
14807                    // Scanner pause backlog handoff conflicts resolve when the
14808                    // surviving membership converges. Retry on a bounded
14809                    // backoff that never writes failure state, clears
14810                    // `start_time`, or releases the capacity reservation.
14811                    let scanner_retry = result
14812                        .as_ref()
14813                        .err()
14814                        .and_then(|err| decommission_scanner_backlog_retry_kind(err, scanner_backlog_handoff_attempt));
14815                    if let Some(DecommissionScannerBacklogRetryKind::HandoffConflict) = scanner_retry {
14816                        scanner_backlog_handoff_attempt += 1;
14817                        let retry_delay = decommission_retry_backoff_delay(
14818                            DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY,
14819                            scanner_backlog_handoff_attempt,
14820                        );
14821                        info!(
14822                            event = EVENT_DECOMMISSION_ENTRY,
14823                            component = LOG_COMPONENT_ECSTORE,
14824                            subsystem = LOG_SUBSYSTEM_POOLS,
14825                            state = "scanner_backlog_handoff_retry",
14826                            pool_index = idx,
14827                            bucket = %bucket,
14828                            object = %entry.name,
14829                            attempt = scanner_backlog_handoff_attempt,
14830                            max_attempts = DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS,
14831                            retry_delay_ms = retry_delay.as_millis(),
14832                            "Decommission scanner pause backlog handoff conflict; retrying entry"
14833                        );
14834                        if wait_decommission_retry_backoff(&rx, retry_delay).await {
14835                            decommission_cancel_signal_result(rx.is_cancelled())?;
14836                        }
14837                        continue;
14838                    }
14839                    break result;
14840                }
14841            };
14842            match attempt_result {
14843                Ok(DecommissionEntryAttemptOutcome::Complete) => return Ok(DecommissionEntryOutcome::Complete),
14844                Ok(DecommissionEntryAttemptOutcome::SourceChanged) => {
14845                    let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_SOURCE_CLEANUP_RETRY_DELAY, entry_attempt);
14846                    warn!(
14847                        event = EVENT_DECOMMISSION_ENTRY,
14848                        component = LOG_COMPONENT_ECSTORE,
14849                        subsystem = LOG_SUBSYSTEM_POOLS,
14850                        state = "source_changed_retry",
14851                        pool_index = idx,
14852                        bucket = %bucket,
14853                        object = %entry.name,
14854                        attempt = entry_attempt,
14855                        max_attempts = DECOMMISSION_ENTRY_MAX_ATTEMPTS,
14856                        retry_delay_ms = retry_delay.as_millis(),
14857                        "Decommission source changed during cleanup preflight; retrying entry"
14858                    );
14859                    if wait_decommission_retry_backoff(&rx, retry_delay).await {
14860                        decommission_cancel_signal_result(rx.is_cancelled())?;
14861                    }
14862                }
14863                Err(err) => {
14864                    return match resolve_decommission_entry_failure(err) {
14865                        Ok(DecommissionEntryOutcome::Deferred(err)) => {
14866                            // Contention belongs to the target pool, not to this
14867                            // object: failing the entry would discard every
14868                            // in-flight object's progress for a condition the
14869                            // deferred round can still absorb.
14870                            warn!(
14871                                event = EVENT_DECOMMISSION_ENTRY,
14872                                component = LOG_COMPONENT_ECSTORE,
14873                                subsystem = LOG_SUBSYSTEM_POOLS,
14874                                state = "entry_deferred",
14875                                pool_index = idx,
14876                                bucket = %bucket,
14877                                object = %entry.name,
14878                                attempts = DECOMMISSION_ENTRY_MAX_ATTEMPTS,
14879                                error = %err,
14880                                "Decommission entry deferred after exhausting its attempts on target capacity contention"
14881                            );
14882                            Ok(DecommissionEntryOutcome::Deferred(err))
14883                        }
14884                        Ok(DecommissionEntryOutcome::Complete) => {
14885                            warn!(
14886                                event = EVENT_DECOMMISSION_ENTRY,
14887                                component = LOG_COMPONENT_ECSTORE,
14888                                subsystem = LOG_SUBSYSTEM_POOLS,
14889                                state = "entry_ignored",
14890                                pool_index = idx,
14891                                bucket = %bucket,
14892                                object = %entry.name,
14893                                "Decommission entry ignored because its mutation is no longer needed"
14894                            );
14895                            Ok(DecommissionEntryOutcome::Complete)
14896                        }
14897                        Err(err) => Err(err),
14898                    };
14899                }
14900            }
14901        }
14902
14903        Err(Error::other(format!(
14904            "decommission entry retry loop ended without a terminal result for {bucket}/{}",
14905            entry.name
14906        )))
14907    }
14908
14909    #[allow(unused_assignments, clippy::too_many_arguments)]
14910    async fn decommission_entry_attempt(
14911        self: &Arc<Self>,
14912        rx: CancellationToken,
14913        idx: usize,
14914        generation: OffsetDateTime,
14915        entry: MetaCacheEntry,
14916        bucket: String,
14917        set: Arc<SetDisks>,
14918        lifecycle_config: Option<BucketLifecycleConfiguration>,
14919        object_lock_config: Option<ObjectLockConfiguration>,
14920        replication_config: Option<(ReplicationConfiguration, OffsetDateTime)>,
14921        expected_bucket_incarnation_id: Option<uuid::Uuid>,
14922        entry_attempt: usize,
14923        scanner_backlog_handoff_attempt: usize,
14924        source_changed_exhaustions: &AtomicUsize,
14925        counted_versions: &mut HashSet<(Option<uuid::Uuid>, bool)>,
14926        attempts: &mut usize,
14927    ) -> Result<DecommissionEntryAttemptOutcome> {
14928        record_decommission_object_attempt(attempts);
14929        debug!(
14930            event = EVENT_DECOMMISSION_ENTRY,
14931            component = LOG_COMPONENT_ECSTORE,
14932            subsystem = LOG_SUBSYSTEM_POOLS,
14933            state = "started",
14934            pool_index = idx,
14935            bucket = %bucket,
14936            object = %entry.name,
14937            attempt = entry_attempt,
14938            max_attempts = DECOMMISSION_ENTRY_MAX_ATTEMPTS,
14939            "Decommission entry started"
14940        );
14941        // Scanner caches describe their own erasure set and are rebuilt there.
14942        // Copying one onto another set can overwrite unrelated cache contents or
14943        // leave an unresolvable target-capacity intent after a conditional PUT.
14944        if is_decommission_set_local_usage_cache(&bucket, &entry.name) {
14945            return Ok(DecommissionEntryAttemptOutcome::Complete);
14946        }
14947        let durable_ilm_record = if bucket == RUSTFS_META_BUCKET {
14948            classify_durable_ilm_record(&entry.name)
14949                .map_err(|err| with_decommission_entry_context("durable_ilm_namespace", &bucket, &entry.name, err))?
14950        } else {
14951            None
14952        };
14953        if self.decommission_cancel_requested(idx, &rx).await {
14954            rx.cancel();
14955        }
14956        decommission_cancel_signal_result(rx.is_cancelled())?;
14957        self.ensure_decommission_generation_current(idx, generation).await?;
14958        self.ensure_decommission_runtime_capacity_available(idx, generation).await?;
14959        let capacity_owner = self.decommission_capacity_owner_for_worker(idx, generation).await?;
14960        let operation_gate = self.ctx.data_movement_operation_gate();
14961
14962        let bucket_incarnation_fence = match expected_bucket_incarnation_id {
14963            Some(expected) => Some(self.acquire_bucket_incarnation_fence(&bucket, expected).await?),
14964            None => None,
14965        };
14966
14967        let mut fivs = load_decommission_entry_exact_versions(&set, &entry, &bucket, "file_info_versions").await?;
14968
14969        if data_movement::scanner_backlog::is_scanner_pause_backlog(&bucket, &entry.name) {
14970            let outcome = self
14971                .retire_scanner_pause_backlog_entry(rx, idx, generation, Arc::clone(&set), fivs.clone(), capacity_owner)
14972                .await;
14973            return match outcome {
14974                Ok(DecommissionEntryAttemptOutcome::Complete) => {
14975                    let mut pool_meta = self.pool_meta.write().await;
14976                    ensure_decommission_generation(&pool_meta, idx, generation)?;
14977                    if let Some(version) = fivs.versions.first()
14978                        && counted_versions.insert((version.version_id, false))
14979                    {
14980                        count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), false)?;
14981                    }
14982                    track_decommission_current_object(&mut pool_meta, idx, &bucket, &entry.name)?;
14983                    drop(pool_meta);
14984                    self.track_decommission_entry_progress_stage(
14985                        idx,
14986                        generation,
14987                        &bucket,
14988                        &entry.name,
14989                        DECOMMISSION_STAGE_ENTRY_FINISHED,
14990                    )
14991                    .await?;
14992                    Ok(DecommissionEntryAttemptOutcome::Complete)
14993                }
14994                Ok(outcome) => Ok(outcome),
14995                Err(err) => {
14996                    // The retry loop owns the handoff-conflict budget, so a
14997                    // retryable conflict must not be attributed here. Once that
14998                    // budget is exhausted the failure is terminal and belongs to
14999                    // this entry rather than to whichever object was processed
15000                    // last.
15001                    let retryable_handoff = data_movement::data_movement_stage_source_as::<
15002                        data_movement::scanner_backlog::ScannerPauseBacklogRetirementError,
15003                    >(&err)
15004                    .is_some_and(
15005                        data_movement::scanner_backlog::ScannerPauseBacklogRetirementError::is_retryable_handoff_conflict,
15006                    );
15007                    let terminal = !retryable_handoff
15008                        || scanner_backlog_handoff_attempt >= DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS;
15009                    if terminal {
15010                        let mut pool_meta = self.pool_meta.write().await;
15011                        ensure_decommission_generation(&pool_meta, idx, generation)?;
15012                        if let Some(version) = fivs.versions.first()
15013                            && counted_versions.insert((version.version_id, true))
15014                        {
15015                            count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), true)?;
15016                        }
15017                        track_decommission_current_object(&mut pool_meta, idx, &bucket, &entry.name)?;
15018                    }
15019                    Err(err)
15020                }
15021            };
15022        }
15023
15024        let pending_mutations = if let Some(owner) = capacity_owner {
15025            self.pool_meta
15026                .read()
15027                .await
15028                .pools
15029                .get(owner.source_pool_index)
15030                .and_then(|pool| pool.decommission.as_ref())
15031                .and_then(|info| info.capacity_reservation.as_ref())
15032                .filter(|reservation| reservation.admits_cleanup_owner(owner))
15033                .map(|reservation| {
15034                    reservation
15035                        .targets
15036                        .iter()
15037                        .filter_map(|target| target.pending_mutation_id)
15038                        .collect::<HashSet<_>>()
15039                })
15040                .unwrap_or_default()
15041        } else {
15042            HashSet::new()
15043        };
15044        fivs.versions.sort_by_key(|version| {
15045            let mutation_id = capacity_owner.map(|owner| decommission_capacity_version_mutation_id(owner, &bucket, version));
15046            (
15047                mutation_id.is_none_or(|mutation_id| !pending_mutations.contains(&mutation_id)),
15048                version.mod_time.is_none(),
15049                std::cmp::Reverse(version.mod_time),
15050            )
15051        });
15052
15053        let mut decommissioned: usize = 0;
15054        let mut expired: usize = 0;
15055        let mut free_version_disposition = DecommissionFreeVersionDisposition::default();
15056        let mut cleanup_preflight_allowed_missing = Vec::new();
15057        let mut entry_blocked = false;
15058
15059        for version in fivs.versions.iter() {
15060            if self.decommission_cancel_requested(idx, &rx).await {
15061                rx.cancel();
15062            }
15063            decommission_cancel_signal_result(rx.is_cancelled())?;
15064
15065            if version.tier_free_version() {
15066                let version_id = version.version_id.map(|v| v.to_string());
15067                let mut migration_error = None;
15068                let mut migrated = false;
15069                let mut consumed = false;
15070                let mut capacity_failure = false;
15071                let mut version_attempt = 1;
15072                let mut target_busy_attempt: usize = 0;
15073                let mut target_permit = None;
15074                while version_attempt <= DECOMMISSION_VERSION_COPY_ATTEMPTS {
15075                    if version_attempt > 1 || target_busy_attempt > 0 {
15076                        record_decommission_object_attempt(attempts);
15077                    }
15078                    let result = self
15079                        .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
15080                            self.decommission_tiered_object(
15081                                bucket.as_str(),
15082                                &version.name,
15083                                version,
15084                                &decommission_capacity_owned_opts(
15085                                    decommission_remote_tiered_opts(
15086                                        version,
15087                                        version_id.clone(),
15088                                        idx,
15089                                        expected_bucket_incarnation_id,
15090                                    ),
15091                                    capacity_owner,
15092                                ),
15093                            )
15094                            .await
15095                        })
15096                        .await;
15097                    drop(target_permit.take());
15098                    if let Some(target_pool_index) = result.as_ref().err().and_then(decommission_capacity_target_gate_busy_index)
15099                    {
15100                        target_busy_attempt = target_busy_attempt.saturating_add(1);
15101                        let Some(permit) = self
15102                            .wait_decommission_target_gate_retry(
15103                                &rx,
15104                                idx,
15105                                generation,
15106                                target_pool_index,
15107                                capacity_owner,
15108                                set.as_ref(),
15109                                &entry,
15110                                &bucket,
15111                                version,
15112                                target_busy_attempt,
15113                            )
15114                            .await?
15115                        else {
15116                            return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
15117                        };
15118                        target_permit = Some(permit);
15119                        continue;
15120                    }
15121
15122                    match classify_decommission_free_version_attempt(result) {
15123                        DecommissionFreeVersionAttempt::Migrated => {
15124                            migrated = true;
15125                            migration_error = None;
15126                            break;
15127                        }
15128                        DecommissionFreeVersionAttempt::Consumed => {
15129                            consumed = true;
15130                            migration_error = None;
15131                            break;
15132                        }
15133                        DecommissionFreeVersionAttempt::CapacityFailure(err) => {
15134                            capacity_failure = true;
15135                            migration_error = Some(err);
15136                            break;
15137                        }
15138                        DecommissionFreeVersionAttempt::Retry(err) => {
15139                            migration_error = Some(err);
15140                            version_attempt += 1;
15141                        }
15142                    }
15143                }
15144
15145                if counted_versions.insert((version.version_id, version.deleted)) {
15146                    let mut pool_meta = self.pool_meta.write().await;
15147                    ensure_decommission_generation(&pool_meta, idx, generation)?;
15148                    if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, !migrated && !consumed) {
15149                        return Err(with_decommission_entry_context(
15150                            "count_decommission_item",
15151                            bucket.as_str(),
15152                            entry.name.as_str(),
15153                            err,
15154                        ));
15155                    }
15156                }
15157
15158                if migrated || consumed {
15159                    decommissioned += 1;
15160                    cleanup_preflight_allowed_missing.push(data_movement::source_cleanup_version_identity(version));
15161                }
15162                if migrated {
15163                    free_version_disposition.record_migrated();
15164                } else if consumed {
15165                    free_version_disposition.record_consumed();
15166                } else {
15167                    free_version_disposition.record_retained();
15168                }
15169
15170                debug!(
15171                    event = EVENT_DECOMMISSION_ENTRY,
15172                    component = LOG_COMPONENT_ECSTORE,
15173                    subsystem = LOG_SUBSYSTEM_POOLS,
15174                    pool_index = idx,
15175                    bucket = %bucket,
15176                    object = %version.name,
15177                    version_id = ?version_id,
15178                    result = ?migration_error,
15179                    reason = if migrated {
15180                        DECOMMISSION_FREE_VERSION_MIGRATED_REASON
15181                    } else if consumed {
15182                        DECOMMISSION_FREE_VERSION_CONSUMED_REASON
15183                    } else {
15184                        DECOMMISSION_FREE_VERSION_RETAINED_REASON
15185                    },
15186                    state = if migrated {
15187                        "free_version_migrated"
15188                    } else if consumed {
15189                        "free_version_consumed"
15190                    } else {
15191                        "free_version_retained"
15192                    },
15193                    "Decommission free-version disposition recorded"
15194                );
15195
15196                if capacity_failure {
15197                    return Err(with_decommission_entry_context(
15198                        "decommission_tier_free_version",
15199                        bucket.as_str(),
15200                        version.name.as_str(),
15201                        migration_error.expect("capacity failure must retain its error"),
15202                    ));
15203                }
15204
15205                if !migrated && !consumed {
15206                    break;
15207                }
15208                continue;
15209            }
15210
15211            if self
15212                .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
15213                    should_skip_lifecycle_for_data_movement(
15214                        self.clone(),
15215                        &bucket,
15216                        version,
15217                        lifecycle_config.as_ref(),
15218                        object_lock_config.as_ref(),
15219                        true,
15220                        &LcEventSrc::Decom,
15221                        None,
15222                    )
15223                    .await
15224                })
15225                .await
15226                .map_err(|err| with_decommission_entry_context("lifecycle_expiry", bucket.as_str(), version.name.as_str(), err))?
15227            {
15228                expired += 1;
15229                cleanup_preflight_allowed_missing.push(data_movement::source_cleanup_version_identity(version));
15230                continue;
15231            }
15232
15233            let remaining_versions = decommission_remaining_version_count(&fivs.versions, expired);
15234            if should_skip_decommission_delete_marker(version, remaining_versions, replication_config.is_some()) {
15235                //
15236                decommissioned += 1;
15237                debug!(
15238                    event = EVENT_DECOMMISSION_ENTRY,
15239                    component = LOG_COMPONENT_ECSTORE,
15240                    subsystem = LOG_SUBSYSTEM_POOLS,
15241                    pool_index = idx,
15242                    bucket = %bucket,
15243                    object = %version.name,
15244                    state = "skipped_delete_marker",
15245                    "Decommission delete marker skipped"
15246                );
15247                continue;
15248            }
15249
15250            let version_id = version.version_id.map(|v| v.to_string());
15251
15252            let mut ignore = false;
15253            let mut cleanup_ignored = false;
15254            let mut failure = false;
15255            let mut error = None;
15256            if version.deleted {
15257                let mut version_attempt = 1;
15258                let mut target_busy_attempt: usize = 0;
15259                let mut target_permit = None;
15260                while version_attempt <= DECOMMISSION_VERSION_COPY_ATTEMPTS {
15261                    if version_attempt > 1 || target_busy_attempt > 0 {
15262                        record_decommission_object_attempt(attempts);
15263                    }
15264                    let result = self
15265                        .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
15266                            self.delete_object(
15267                                bucket.as_str(),
15268                                &version.name,
15269                                decommission_capacity_owned_opts(
15270                                    decommission_delete_marker_opts(
15271                                        version,
15272                                        version_id.clone(),
15273                                        idx,
15274                                        expected_bucket_incarnation_id,
15275                                    ),
15276                                    capacity_owner,
15277                                ),
15278                            )
15279                            .await
15280                        })
15281                        .await;
15282                    drop(target_permit.take());
15283                    if let Some(target_pool_index) = result.as_ref().err().and_then(decommission_capacity_target_gate_busy_index)
15284                    {
15285                        target_busy_attempt = target_busy_attempt.saturating_add(1);
15286                        let Some(permit) = self
15287                            .wait_decommission_target_gate_retry(
15288                                &rx,
15289                                idx,
15290                                generation,
15291                                target_pool_index,
15292                                capacity_owner,
15293                                set.as_ref(),
15294                                &entry,
15295                                &bucket,
15296                                version,
15297                                target_busy_attempt,
15298                            )
15299                            .await?
15300                        else {
15301                            return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
15302                        };
15303                        target_permit = Some(permit);
15304                        continue;
15305                    }
15306                    #[cfg(test)]
15307                    let result = decommission_test_wrap_result(
15308                        "delete_marker_copy",
15309                        bucket.as_str(),
15310                        version.name.as_str(),
15311                        version_attempt,
15312                        result,
15313                    );
15314
15315                    match result {
15316                        Ok(_) => {
15317                            failure = false;
15318                            error = None;
15319                            break;
15320                        }
15321                        Err(err) if is_decommission_copy_cleanup_safe_error(&err) => {
15322                            warn!(
15323                                event = EVENT_DECOMMISSION_ENTRY,
15324                                component = LOG_COMPONENT_ECSTORE,
15325                                subsystem = LOG_SUBSYSTEM_POOLS,
15326                                state = "ignored_delete_marker_copy",
15327                                pool_index = idx,
15328                                bucket = %bucket,
15329                                object = %version.name,
15330                                version_id = ?version_id,
15331                                error = ?err,
15332                                "Decommission delete marker copy ignored"
15333                            );
15334                            ignore = true;
15335                            cleanup_ignored = true;
15336                            break;
15337                        }
15338                        Err(err) if is_decommission_target_capacity_error(&err) => {
15339                            return Err(with_decommission_entry_context(
15340                                "delete_marker_copy",
15341                                bucket.as_str(),
15342                                version.name.as_str(),
15343                                err,
15344                            ));
15345                        }
15346                        Err(err) => {
15347                            failure = true;
15348                            if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS {
15349                                error!(
15350                                    event = EVENT_DECOMMISSION_ENTRY,
15351                                    component = LOG_COMPONENT_ECSTORE,
15352                                    subsystem = LOG_SUBSYSTEM_POOLS,
15353                                    state = "delete_marker_copy_failed",
15354                                    pool_index = idx,
15355                                    bucket = %bucket,
15356                                    object = %version.name,
15357                                    version_id = ?version_id,
15358                                    attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
15359                                    error = ?err,
15360                                    "Decommission delete marker copy failed"
15361                                );
15362                                error = Some(err);
15363                                break;
15364                            }
15365
15366                            let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt);
15367                            warn!(
15368                                event = EVENT_DECOMMISSION_ENTRY,
15369                                component = LOG_COMPONENT_ECSTORE,
15370                                subsystem = LOG_SUBSYSTEM_POOLS,
15371                                state = "delete_marker_copy_retry",
15372                                pool_index = idx,
15373                                bucket = %bucket,
15374                                object = %version.name,
15375                                version_id = ?version_id,
15376                                attempt = version_attempt,
15377                                max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
15378                                retry_delay_ms = retry_delay.as_millis(),
15379                                error = ?err,
15380                                "Decommission delete marker copy failed; retrying"
15381                            );
15382                            error = Some(err);
15383                            if wait_decommission_retry_backoff(&rx, retry_delay).await {
15384                                decommission_cancel_signal_result(rx.is_cancelled())?;
15385                            }
15386                            version_attempt += 1;
15387                        }
15388                    }
15389                }
15390
15391                if ignore {
15392                    if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
15393                        decommissioned += 1;
15394                    }
15395                    debug!(
15396                        event = EVENT_DECOMMISSION_ENTRY,
15397                        component = LOG_COMPONENT_ECSTORE,
15398                        subsystem = LOG_SUBSYSTEM_POOLS,
15399                        pool_index = idx,
15400                        bucket = %bucket,
15401                        object = %version.name,
15402                        state = "ignored",
15403                        "Decommission entry ignored"
15404                    );
15405                    continue;
15406                }
15407
15408                if counted_versions.insert((version.version_id, version.deleted)) {
15409                    let mut pool_meta = self.pool_meta.write().await;
15410                    ensure_decommission_generation(&pool_meta, idx, generation)?;
15411                    if let Err(err) = count_decommission_item(&mut pool_meta, idx, 0, failure) {
15412                        return Err(with_decommission_entry_context(
15413                            "count_decommission_item",
15414                            bucket.as_str(),
15415                            entry.name.as_str(),
15416                            err,
15417                        ));
15418                    }
15419                }
15420
15421                if !failure {
15422                    decommissioned += 1;
15423                }
15424
15425                if !failure {
15426                    debug!(
15427                        event = EVENT_DECOMMISSION_ENTRY,
15428                        component = LOG_COMPONENT_ECSTORE,
15429                        subsystem = LOG_SUBSYSTEM_POOLS,
15430                        state = "delete_marker_copied",
15431                        pool_index = idx,
15432                        bucket = %bucket,
15433                        object = %version.name,
15434                        version_id = ?version_id,
15435                        result = ?error,
15436                        "Decommission delete marker copied"
15437                    );
15438                }
15439                continue;
15440            }
15441
15442            let mut version_attempt = 1;
15443            let mut target_busy_attempt: usize = 0;
15444            let mut target_permit = None;
15445            while version_attempt <= DECOMMISSION_VERSION_COPY_ATTEMPTS {
15446                if version_attempt > 1 || target_busy_attempt > 0 {
15447                    record_decommission_object_attempt(attempts);
15448                }
15449                if version.is_remote() {
15450                    let result = self
15451                        .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
15452                            self.decommission_tiered_object(
15453                                bucket.as_str(),
15454                                &version.name,
15455                                version,
15456                                &decommission_capacity_owned_opts(
15457                                    decommission_remote_tiered_opts(
15458                                        version,
15459                                        version_id.clone(),
15460                                        idx,
15461                                        expected_bucket_incarnation_id,
15462                                    ),
15463                                    capacity_owner,
15464                                ),
15465                            )
15466                            .await
15467                        })
15468                        .await;
15469                    drop(target_permit.take());
15470                    if let Some(target_pool_index) = result.as_ref().err().and_then(decommission_capacity_target_gate_busy_index)
15471                    {
15472                        target_busy_attempt = target_busy_attempt.saturating_add(1);
15473                        let Some(permit) = self
15474                            .wait_decommission_target_gate_retry(
15475                                &rx,
15476                                idx,
15477                                generation,
15478                                target_pool_index,
15479                                capacity_owner,
15480                                set.as_ref(),
15481                                &entry,
15482                                &bucket,
15483                                version,
15484                                target_busy_attempt,
15485                            )
15486                            .await?
15487                        else {
15488                            return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
15489                        };
15490                        target_permit = Some(permit);
15491                        continue;
15492                    }
15493                    #[cfg(test)]
15494                    let result = decommission_test_wrap_result(
15495                        "decommission_tiered_object",
15496                        bucket.as_str(),
15497                        version.name.as_str(),
15498                        version_attempt,
15499                        result,
15500                    );
15501
15502                    match result {
15503                        Ok(_) => {
15504                            failure = false;
15505                            error = None;
15506                        }
15507                        Err(err) if is_decommission_copy_cleanup_safe_error(&err) => {
15508                            ignore = true;
15509                            cleanup_ignored = true;
15510                        }
15511                        Err(err) if is_decommission_target_capacity_error(&err) => {
15512                            return Err(with_decommission_entry_context(
15513                                "decommission_tiered_object",
15514                                bucket.as_str(),
15515                                version.name.as_str(),
15516                                err,
15517                            ));
15518                        }
15519                        Err(err) => {
15520                            failure = true;
15521                            if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS {
15522                                error!(
15523                                    event = EVENT_DECOMMISSION_ENTRY,
15524                                    component = LOG_COMPONENT_ECSTORE,
15525                                    subsystem = LOG_SUBSYSTEM_POOLS,
15526                                    state = "tiered_copy_failed",
15527                                    pool_index = idx,
15528                                    bucket = %bucket,
15529                                    object = %version.name,
15530                                    version_id = ?version_id,
15531                                    attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
15532                                    error = ?err,
15533                                    "Decommission tiered version copy failed"
15534                                );
15535                                error = Some(err);
15536                            } else {
15537                                let retry_delay =
15538                                    decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt);
15539                                warn!(
15540                                    event = EVENT_DECOMMISSION_ENTRY,
15541                                    component = LOG_COMPONENT_ECSTORE,
15542                                    subsystem = LOG_SUBSYSTEM_POOLS,
15543                                    state = "tiered_copy_retry",
15544                                    pool_index = idx,
15545                                    bucket = %bucket,
15546                                    object = %version.name,
15547                                    version_id = ?version_id,
15548                                    attempt = version_attempt,
15549                                    max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
15550                                    retry_delay_ms = retry_delay.as_millis(),
15551                                    error = ?err,
15552                                    "Decommission tiered version copy failed; retrying"
15553                                );
15554                                error = Some(err);
15555                                if wait_decommission_retry_backoff(&rx, retry_delay).await {
15556                                    decommission_cancel_signal_result(rx.is_cancelled())?;
15557                                }
15558                                version_attempt += 1;
15559                                continue;
15560                            }
15561                        }
15562                    }
15563                    break;
15564                }
15565
15566                let bucket = bucket.clone();
15567
15568                let read_result = set
15569                    .get_object_reader(
15570                        bucket.as_str(),
15571                        &encode_dir_object(&version.name),
15572                        None,
15573                        HeaderMap::new(),
15574                        &decommission_object_migration_read_opts(version_id.clone()),
15575                    )
15576                    .await;
15577                #[cfg(test)]
15578                let read_result =
15579                    decommission_test_wrap_result("object_read", &bucket, &version.name, version_attempt, read_result);
15580                let rd = match read_result {
15581                    Ok(rd) => rd,
15582                    Err(err) => {
15583                        if is_err_object_not_found(&err) || is_err_version_not_found(&err) {
15584                            ignore = true;
15585                            cleanup_ignored = true;
15586                            break;
15587                        }
15588
15589                        failure = true;
15590                        if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS {
15591                            error!(
15592                                event = EVENT_DECOMMISSION_ENTRY,
15593                                component = LOG_COMPONENT_ECSTORE,
15594                                subsystem = LOG_SUBSYSTEM_POOLS,
15595                                state = "object_read_failed",
15596                                pool_index = idx,
15597                                bucket = %bucket,
15598                                object = %version.name,
15599                                version_id = ?version_id,
15600                                attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
15601                                error = ?err,
15602                                "Decommission source object read failed"
15603                            );
15604                            error = Some(err);
15605                            break;
15606                        }
15607
15608                        let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt);
15609                        warn!(
15610                            event = EVENT_DECOMMISSION_ENTRY,
15611                            component = LOG_COMPONENT_ECSTORE,
15612                            subsystem = LOG_SUBSYSTEM_POOLS,
15613                            state = "object_read_retry",
15614                            pool_index = idx,
15615                            bucket = %bucket,
15616                            object = %version.name,
15617                            version_id = ?version_id,
15618                            attempt = version_attempt,
15619                            max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
15620                            retry_delay_ms = retry_delay.as_millis(),
15621                            error = ?err,
15622                            "Decommission source object read failed; retrying"
15623                        );
15624                        error = Some(err);
15625                        if wait_decommission_retry_backoff(&rx, retry_delay).await {
15626                            decommission_cancel_signal_result(rx.is_cancelled())?;
15627                        }
15628                        version_attempt += 1;
15629                        continue;
15630                    }
15631                };
15632
15633                let bucket_name = bucket.clone();
15634                let object_name = rd.object_info.name.clone();
15635
15636                self.track_decommission_entry_progress_stage(
15637                    idx,
15638                    generation,
15639                    bucket_name.as_str(),
15640                    object_name.as_str(),
15641                    DECOMMISSION_STAGE_MIGRATE_OBJECT,
15642                )
15643                .await?;
15644
15645                let migrate_result = self
15646                    .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
15647                        self.clone()
15648                            .decommission_object(idx, bucket, rd, expected_bucket_incarnation_id, capacity_owner)
15649                            .await
15650                    })
15651                    .await;
15652                drop(target_permit.take());
15653                if let Some(target_pool_index) = migrate_result
15654                    .as_ref()
15655                    .err()
15656                    .and_then(decommission_capacity_target_gate_busy_index)
15657                {
15658                    target_busy_attempt = target_busy_attempt.saturating_add(1);
15659                    let Some(permit) = self
15660                        .wait_decommission_target_gate_retry(
15661                            &rx,
15662                            idx,
15663                            generation,
15664                            target_pool_index,
15665                            capacity_owner,
15666                            set.as_ref(),
15667                            &entry,
15668                            &bucket_name,
15669                            version,
15670                            target_busy_attempt,
15671                        )
15672                        .await?
15673                    else {
15674                        return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
15675                    };
15676                    target_permit = Some(permit);
15677                    continue;
15678                }
15679                #[cfg(test)]
15680                let migrate_result = decommission_test_wrap_result(
15681                    DECOMMISSION_STAGE_MIGRATE_OBJECT,
15682                    bucket_name.as_str(),
15683                    object_name.as_str(),
15684                    version_attempt,
15685                    migrate_result,
15686                );
15687
15688                if let Err(err) = migrate_result {
15689                    if is_decommission_copy_cleanup_safe_error(&err) {
15690                        ignore = true;
15691                        cleanup_ignored = true;
15692                        break;
15693                    }
15694
15695                    if is_decommission_target_capacity_error(&err) {
15696                        return Err(with_decommission_entry_context(
15697                            DECOMMISSION_STAGE_MIGRATE_OBJECT,
15698                            bucket_name.as_str(),
15699                            object_name.as_str(),
15700                            err,
15701                        ));
15702                    }
15703
15704                    failure = true;
15705                    if version_attempt == DECOMMISSION_VERSION_COPY_ATTEMPTS {
15706                        error!(
15707                            event = EVENT_DECOMMISSION_ENTRY,
15708                            component = LOG_COMPONENT_ECSTORE,
15709                            subsystem = LOG_SUBSYSTEM_POOLS,
15710                            state = "object_migration_failed",
15711                            pool_index = idx,
15712                            bucket = %bucket_name,
15713                            object = %object_name,
15714                            version = %version.name,
15715                            attempt = version_attempt,
15716                            max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
15717                            error = ?err,
15718                            "Decommission object migration failed"
15719                        );
15720                        error = Some(err);
15721                        break;
15722                    }
15723
15724                    let retry_delay = decommission_retry_backoff_delay(DECOMMISSION_COPY_RETRY_DELAY, version_attempt);
15725                    warn!(
15726                        event = EVENT_DECOMMISSION_ENTRY,
15727                        component = LOG_COMPONENT_ECSTORE,
15728                        subsystem = LOG_SUBSYSTEM_POOLS,
15729                        state = "object_migration_retry",
15730                        pool_index = idx,
15731                        bucket = %bucket_name,
15732                        object = %object_name,
15733                        version = %version.name,
15734                        attempt = version_attempt,
15735                        max_attempts = DECOMMISSION_VERSION_COPY_ATTEMPTS,
15736                        retry_delay_ms = retry_delay.as_millis(),
15737                        error = ?err,
15738                        "Decommission object migration failed; retrying"
15739                    );
15740                    error = Some(err);
15741                    if wait_decommission_retry_backoff(&rx, retry_delay).await {
15742                        decommission_cancel_signal_result(rx.is_cancelled())?;
15743                    }
15744                    version_attempt += 1;
15745                    continue;
15746                }
15747
15748                warn!(
15749                    event = EVENT_DECOMMISSION_ENTRY,
15750                    component = LOG_COMPONENT_ECSTORE,
15751                    subsystem = LOG_SUBSYSTEM_POOLS,
15752                    pool_index = idx,
15753                    bucket = %bucket_name,
15754                    object = %object_name,
15755                    version = %version.name,
15756                    state = "object_migrated",
15757                    "Decommission object migrated"
15758                );
15759
15760                failure = false;
15761                break;
15762            }
15763
15764            if ignore {
15765                if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
15766                    decommissioned += 1;
15767                }
15768                debug!(
15769                    event = EVENT_DECOMMISSION_ENTRY,
15770                    component = LOG_COMPONENT_ECSTORE,
15771                    subsystem = LOG_SUBSYSTEM_POOLS,
15772                    pool_index = idx,
15773                    bucket = %bucket,
15774                    object = %version.name,
15775                    state = "ignored",
15776                    "Decommission entry ignored"
15777                );
15778                continue;
15779            }
15780
15781            if counted_versions.insert((version.version_id, version.deleted)) {
15782                let mut pool_meta = self.pool_meta.write().await;
15783                ensure_decommission_generation(&pool_meta, idx, generation)?;
15784                if let Err(err) = count_decommission_item(&mut pool_meta, idx, decommission_item_size(version.size), failure) {
15785                    return Err(with_decommission_entry_context(
15786                        "count_decommission_item",
15787                        bucket.as_str(),
15788                        entry.name.as_str(),
15789                        err,
15790                    ));
15791                }
15792            }
15793
15794            if failure {
15795                break;
15796            }
15797
15798            if should_count_decommission_version_complete(ignore, cleanup_ignored, failure) {
15799                decommissioned += 1;
15800            }
15801        }
15802
15803        if free_version_disposition.total() > 0 {
15804            debug!(
15805                event = EVENT_DECOMMISSION_ENTRY,
15806                component = LOG_COMPONENT_ECSTORE,
15807                subsystem = LOG_SUBSYSTEM_POOLS,
15808                pool_index = idx,
15809                bucket = %bucket,
15810                object = %entry.name,
15811                free_versions_migrated = free_version_disposition.migrated,
15812                free_versions_consumed = free_version_disposition.consumed,
15813                free_versions_retained = free_version_disposition.retained,
15814                free_versions_total = free_version_disposition.total(),
15815                reason = DECOMMISSION_FREE_VERSION_DISPOSITION_REASON,
15816                state = "free_version_disposition",
15817                "Decommission free-version disposition summary"
15818            );
15819        }
15820
15821        if should_cleanup_decommission_source_entry(decommissioned, fivs.versions.len(), expired) && durable_ilm_record.is_none()
15822        {
15823            if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
15824                return Err(Error::other("decommission bucket incarnation fence was lost before source cleanup"));
15825            }
15826            decommission_cancel_signal_result(rx.is_cancelled())?;
15827            self.ensure_decommission_generation_current(idx, generation).await?;
15828
15829            self.track_decommission_entry_progress_stage(
15830                idx,
15831                generation,
15832                bucket.as_str(),
15833                entry.name.as_str(),
15834                DECOMMISSION_STAGE_CLEANUP_PREFLIGHT,
15835            )
15836            .await?;
15837
15838            self.track_decommission_entry_progress_stage(
15839                idx,
15840                generation,
15841                bucket.as_str(),
15842                entry.name.as_str(),
15843                DECOMMISSION_STAGE_SOURCE_CLEANUP,
15844            )
15845            .await?;
15846
15847            #[cfg(test)]
15848            run_decommission_cleanup_mutation_hook(bucket.as_str(), entry.name.as_str(), entry_attempt).await;
15849
15850            let source_cleanup_mutation_fence = self
15851                .acquire_decommission_source_cleanup_fence(bucket.as_str(), entry.name.as_str(), set.as_ref())
15852                .await?;
15853            let cleanup_result = self
15854                .run_guarded_decommission_side_effect(&rx, &operation_gate, || async {
15855                    data_movement::cleanup_source_entry_if_unchanged(
15856                        set.clone(),
15857                        bucket.as_str(),
15858                        entry.name.as_str(),
15859                        &fivs,
15860                        &cleanup_preflight_allowed_missing,
15861                        data_movement::SourceCleanupBucketFence {
15862                            expected_incarnation_id: expected_bucket_incarnation_id,
15863                            lifecycle_guard: bucket_incarnation_fence
15864                                .as_ref()
15865                                .and_then(|guard| guard.namespace_lock_guard()),
15866                            namespace_lock_lost_signal: None,
15867                            object_mutation_fence: Some(&source_cleanup_mutation_fence),
15868                        },
15869                        "decommission",
15870                    )
15871                    .await
15872                })
15873                .await;
15874            match cleanup_result {
15875                Ok(_) => {}
15876                Err(data_movement::SourceCleanupError::Storage(err)) => {
15877                    resolve_decommission_entry_cleanup_delete_result(
15878                        Err::<(), Error>(err),
15879                        bucket.as_str(),
15880                        entry.name.as_str(),
15881                    )?;
15882                }
15883                Err(data_movement::SourceCleanupError::SourceChanged) if entry_attempt < DECOMMISSION_ENTRY_MAX_ATTEMPTS => {
15884                    return Ok(DecommissionEntryAttemptOutcome::SourceChanged);
15885                }
15886                Err(data_movement::SourceCleanupError::SourceChanged) => {
15887                    let exhausted_entries = source_changed_exhaustions.fetch_add(1, Ordering::Relaxed) + 1;
15888                    if should_fail_decommission_pool_after_exhausted_source_changed(exhausted_entries) {
15889                        return Err(Error::other(format!(
15890                            "decommission source cleanup retries exhausted for {}/{} on all {} attempts; exhausted entries exceed pool limit {}",
15891                            bucket, entry.name, DECOMMISSION_ENTRY_MAX_ATTEMPTS, DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT
15892                        )));
15893                    }
15894
15895                    {
15896                        let mut pool_meta = self.pool_meta.write().await;
15897                        ensure_decommission_generation(&pool_meta, idx, generation)?;
15898                        count_decommission_item(&mut pool_meta, idx, 0, true).map_err(|err| {
15899                            with_decommission_entry_context(
15900                                "count_source_changed_exhaustion",
15901                                bucket.as_str(),
15902                                entry.name.as_str(),
15903                                err,
15904                            )
15905                        })?;
15906                    }
15907
15908                    error!(
15909                        event = EVENT_DECOMMISSION_ENTRY,
15910                        component = LOG_COMPONENT_ECSTORE,
15911                        subsystem = LOG_SUBSYSTEM_POOLS,
15912                        state = "source_cleanup_exhausted",
15913                        pool_index = idx,
15914                        bucket = %bucket,
15915                        object = %entry.name,
15916                        attempts = DECOMMISSION_ENTRY_MAX_ATTEMPTS,
15917                        exhausted_entries,
15918                        exhaustion_limit = DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT,
15919                        "Decommission source cleanup retries exhausted; source retained and entry marked failed"
15920                    );
15921                    entry_blocked = true;
15922                }
15923            }
15924        } else if durable_ilm_record.is_some() {
15925            debug!(
15926                event = EVENT_DECOMMISSION_ENTRY,
15927                component = LOG_COMPONENT_ECSTORE,
15928                subsystem = LOG_SUBSYSTEM_POOLS,
15929                pool_index = idx,
15930                bucket = %bucket,
15931                object = %entry.name,
15932                state = "retained_for_final_verification",
15933                "Decommission durable ILM source retained for final verification"
15934            );
15935        } else if decommissioned != fivs.versions.len() || expired > 0 {
15936            warn!(
15937                event = EVENT_DECOMMISSION_ENTRY,
15938                component = LOG_COMPONENT_ECSTORE,
15939                subsystem = LOG_SUBSYSTEM_POOLS,
15940                pool_index = idx,
15941                bucket = %bucket,
15942                object = %entry.name,
15943                decommissioned,
15944                total_versions = fivs.versions.len(),
15945                expired,
15946                state = "source_retained",
15947                "Decommission source object retained"
15948            );
15949        }
15950
15951        let should_save_progress = {
15952            let mut pool_meta = self.pool_meta.write().await;
15953            ensure_decommission_generation(&pool_meta, idx, generation)?;
15954
15955            if let Err(err) = track_decommission_current_object(&mut pool_meta, idx, bucket.as_str(), entry.name.as_str()) {
15956                return Err(with_decommission_entry_context(
15957                    "track_decommission_current_object",
15958                    bucket.as_str(),
15959                    entry.name.as_str(),
15960                    err,
15961                ));
15962            }
15963
15964            match resolve_decommission_update_after_result(pool_meta.update_after(idx, DECOMMISSION_PROGRESS_SAVE_INTERVAL)) {
15965                Ok(ok) => ok,
15966                Err(err) => {
15967                    return Err(with_decommission_entry_context("update_after", bucket.as_str(), entry.name.as_str(), err));
15968                }
15969            }
15970        };
15971
15972        self.track_decommission_entry_progress_stage(
15973            idx,
15974            generation,
15975            bucket.as_str(),
15976            entry.name.as_str(),
15977            DECOMMISSION_STAGE_ENTRY_FINISHED,
15978        )
15979        .await?;
15980
15981        if should_save_progress {
15982            match self.save_decommission_progress_checkpoint(idx, generation).await {
15983                Ok(true) => {
15984                    if let Some(notification_sys) = runtime_sources::notification_sys()
15985                        && let Err(err) = resolve_decommission_entry_reload_result(
15986                            notification_sys.reload_pool_meta().await,
15987                            bucket.as_str(),
15988                            entry.name.as_str(),
15989                        )
15990                    {
15991                        warn!("{err}");
15992                    }
15993                }
15994                Ok(false) => {}
15995                Err(err) => {
15996                    if let Some(err) = resolve_decommission_progress_save_result(Err(err)) {
15997                        warn!(
15998                            event = EVENT_DECOMMISSION_ENTRY,
15999                            component = LOG_COMPONENT_ECSTORE,
16000                            subsystem = LOG_SUBSYSTEM_POOLS,
16001                            pool_index = idx,
16002                            bucket = %bucket,
16003                            object = %entry.name,
16004                            state = "progress_save_failed",
16005                            error = %err,
16006                            "Decommission progress save failed; continuing and will retry at the next checkpoint"
16007                        );
16008                    }
16009                }
16010            }
16011        }
16012
16013        debug!(
16014            event = EVENT_DECOMMISSION_ENTRY,
16015            component = LOG_COMPONENT_ECSTORE,
16016            subsystem = LOG_SUBSYSTEM_POOLS,
16017            state = if entry_blocked { "blocked" } else { "completed" },
16018            pool_index = idx,
16019            bucket = %bucket,
16020            object = %entry.name,
16021            "Decommission entry finished"
16022        );
16023        Ok(DecommissionEntryAttemptOutcome::Complete)
16024    }
16025
16026    #[cfg(test)]
16027    pub(crate) async fn decommission_entry_for_test(
16028        self: &Arc<Self>,
16029        idx: usize,
16030        entry: MetaCacheEntry,
16031        bucket: String,
16032        set: Arc<SetDisks>,
16033    ) -> Result<()> {
16034        self.decommission_entry_with_retry_state_for_test(
16035            CancellationToken::new(),
16036            idx,
16037            entry,
16038            bucket,
16039            set,
16040            None,
16041            Arc::new(AtomicUsize::new(0)),
16042        )
16043        .await
16044    }
16045
16046    #[cfg(test)]
16047    #[allow(clippy::too_many_arguments)]
16048    pub(crate) async fn decommission_entry_with_retry_state_for_test(
16049        self: &Arc<Self>,
16050        rx: CancellationToken,
16051        idx: usize,
16052        entry: MetaCacheEntry,
16053        bucket: String,
16054        set: Arc<SetDisks>,
16055        expected_bucket_incarnation_id: Option<uuid::Uuid>,
16056        source_changed_exhaustions: Arc<AtomicUsize>,
16057    ) -> Result<()> {
16058        {
16059            let mut cancelers = self.decommission_cancelers.write().await;
16060            if cancelers.get(idx).and_then(Option::as_ref).is_none()
16061                && let Some(slot) = cancelers.get_mut(idx)
16062            {
16063                *slot = Some(DecommissionCanceler::new(CancellationToken::new()));
16064            }
16065        }
16066        let needs_capacity_reservation = {
16067            let pool_meta = self.pool_meta.read().await;
16068            pool_meta
16069                .pools
16070                .get(idx)
16071                .and_then(|pool| pool.decommission.as_ref())
16072                .and_then(|info| info.capacity_reservation.as_ref())
16073                .is_some_and(|reservation| !reservation.active())
16074        };
16075        if needs_capacity_reservation {
16076            let capacity_infos = self.get_decommission_all_pool_capacity_infos_with_owned_usage().await?;
16077            let mut pool_meta = self.pool_meta.write().await;
16078            let version = pool_meta.version;
16079            pool_meta.version = POOL_META_VERSION;
16080            recover_decommission_capacity_reservations(
16081                &mut pool_meta,
16082                &capacity_infos,
16083                OffsetDateTime::now_utc(),
16084                crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof().is_some(),
16085            )?;
16086            pool_meta.version = version;
16087        }
16088        let generation = self.active_decommission_generation(idx).await?;
16089        resolve_decommission_entry_test_outcome(
16090            self.decommission_entry(
16091                rx,
16092                idx,
16093                generation,
16094                entry,
16095                bucket,
16096                set,
16097                None,
16098                None,
16099                None,
16100                expected_bucket_incarnation_id,
16101                source_changed_exhaustions,
16102                &mut 0,
16103            )
16104            .await,
16105        )
16106    }
16107
16108    #[cfg(all(test, feature = "test-util"))]
16109    pub(crate) async fn decommission_entry_for_test_with_bucket_incarnation(
16110        self: &Arc<Self>,
16111        idx: usize,
16112        entry: MetaCacheEntry,
16113        bucket: String,
16114        set: Arc<SetDisks>,
16115    ) -> Result<()> {
16116        let expected_bucket_incarnation_id = if is_meta_bucketname(&bucket) {
16117            None
16118        } else {
16119            Some(self.bucket_incarnation_id_from_disk(&bucket).await?)
16120        };
16121        let generation = self.active_decommission_generation(idx).await?;
16122        resolve_decommission_entry_test_outcome(
16123            self.decommission_entry(
16124                CancellationToken::new(),
16125                idx,
16126                generation,
16127                entry,
16128                bucket,
16129                set,
16130                None,
16131                None,
16132                None,
16133                expected_bucket_incarnation_id,
16134                Arc::new(AtomicUsize::new(0)),
16135                &mut 0,
16136            )
16137            .await,
16138        )
16139    }
16140
16141    #[tracing::instrument(skip(self, rx))]
16142    async fn decommission_pool(
16143        self: &Arc<Self>,
16144        rx: CancellationToken,
16145        idx: usize,
16146        pool: Arc<Sets>,
16147        bi: DecomBucketInfo,
16148        entry_budget: Arc<Semaphore>,
16149        source_changed_exhaustions: Arc<AtomicUsize>,
16150    ) -> Result<()> {
16151        let entry_error = Arc::new(tokio::sync::Mutex::new(None::<Error>));
16152        let generation = self.active_decommission_generation(idx).await?;
16153        let mut listing_workers = Vec::with_capacity(pool.disk_set.len());
16154
16155        let mut lifecycle_config = None;
16156        let mut object_lock_config = None;
16157        let mut replication_config = None;
16158        let expected_bucket_incarnation_id = if bi.name == RUSTFS_META_BUCKET {
16159            None
16160        } else {
16161            Some(self.bucket_incarnation_id_from_disk(&bi.name).await?)
16162        };
16163
16164        if bi.name != RUSTFS_META_BUCKET {
16165            let _ = resolve_decommission_optional_bucket_config_result(
16166                &bi.name,
16167                "versioning",
16168                BucketVersioningSys::get_in(&self.ctx, &bi.name).await,
16169            )?;
16170            let expiry_configs = get_expiry_configs(self, &bi.name).await?;
16171            lifecycle_config = expiry_configs.lifecycle.map(|config| (*config).clone());
16172            object_lock_config = expiry_configs.object_lock.map(|config| (*config).clone());
16173            replication_config = resolve_decommission_optional_bucket_config_result(
16174                &bi.name,
16175                "replication",
16176                metadata_sys::get_replication_config_in(&self.ctx, &bi.name).await,
16177            )?;
16178        }
16179
16180        for (set_idx, set) in pool.disk_set.iter().enumerate() {
16181            debug!(
16182                event = EVENT_DECOMMISSION_BUCKET,
16183                component = LOG_COMPONENT_ECSTORE,
16184                subsystem = LOG_SUBSYSTEM_POOLS,
16185                pool_index = idx,
16186                set_index = set_idx,
16187                bucket = %bi.name,
16188                state = "listing_worker_started",
16189                "Decommission listing worker started"
16190            );
16191
16192            let set = set.clone();
16193            let store = Arc::clone(self);
16194            let rx_clone = rx.clone();
16195            let bi_clone = bi.clone();
16196            let lifecycle_config = lifecycle_config.clone();
16197            let object_lock_config = object_lock_config.clone();
16198            let replication_config = replication_config.clone();
16199            let entry_budget = entry_budget.clone();
16200            let source_changed_exhaustions = Arc::clone(&source_changed_exhaustions);
16201            let entry_error = entry_error.clone();
16202            let worker = tokio::spawn(async move {
16203                store
16204                    .decommission_set(
16205                        rx_clone,
16206                        idx,
16207                        set_idx,
16208                        generation,
16209                        set,
16210                        bi_clone,
16211                        lifecycle_config,
16212                        object_lock_config,
16213                        replication_config,
16214                        expected_bucket_incarnation_id,
16215                        source_changed_exhaustions,
16216                        entry_budget,
16217                        entry_error,
16218                    )
16219                    .await
16220            });
16221            listing_workers.push((set_idx, worker));
16222        }
16223
16224        debug!(
16225            event = EVENT_DECOMMISSION_BUCKET,
16226            component = LOG_COMPONENT_ECSTORE,
16227            subsystem = LOG_SUBSYSTEM_POOLS,
16228            pool_index = idx,
16229            bucket = %bi.name,
16230            state = "waiting_for_workers",
16231            "Decommission waiting for workers"
16232        );
16233
16234        let mut listing_worker_error = None;
16235        for (set_id, worker) in listing_workers {
16236            if let Err(err) = resolve_decommission_listing_worker_result(set_id, worker.await) {
16237                rx.cancel();
16238                if listing_worker_error.is_none() {
16239                    listing_worker_error = Some(err);
16240                }
16241            }
16242        }
16243
16244        if let Some(err) = listing_worker_error {
16245            return Err(err);
16246        }
16247
16248        if let Some(err) = entry_error.lock().await.clone() {
16249            return Err(err);
16250        }
16251
16252        if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) {
16253            warn!(
16254                event = EVENT_DECOMMISSION_BUCKET,
16255                component = LOG_COMPONENT_ECSTORE,
16256                subsystem = LOG_SUBSYSTEM_POOLS,
16257                pool_index = idx,
16258                bucket = %bi.name,
16259                state = "cancelled_after_wait",
16260                error = %err,
16261                "Decommission bucket cancelled after wait"
16262            );
16263            return Err(err);
16264        }
16265
16266        debug!(
16267            event = EVENT_DECOMMISSION_BUCKET,
16268            component = LOG_COMPONENT_ECSTORE,
16269            subsystem = LOG_SUBSYSTEM_POOLS,
16270            pool_index = idx,
16271            bucket = %bi.name,
16272            state = "completed",
16273            "Decommission bucket completed"
16274        );
16275
16276        Ok(())
16277    }
16278
16279    #[cfg(all(test, feature = "test-util"))]
16280    pub(crate) async fn decommission_pool_for_test(
16281        self: &Arc<Self>,
16282        rx: CancellationToken,
16283        idx: usize,
16284        pool: Arc<Sets>,
16285        bucket: DecomBucketInfo,
16286    ) -> Result<()> {
16287        self.decommission_pool(
16288            rx,
16289            idx,
16290            pool,
16291            bucket,
16292            Arc::new(Semaphore::new(decommission_entry_concurrency_limit())),
16293            Arc::new(AtomicUsize::new(0)),
16294        )
16295        .await
16296    }
16297
16298    #[tracing::instrument(skip(self, canceler))]
16299    pub async fn do_decommission_in_routine(
16300        self: &Arc<Self>,
16301        canceler: DecommissionCanceler,
16302        idx: usize,
16303        entry_budget: Arc<Semaphore>,
16304    ) -> Result<()> {
16305        let rx = canceler.token().clone();
16306        self.run_decommission_in_routine(rx, idx, &canceler, entry_budget).await
16307    }
16308
16309    async fn run_decommission_in_routine(
16310        self: &Arc<Self>,
16311        rx: CancellationToken,
16312        idx: usize,
16313        canceler: &DecommissionCanceler,
16314        entry_budget: Arc<Semaphore>,
16315    ) -> Result<()> {
16316        self.ensure_pool_meta_side_effects_safe("decommission cannot run while pool metadata requires recovery")
16317            .await?;
16318        let generation = match self.promote_queued_decommission(idx, canceler).await {
16319            Ok(generation) => generation,
16320            Err(Error::OperationCanceled) => return Ok(()),
16321            Err(err) if is_decommission_capacity_blocked_error(&err) || is_decommission_target_capacity_error(&err) => {
16322                if let Err(pause_err) = self.pause_decommission_for_capacity(idx, &err).await {
16323                    return Err(decommission_capacity_blocked_error(format!(
16324                        "failed to persist paused state for pool {idx}: {pause_err}"
16325                    )));
16326                }
16327                return Ok(());
16328            }
16329            Err(err) => {
16330                resolve_decommission_terminal_mark_after_error_result(
16331                    self.decommission_failed_for_operation(idx, canceler).await,
16332                    idx,
16333                    &err,
16334                )?;
16335                return Err(err);
16336            }
16337        };
16338        if rx.is_cancelled() {
16339            let already_canceled = {
16340                let pool_meta = self.pool_meta.read().await;
16341                should_skip_canceled_decommission_routine(true, pool_meta.pools.get(idx))
16342            };
16343            if already_canceled {
16344                warn!(
16345                    event = EVENT_DECOMMISSION_STATE,
16346                    component = LOG_COMPONENT_ECSTORE,
16347                    subsystem = LOG_SUBSYSTEM_POOLS,
16348                    pool_index = idx,
16349                    state = "canceled_preserved",
16350                    "Decommission routine skipped because pool is already canceled"
16351                );
16352                return Ok(());
16353            }
16354            if let Err(err) = self.decommission_cancel_for_operation(idx, canceler).await {
16355                resolve_decommission_terminal_mark_after_error_result(
16356                    self.decommission_failed_for_operation(idx, canceler).await,
16357                    idx,
16358                    &err,
16359                )?;
16360                return Err(err);
16361            }
16362            return Ok(());
16363        }
16364        let result = self
16365            .decommission_in_background(rx.clone(), idx, generation, entry_budget)
16366            .await;
16367
16368        if let Err(err) = &result
16369            && (is_decommission_capacity_blocked_error(err) || is_decommission_target_capacity_error(err))
16370        {
16371            if let Err(pause_err) = self.pause_decommission_for_capacity(idx, err).await {
16372                return Err(decommission_capacity_blocked_error(format!(
16373                    "failed to persist paused state for pool {idx}: {pause_err}"
16374                )));
16375            }
16376            return Ok(());
16377        }
16378
16379        let (final_state, canceled, cmd_line) = {
16380            let pool_meta = self.pool_meta.read().await;
16381            let Some(pool) = pool_meta.pools.get(idx) else {
16382                error!(
16383                    event = EVENT_DECOMMISSION_STATE,
16384                    component = LOG_COMPONENT_ECSTORE,
16385                    subsystem = LOG_SUBSYSTEM_POOLS,
16386                    pool_index = idx,
16387                    state = "pool_metadata_missing",
16388                    "Decommission pool metadata missing"
16389                );
16390                return Err(Error::other(format!(
16391                    "failed to resolve decommission final state: pool metadata missing for idx {idx}"
16392                )));
16393            };
16394
16395            let (final_state, canceled) = if let Some(info) = &pool.decommission {
16396                (
16397                    determine_decommission_final_state(info.items_decommission_failed, info.canceled),
16398                    info.canceled,
16399                )
16400            } else {
16401                (DecommissionFinalState::Failed, false)
16402            };
16403            let cmd_line = pool.cmd_line.clone();
16404            (final_state, canceled, cmd_line)
16405        };
16406
16407        if let Err(err) = result {
16408            error!(
16409                event = EVENT_DECOMMISSION_STATE,
16410                component = LOG_COMPONENT_ECSTORE,
16411                subsystem = LOG_SUBSYSTEM_POOLS,
16412                pool_index = idx,
16413                state = "background_failed",
16414                error = ?err,
16415                "Decommission background routine failed"
16416            );
16417
16418            if should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
16419                warn!(
16420                    event = EVENT_DECOMMISSION_STATE,
16421                    component = LOG_COMPONENT_ECSTORE,
16422                    subsystem = LOG_SUBSYSTEM_POOLS,
16423                    pool_index = idx,
16424                    cmd_line = %cmd_line,
16425                    state = "cancelled_preserved",
16426                    "Decommission cancelled; preserving canceled state"
16427                );
16428                return Ok(());
16429            }
16430
16431            resolve_decommission_terminal_mark_after_error_result(
16432                self.decommission_failed_for_operation(idx, canceler).await,
16433                idx,
16434                &err,
16435            )?;
16436            warn!(
16437                event = EVENT_DECOMMISSION_STATE,
16438                component = LOG_COMPONENT_ECSTORE,
16439                subsystem = LOG_SUBSYSTEM_POOLS,
16440                pool_index = idx,
16441                state = "marked_failed",
16442                "Decommission marked failed"
16443            );
16444
16445            return Ok(());
16446        }
16447
16448        debug!(
16449            event = EVENT_DECOMMISSION_STATE,
16450            component = LOG_COMPONENT_ECSTORE,
16451            subsystem = LOG_SUBSYSTEM_POOLS,
16452            pool_index = idx,
16453            state = "background_complete",
16454            "Decommission background routine completed"
16455        );
16456
16457        if should_preserve_decommission_canceled_state(canceled, rx.is_cancelled()) {
16458            warn!(
16459                event = EVENT_DECOMMISSION_STATE,
16460                component = LOG_COMPONENT_ECSTORE,
16461                subsystem = LOG_SUBSYSTEM_POOLS,
16462                pool_index = idx,
16463                cmd_line = %cmd_line,
16464                state = "terminal_state_preserved",
16465                "Decommission terminal state preserved after cancellation"
16466            );
16467            return Ok(());
16468        }
16469
16470        match final_state {
16471            DecommissionFinalState::Complete => {
16472                debug!(
16473                    event = EVENT_DECOMMISSION_STATE,
16474                    component = LOG_COMPONENT_ECSTORE,
16475                    subsystem = LOG_SUBSYSTEM_POOLS,
16476                    pool_index = idx,
16477                    cmd_line = %cmd_line,
16478                    state = "verifying_completion",
16479                    "Decommission completion verification started"
16480                );
16481                let verified_unresolved_entries = match self.check_after_decommission(idx, &rx, generation).await {
16482                    Ok(verified_entries) => verified_entries,
16483                    Err(err) => {
16484                        if is_err_operation_canceled(&err) {
16485                            return Err(err);
16486                        }
16487                        resolve_decommission_terminal_mark_result(
16488                            self.decommission_failed_for_operation(idx, canceler).await,
16489                            "failed",
16490                            &cmd_line,
16491                        )?;
16492                        return Err(Error::other(format!(
16493                            "failed to finalize decommission for pool {cmd_line}: post-check failed: {err}"
16494                        )));
16495                    }
16496                };
16497                info!(
16498                    event = EVENT_DECOMMISSION_STATE,
16499                    component = LOG_COMPONENT_ECSTORE,
16500                    subsystem = LOG_SUBSYSTEM_POOLS,
16501                    pool_index = idx,
16502                    cmd_line = %cmd_line,
16503                    state = "marking_completed",
16504                    "Decommission marking completed state"
16505                );
16506                if let Err(err) = self
16507                    .complete_decommission_for_operation(idx, canceler, generation, verified_unresolved_entries)
16508                    .await
16509                {
16510                    resolve_decommission_terminal_mark_result(
16511                        self.decommission_failed_for_operation(idx, canceler).await,
16512                        "failed",
16513                        &cmd_line,
16514                    )?;
16515                    return Err(Error::other(format!("failed to finalize decommission for pool {cmd_line}: {err}")));
16516                }
16517            }
16518            DecommissionFinalState::Failed => {
16519                warn!(
16520                    event = EVENT_DECOMMISSION_STATE,
16521                    component = LOG_COMPONENT_ECSTORE,
16522                    subsystem = LOG_SUBSYSTEM_POOLS,
16523                    pool_index = idx,
16524                    cmd_line = %cmd_line,
16525                    state = "marking_failed",
16526                    "Decommission marking failed state"
16527                );
16528                resolve_decommission_terminal_mark_result(
16529                    self.decommission_failed_for_operation(idx, canceler).await,
16530                    "failed",
16531                    &cmd_line,
16532                )?;
16533            }
16534        }
16535
16536        info!(
16537            event = EVENT_DECOMMISSION_STATE,
16538            component = LOG_COMPONENT_ECSTORE,
16539            subsystem = LOG_SUBSYSTEM_POOLS,
16540            pool_index = idx,
16541            cmd_line = %cmd_line,
16542            state = "completed",
16543            "Decommission completed"
16544        );
16545        Ok(())
16546    }
16547
16548    #[tracing::instrument(skip(self))]
16549    pub async fn decommission_failed(&self, idx: usize) -> Result<()> {
16550        self.decommission_failed_with_owner(idx, None).await
16551    }
16552
16553    async fn decommission_failed_for_operation(&self, idx: usize, owner: &DecommissionCanceler) -> Result<()> {
16554        self.decommission_failed_with_owner(idx, Some(owner)).await
16555    }
16556
16557    async fn decommission_failed_with_owner(&self, idx: usize, owner: Option<&DecommissionCanceler>) -> Result<()> {
16558        self.decommission_failed_with_owner_and_save(idx, owner, self.save_current_pool_meta(&[idx]))
16559            .await
16560    }
16561
16562    async fn decommission_failed_with_owner_and_save<SaveFuture>(
16563        &self,
16564        idx: usize,
16565        owner: Option<&DecommissionCanceler>,
16566        save_pool_meta: SaveFuture,
16567    ) -> Result<()>
16568    where
16569        SaveFuture: Future<Output = Result<()>>,
16570    {
16571        ensure_decommission_terminal_operation_supported(self.single_pool(), "mark decommission failed")?;
16572        let _start_guard = self.start_gate.lock().await;
16573        let movement_gate = self.ctx.data_movement_operation_gate();
16574        let _movement_guard = movement_gate.write().await;
16575
16576        // Lock order: movement gate, rebalance_meta, decommission_cancelers,
16577        // then pool_meta. Holding both state locks makes owner validation and
16578        // the terminal transition one atomic operation.
16579        let rebalance_meta = self.rebalance_meta.read().await.clone();
16580        let terminal_at = OffsetDateTime::now_utc();
16581        let (should_reload_pool_meta, previous_pool_meta, terminal_canceler) = {
16582            let cancelers = self.decommission_cancelers.read().await;
16583            let mut pool_meta = self.pool_meta.write().await;
16584            let previous_pool_meta = pool_meta.clone();
16585            let Some(changed) =
16586                update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, idx, owner, |pool_meta| {
16587                    pool_meta.decommission_failed_at(idx, terminal_at, rebalance_meta.as_ref())
16588                })
16589            else {
16590                return Ok(());
16591            };
16592            let terminal_canceler = if let Some(owner) = owner {
16593                Some(owner.clone())
16594            } else {
16595                cancelers.get(idx).and_then(Option::as_ref).cloned()
16596            };
16597            (changed, changed.then_some(previous_pool_meta), terminal_canceler)
16598        };
16599
16600        if should_reload_pool_meta && let Err(err) = save_pool_meta.await {
16601            if let Some(previous_pool_meta) = previous_pool_meta {
16602                let mut pool_meta = self.pool_meta.write().await;
16603                rollback_decommission_pool_meta(&mut pool_meta, &previous_pool_meta, &[idx]);
16604            }
16605            return Err(err);
16606        }
16607        if should_reload_pool_meta {
16608            {
16609                let mut pool_meta = self.pool_meta.write().await;
16610                pool_meta.mark_decommission_progress_saved();
16611            }
16612        }
16613        if let Some(canceler) = terminal_canceler.as_ref() {
16614            self.release_decommission_canceler_slot(idx, canceler).await;
16615        }
16616
16617        if should_reload_pool_meta {
16618            self.ctx.advance_data_movement_operation_epoch();
16619        }
16620        drop(_movement_guard);
16621
16622        if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
16623            let stage = format!("decommission_failed for pool {idx}");
16624            if let Some(err) = observe_decommission_terminal_reload_result(
16625                resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()),
16626                stage.as_str(),
16627            ) {
16628                if let Err(record_err) = self
16629                    .record_decommission_terminal_reload_failure(idx, stage.as_str(), err.clone())
16630                    .await
16631                {
16632                    warn!(
16633                        event = EVENT_DECOMMISSION_STATE,
16634                        component = LOG_COMPONENT_ECSTORE,
16635                        subsystem = LOG_SUBSYSTEM_POOLS,
16636                        pool_index = idx,
16637                        state = "terminal_reload_record_failed",
16638                        error = %record_err,
16639                        original_error = %err,
16640                        "Decommission terminal reload failure record failed"
16641                    );
16642                }
16643                warn!(
16644                    event = EVENT_DECOMMISSION_STATE,
16645                    component = LOG_COMPONENT_ECSTORE,
16646                    subsystem = LOG_SUBSYSTEM_POOLS,
16647                    pool_index = idx,
16648                    state = "terminal_reload_failed",
16649                    error = %err,
16650                    "Decommission terminal state saved but pool meta reload failed"
16651                );
16652            }
16653        }
16654
16655        Ok(())
16656    }
16657
16658    #[tracing::instrument(skip(self))]
16659    pub async fn complete_decommission(&self, idx: usize) -> Result<()> {
16660        self.complete_decommission_with_owner(idx, None, None, None).await
16661    }
16662
16663    async fn complete_decommission_for_operation(
16664        &self,
16665        idx: usize,
16666        owner: &DecommissionCanceler,
16667        verified_generation: OffsetDateTime,
16668        verified_unresolved_entries: Vec<DecommissionUnresolvedEntry>,
16669    ) -> Result<()> {
16670        self.complete_decommission_with_owner(idx, Some(owner), Some(verified_generation), Some(verified_unresolved_entries))
16671            .await
16672    }
16673
16674    async fn complete_decommission_with_owner(
16675        &self,
16676        idx: usize,
16677        owner: Option<&DecommissionCanceler>,
16678        verified_generation: Option<OffsetDateTime>,
16679        verified_unresolved_entries: Option<Vec<DecommissionUnresolvedEntry>>,
16680    ) -> Result<()> {
16681        ensure_decommission_terminal_operation_supported(self.single_pool(), "complete decommission")?;
16682        ensure_valid_decommission_pool_index(self.pools.len(), idx)?;
16683        if let Some(owner) = owner {
16684            let cancelers = self.decommission_cancelers.read().await;
16685            if !decommission_canceler_is_owned_by(cancelers.as_slice(), idx, owner) {
16686                owner.release();
16687                return Ok(());
16688            }
16689        }
16690        self.verify_decommission_durable_ilm_receipts(idx).await?;
16691        let _start_guard = self.start_gate.lock().await;
16692        let movement_gate = self.ctx.data_movement_operation_gate();
16693        let _movement_guard = movement_gate.write().await;
16694
16695        // Lock order: movement gate, rebalance_meta, decommission_cancelers,
16696        // then pool_meta. Holding both state locks makes owner validation and
16697        // the terminal transition one atomic operation.
16698        let rebalance_meta = self.rebalance_meta.read().await.clone();
16699        let terminal_at = OffsetDateTime::now_utc();
16700        let (should_reload_pool_meta, completed, previous_pool_meta, terminal_canceler) = {
16701            let cancelers = self.decommission_cancelers.read().await;
16702            let mut pool_meta = self.pool_meta.write().await;
16703            let previous_pool_meta = pool_meta.clone();
16704            let Some(changed) =
16705                update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, idx, owner, |pool_meta| {
16706                    reconcile_decommission_unresolved_entries_for_completion(
16707                        pool_meta,
16708                        idx,
16709                        verified_generation,
16710                        verified_unresolved_entries.as_deref(),
16711                    )?;
16712                    Ok::<bool, Error>(pool_meta.decommission_complete_at(idx, terminal_at, rebalance_meta.as_ref()))
16713                })
16714            else {
16715                return Ok(());
16716            };
16717            let changed = changed?;
16718            let completed = pool_meta
16719                .pools
16720                .get(idx)
16721                .and_then(|pool| pool.decommission.as_ref())
16722                .is_some_and(|decommission| decommission.complete);
16723            let terminal_canceler = if let Some(owner) = owner {
16724                Some(owner.clone())
16725            } else {
16726                cancelers.get(idx).and_then(Option::as_ref).cloned()
16727            };
16728            (changed, completed, changed.then_some(previous_pool_meta), terminal_canceler)
16729        };
16730
16731        if should_reload_pool_meta && let Err(err) = self.save_current_pool_meta(&[idx]).await {
16732            if let Some(previous_pool_meta) = previous_pool_meta {
16733                let mut pool_meta = self.pool_meta.write().await;
16734                rollback_decommission_pool_meta(&mut pool_meta, &previous_pool_meta, &[idx]);
16735            }
16736            return Err(err);
16737        }
16738        if should_reload_pool_meta {
16739            {
16740                let mut pool_meta = self.pool_meta.write().await;
16741                pool_meta.mark_decommission_progress_saved();
16742            }
16743        }
16744        if let Some(canceler) = terminal_canceler.as_ref() {
16745            self.release_decommission_canceler_slot(idx, canceler).await;
16746        }
16747
16748        if should_reload_pool_meta {
16749            self.ctx.advance_data_movement_operation_epoch();
16750        }
16751        drop(_movement_guard);
16752
16753        if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() {
16754            let stage = format!("complete_decommission for pool {idx}");
16755            if let Some(err) = observe_decommission_terminal_reload_result(
16756                resolve_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await, stage.as_str()),
16757                stage.as_str(),
16758            ) {
16759                if let Err(record_err) = self
16760                    .record_decommission_terminal_reload_failure(idx, stage.as_str(), err.clone())
16761                    .await
16762                {
16763                    warn!(
16764                        event = EVENT_DECOMMISSION_STATE,
16765                        component = LOG_COMPONENT_ECSTORE,
16766                        subsystem = LOG_SUBSYSTEM_POOLS,
16767                        pool_index = idx,
16768                        state = "terminal_reload_record_failed",
16769                        error = %record_err,
16770                        original_error = %err,
16771                        "Decommission terminal reload failure record failed"
16772                    );
16773                }
16774                warn!(
16775                    event = EVENT_DECOMMISSION_STATE,
16776                    component = LOG_COMPONENT_ECSTORE,
16777                    subsystem = LOG_SUBSYSTEM_POOLS,
16778                    pool_index = idx,
16779                    state = "terminal_reload_failed",
16780                    error = %err,
16781                    "Decommission terminal state saved but pool meta reload failed"
16782                );
16783            }
16784        }
16785
16786        if completed && let Err(err) = self.cleanup_decommission_durable_ilm_receipts(idx).await {
16787            warn!(
16788                event = EVENT_DECOMMISSION_STATE,
16789                component = LOG_COMPONENT_ECSTORE,
16790                subsystem = LOG_SUBSYSTEM_POOLS,
16791                pool_index = idx,
16792                state = "receipt_cleanup_failed",
16793                error = %err,
16794                "Decommission durable ILM receipt cleanup failed"
16795            );
16796        }
16797
16798        Ok(())
16799    }
16800
16801    async fn decommission_pending_bucket(
16802        self: &Arc<Self>,
16803        rx: CancellationToken,
16804        idx: usize,
16805        generation: OffsetDateTime,
16806        bucket: DecomBucketInfo,
16807        entry_budget: Arc<Semaphore>,
16808        source_changed_exhaustions: Arc<AtomicUsize>,
16809    ) -> Result<()> {
16810        let is_decommissioned = {
16811            let pool_meta = self.pool_meta.read().await;
16812            ensure_decommission_generation(&pool_meta, idx, generation)?;
16813            resolve_decommission_bucket_state(&pool_meta, idx, &bucket)?
16814        };
16815
16816        if is_decommissioned {
16817            warn!("decommission: already done, moving on {}", bucket.to_string());
16818
16819            self.mark_decommission_bucket_done_and_save(idx, generation, &bucket).await?;
16820            return Ok(());
16821        }
16822
16823        warn!("decommission: currently on bucket {}", &bucket.name);
16824
16825        let pool = get_by_index(self.pools.as_slice(), idx, "load decommission background pool")?.clone();
16826        if let Err(err) = self
16827            .decommission_pool(rx.clone(), idx, pool, bucket.clone(), entry_budget, source_changed_exhaustions)
16828            .await
16829        {
16830            error!("decommission: decommission_pool err {:?}", &err);
16831            return Err(err);
16832        } else {
16833            warn!("decommission: decommission_pool done {}", &bucket.name);
16834        }
16835
16836        if let Err(err) = decommission_cancel_signal_result(rx.is_cancelled()) {
16837            warn!("decommission: cancellation observed after decommission_pool {}", &bucket.name);
16838            return Err(err);
16839        }
16840
16841        self.mark_decommission_bucket_done_and_save(idx, generation, &bucket).await?;
16842
16843        warn!("decommission: decommission_pool bucket_done {}", &bucket.name);
16844
16845        Ok(())
16846    }
16847
16848    #[tracing::instrument(skip(self, rx))]
16849    async fn decommission_in_background(
16850        self: &Arc<Self>,
16851        rx: CancellationToken,
16852        idx: usize,
16853        generation: OffsetDateTime,
16854        entry_budget: Arc<Semaphore>,
16855    ) -> Result<()> {
16856        self.ensure_decommission_runtime_capacity_available(idx, generation).await?;
16857        let pending = {
16858            let pool_meta = self.pool_meta.read().await;
16859            pool_meta.pending_buckets(idx)
16860        };
16861        let source_changed_exhaustions = Arc::new(AtomicUsize::new(0));
16862        let bucket_concurrency = decommission_bucket_concurrency_limit();
16863        let (regular_buckets, meta_buckets) = split_decommission_buckets(pending);
16864        let store = Arc::clone(self);
16865        run_decommission_phases(rx.clone(), regular_buckets, meta_buckets, bucket_concurrency, move |bucket, rx| {
16866            let store = Arc::clone(&store);
16867            let entry_budget = entry_budget.clone();
16868            let source_changed_exhaustions = Arc::clone(&source_changed_exhaustions);
16869            Box::pin(async move {
16870                store
16871                    .decommission_pending_bucket(rx, idx, generation, bucket, entry_budget, source_changed_exhaustions)
16872                    .await
16873            })
16874        })
16875        .await
16876    }
16877
16878    #[tracing::instrument(skip(self))]
16879    pub async fn start_decommission(&self, indices: Vec<usize>) -> Result<()> {
16880        self.start_decommission_inner(indices, None).await.map(|_| ())
16881    }
16882
16883    async fn start_decommission_with_routines(
16884        &self,
16885        indices: Vec<usize>,
16886        rx: &CancellationToken,
16887        local_indices: &[usize],
16888    ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
16889        self.start_decommission_inner(indices, Some((rx, local_indices))).await
16890    }
16891
16892    async fn start_decommission_inner(
16893        &self,
16894        indices: Vec<usize>,
16895        reservation: Option<(&CancellationToken, &[usize])>,
16896    ) -> Result<Vec<(usize, DecommissionCancelerGuard)>> {
16897        let indices = dedup_indices(&indices);
16898        validate_start_decommission_request(&indices, self.single_pool())?;
16899
16900        self.ensure_decommission_rebalance_idle_after_refresh().await?;
16901        #[cfg(test)]
16902        let endpoints = self.instance_endpoints().unwrap_or_else(|| self.endpoints());
16903        #[cfg(not(test))]
16904        let endpoints = self.endpoints();
16905        ensure_decommission_start_local_leader(&endpoints, &indices)?;
16906
16907        for idx in indices.iter().copied() {
16908            ensure_valid_decommission_pool_index(self.pools.len(), idx)?;
16909        }
16910
16911        {
16912            let pool_meta = self.pool_meta.read().await;
16913            ensure_decommission_start_pool_states(&pool_meta, &indices)?;
16914            ensure_decommission_ledger_persistence_supported(&pool_meta)?;
16915            ensure_decommission_capacity_writer_supported(&pool_meta)?;
16916        }
16917
16918        #[cfg(test)]
16919        observe_pool_activation_preflight_side_effect_attempt(PoolActivationStartKind::Decommission);
16920        let decom_buckets = self.get_buckets_to_decommission().await?;
16921
16922        let mut healed_buckets = HashSet::with_capacity(decom_buckets.len());
16923        for bk in decom_buckets.iter() {
16924            if healed_buckets.insert(bk.name.as_str()) {
16925                resolve_decommission_preflight_heal_result(&bk.name, self.heal_bucket(&bk.name, &HealOpts::default()).await)?;
16926            }
16927        }
16928
16929        let meta_bucket_opts = decommission_meta_bucket_options();
16930        for prefix in DECOMMISSION_META_PREFIXES {
16931            let bk = path_join(&[PathBuf::from(RUSTFS_META_BUCKET), PathBuf::from(prefix)]);
16932            if let Err(err) = self
16933                .make_bucket(bk.to_string_lossy().to_string().as_str(), &meta_bucket_opts)
16934                .await
16935                && !is_err_bucket_exists(&err)
16936            {
16937                error!("decommission: make bucket failed: {err}");
16938                return Err(err);
16939            }
16940        }
16941
16942        let _start_guard = self.start_gate.lock().await;
16943        self.ensure_decommission_rebalance_idle_after_refresh_under_start_gate()
16944            .await?;
16945
16946        let all_capacity_infos = self.get_decommission_all_pool_capacity_infos_with_owned_usage().await?;
16947        let target_fence_proof_available =
16948            crate::services::notification_sys::acquire_decommission_target_fence_fleet_proof().is_some();
16949        // Signal cancellation before waiting for the movement writer so active
16950        // object operations can observe the signal and release read guards.
16951        self.cancel_decommission_routines(&indices).await;
16952        let movement_gate = self.ctx.data_movement_operation_gate();
16953        let _movement_guard = movement_gate.write().await;
16954
16955        let index_cancelers = if let Some((rx, local_indices)) = reservation {
16956            // Lock order matches terminal transitions: movement gate, then
16957            // decommission_cancelers, then pool_meta while start_gate excludes
16958            // another start.
16959            let mut cancelers = self.decommission_cancelers.write().await;
16960            let pool_meta = self.pool_meta.read().await;
16961            ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_capacity_infos, target_fence_proof_available)?;
16962            reserve_decommission_start_cancelers(&pool_meta, &indices, local_indices, rx, cancelers.as_mut_slice())?
16963        } else {
16964            let pool_meta = self.pool_meta.read().await;
16965            ensure_decommission_start_pool_states(&pool_meta, &indices)?;
16966            ensure_decommission_start_target_capacity(&pool_meta, &indices, &all_capacity_infos, target_fence_proof_available)?;
16967            Vec::new()
16968        };
16969
16970        let previous_pool_meta = self
16971            .save_current_pool_meta_for_decommission_start(&indices, decom_buckets)
16972            .await?;
16973        self.ctx.advance_data_movement_operation_epoch();
16974        // The local durable transition is now fenced. Release the writer
16975        // before any peer RPC; remote reload must not block scanner admission.
16976        drop(_movement_guard);
16977
16978        if let Some(notification_sys) = runtime_sources::notification_sys()
16979            && let Err(err) = resolve_start_decommission_pool_meta_reload_result(notification_sys.reload_pool_meta().await)
16980        {
16981            warn!(
16982                event = EVENT_DECOMMISSION_STATE,
16983                component = LOG_COMPONENT_ECSTORE,
16984                subsystem = LOG_SUBSYSTEM_POOLS,
16985                state = "start_failed",
16986                stage = "reload_pool_meta",
16987                error = %err,
16988                "Decommission start failed after pool metadata save"
16989            );
16990
16991            let rollback_result = self
16992                .rollback_decommission_start_after_reload_failure(&movement_gate, &previous_pool_meta, &indices)
16993                .await;
16994            if let Err(rollback_save_err) = rollback_result {
16995                error!(
16996                    event = EVENT_DECOMMISSION_STATE,
16997                    component = LOG_COMPONENT_ECSTORE,
16998                    subsystem = LOG_SUBSYSTEM_POOLS,
16999                    state = "rollback_failed",
17000                    stage = "save_pool_meta",
17001                    error = %rollback_save_err,
17002                    original_error = %err,
17003                    "Decommission rollback failed after pool metadata reload failure"
17004                );
17005                return Err(Error::other(format!(
17006                    "{err}; decommission start rollback save failed: {rollback_save_err}"
17007                )));
17008            }
17009
17010            if let Err(rollback_reload_err) = resolve_decommission_pool_meta_reload_result(
17011                notification_sys.reload_pool_meta().await,
17012                "start_decommission_rollback",
17013            ) {
17014                error!(
17015                    event = EVENT_DECOMMISSION_STATE,
17016                    component = LOG_COMPONENT_ECSTORE,
17017                    subsystem = LOG_SUBSYSTEM_POOLS,
17018                    state = "rollback_partial",
17019                    stage = "reload_pool_meta",
17020                    error = %rollback_reload_err,
17021                    original_error = %err,
17022                    "Decommission rollback metadata reload failed after local rollback save"
17023                );
17024                return Err(Error::other(format!(
17025                    "{err}; decommission start rollback saved locally but peer reload failed: {rollback_reload_err}"
17026                )));
17027            }
17028
17029            warn!(
17030                event = EVENT_DECOMMISSION_STATE,
17031                component = LOG_COMPONENT_ECSTORE,
17032                subsystem = LOG_SUBSYSTEM_POOLS,
17033                state = "rollback_success",
17034                original_error = %err,
17035                "Decommission start rolled back after pool metadata reload failure"
17036            );
17037            return Err(Error::other(format!("{err}; decommission start rollback succeeded")));
17038        }
17039
17040        Ok(index_cancelers)
17041    }
17042
17043    async fn get_buckets_to_decommission(&self) -> Result<Vec<DecomBucketInfo>> {
17044        let buckets = self.list_bucket(&BucketOptions::default()).await?;
17045
17046        let mut ret: Vec<DecomBucketInfo> = buckets
17047            .iter()
17048            .map(|v| DecomBucketInfo {
17049                name: v.name.clone(),
17050                ..Default::default()
17051            })
17052            .collect();
17053
17054        ret.extend(decommission_meta_buckets());
17055
17056        Ok(ret)
17057    }
17058
17059    async fn durable_ilm_receipt_run_token(&self, source_pool_idx: usize) -> Result<String> {
17060        let pool_meta = self.pool_meta.read().await;
17061        let pool = pool_meta
17062            .pools
17063            .get(source_pool_idx)
17064            .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), source_pool_idx))?;
17065        let start_time = pool
17066            .decommission
17067            .as_ref()
17068            .and_then(|info| info.start_time)
17069            .ok_or_else(|| Error::other(format!("decommission run identity is missing for pool {source_pool_idx}")))?;
17070        Ok(decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time))
17071    }
17072
17073    async fn durable_ilm_receipt_run_token_for_generation(
17074        &self,
17075        source_pool_idx: usize,
17076        generation: OffsetDateTime,
17077    ) -> Result<String> {
17078        let pool_meta = self.pool_meta.read().await;
17079        ensure_decommission_generation(&pool_meta, source_pool_idx, generation)?;
17080        let pool = pool_meta
17081            .pools
17082            .get(source_pool_idx)
17083            .ok_or_else(|| invalid_decommission_pool_index_error(pool_meta.pools.len(), source_pool_idx))?;
17084        Ok(decommission_durable_ilm_receipt_run_token(&pool.cmd_line, generation))
17085    }
17086
17087    async fn load_decommissioned_durable_ilm_target(
17088        &self,
17089        source_pool_idx: usize,
17090        path: &str,
17091        max_record_size: usize,
17092        record_context: &str,
17093    ) -> Result<Option<(usize, Vec<u8>)>> {
17094        let mut target = None::<(usize, Vec<u8>)>;
17095        let mut first_read_error = None;
17096        for (target_pool_idx, pool) in self.pools.iter().enumerate() {
17097            if target_pool_idx == source_pool_idx {
17098                continue;
17099            }
17100            match read_config_limited_preserve_empty(pool.clone(), path, max_record_size).await {
17101                Ok(data) => {
17102                    if let Some((existing_pool_idx, existing)) = target.as_ref()
17103                        && existing != &data
17104                    {
17105                        return Err(Error::other(format!(
17106                            "divergent target durable ILM records at path `{path}` {record_context} in pools {existing_pool_idx} and {target_pool_idx}"
17107                        )));
17108                    }
17109                    target = Some((target_pool_idx, data));
17110                }
17111                Err(err)
17112                    if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17113                        || is_err_object_not_found(&err)
17114                        || is_err_version_not_found(&err) => {}
17115                Err(err) => {
17116                    first_read_error.get_or_insert_with(|| {
17117                        Error::other(format!(
17118                            "failed to read target durable ILM record at path `{path}` {record_context} from pool {target_pool_idx}: {err}"
17119                        ))
17120                    });
17121                }
17122            }
17123        }
17124
17125        if let Some(err) = first_read_error {
17126            return Err(err);
17127        }
17128        Ok(target)
17129    }
17130
17131    async fn list_decommission_durable_ilm_receipt_paths_in_pool(&self, pool_idx: usize, prefix: &str) -> Result<Vec<String>> {
17132        let pool = self
17133            .pools
17134            .get(pool_idx)
17135            .ok_or_else(|| invalid_decommission_pool_index_error(self.pools.len(), pool_idx))?;
17136        let mut receipts = Vec::new();
17137        let mut continuation = None;
17138        loop {
17139            let page = pool
17140                .clone()
17141                .list_objects_v2(RUSTFS_META_BUCKET, prefix, continuation, None, 1000, false, None, false)
17142                .await
17143                .map_err(|err| {
17144                    Error::other(format!(
17145                        "failed to list durable ILM decommission receipts under `{prefix}` in pool {pool_idx}: {err}"
17146                    ))
17147                })?;
17148            receipts.extend(page.objects.into_iter().map(|object| object.name));
17149            if !page.is_truncated {
17150                break;
17151            }
17152            continuation = Some(page.next_continuation_token.ok_or_else(|| {
17153                Error::other(format!(
17154                    "durable ILM decommission receipt listing under `{prefix}` in pool {pool_idx} was truncated without a continuation token"
17155                ))
17156            })?);
17157        }
17158        Ok(receipts)
17159    }
17160
17161    async fn list_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<Vec<(usize, String)>> {
17162        let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
17163        self.list_decommission_durable_ilm_receipts_for_run(source_pool_idx, &run_token)
17164            .await
17165    }
17166
17167    async fn list_decommission_durable_ilm_receipts_for_run(
17168        &self,
17169        source_pool_idx: usize,
17170        run_token: &str,
17171    ) -> Result<Vec<(usize, String)>> {
17172        let prefix = decommission_durable_ilm_receipt_run_prefix(run_token);
17173        let mut receipts = Vec::new();
17174        for pool_idx in 0..self.pools.len() {
17175            if pool_idx == source_pool_idx {
17176                continue;
17177            }
17178            for receipt_path in self
17179                .list_decommission_durable_ilm_receipt_paths_in_pool(pool_idx, &prefix)
17180                .await?
17181            {
17182                let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
17183                if locator.run_token != run_token {
17184                    return Err(Error::other(format!(
17185                        "durable ILM receipt path `{receipt_path}` has an unexpected run token"
17186                    )));
17187                }
17188                receipts.push((pool_idx, receipt_path));
17189            }
17190        }
17191        Ok(receipts)
17192    }
17193
17194    async fn list_decommission_durable_ilm_manifest_receipts(&self, source_pool_idx: usize) -> Result<Vec<String>> {
17195        let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
17196        self.list_decommission_durable_ilm_manifest_receipts_for_run(source_pool_idx, &run_token)
17197            .await
17198    }
17199
17200    async fn list_decommission_durable_ilm_manifest_receipts_for_run(
17201        &self,
17202        source_pool_idx: usize,
17203        run_token: &str,
17204    ) -> Result<Vec<String>> {
17205        let prefix = decommission_durable_ilm_receipt_run_prefix(run_token);
17206        let receipt_paths = self
17207            .list_decommission_durable_ilm_receipt_paths_in_pool(source_pool_idx, &prefix)
17208            .await?;
17209        for receipt_path in &receipt_paths {
17210            let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?;
17211            if locator.run_token != run_token {
17212                return Err(Error::other(format!(
17213                    "durable ILM expected manifest receipt path `{receipt_path}` has an unexpected run token"
17214                )));
17215            }
17216        }
17217        Ok(receipt_paths)
17218    }
17219
17220    #[cfg(all(test, feature = "test-util"))]
17221    async fn persist_decommission_durable_ilm_manifest(&self, source_pool_idx: usize) -> Result<()> {
17222        let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
17223        self.persist_decommission_durable_ilm_manifest_for_run(source_pool_idx, &run_token)
17224            .await
17225    }
17226
17227    async fn persist_decommission_durable_ilm_manifest_for_run(&self, source_pool_idx: usize, run_token: &str) -> Result<()> {
17228        let receipt_paths = self
17229            .list_decommission_durable_ilm_manifest_receipts_for_run(source_pool_idx, run_token)
17230            .await?;
17231        for receipt_path in &receipt_paths {
17232            self.read_decommission_durable_ilm_receipt(source_pool_idx, receipt_path)
17233                .await?;
17234        }
17235        let manifest = DecommissionDurableIlmManifest::new(run_token, &receipt_paths)?;
17236        let manifest_path = decommission_durable_ilm_manifest_path(run_token);
17237        let encoded = manifest.encode()?;
17238        let mut attempt = 1;
17239        loop {
17240            match read_config_limited_preserve_empty(
17241                self.pools[source_pool_idx].clone(),
17242                &manifest_path,
17243                DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE,
17244            )
17245            .await
17246            {
17247                Ok(existing) => {
17248                    DecommissionDurableIlmManifest::decode(&existing, run_token, &receipt_paths).map_err(|err| {
17249                        Error::other(format!(
17250                            "durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx} is invalid: {err}"
17251                        ))
17252                    })?;
17253                    return Ok(());
17254                }
17255                Err(err)
17256                    if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17257                        || is_err_object_not_found(&err)
17258                        || is_err_version_not_found(&err) => {}
17259                Err(err) => {
17260                    return Err(Error::other(format!(
17261                        "failed to read durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}"
17262                    )));
17263                }
17264            }
17265            match save_config_with_opts(
17266                self.pools[source_pool_idx].clone(),
17267                &manifest_path,
17268                encoded.clone(),
17269                &ObjectOptions {
17270                    max_parity: true,
17271                    write_completion: crate::object_api::WriteCompletion::TailDrained,
17272                    http_preconditions: Some(HTTPPreconditions {
17273                        if_none_match: Some("*".to_string()),
17274                        ..Default::default()
17275                    }),
17276                    ..Default::default()
17277                },
17278            )
17279            .await
17280            {
17281                Ok(()) => return Ok(()),
17282                Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => {
17283                    attempt += 1;
17284                }
17285                Err(Error::PreconditionFailed) => {
17286                    return Err(Error::other(format!(
17287                        "failed to persist durable ILM expected manifest `{manifest_path}` after concurrent updates"
17288                    )));
17289                }
17290                Err(err) => {
17291                    return Err(Error::other(format!(
17292                        "failed to persist durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx}: {err}"
17293                    )));
17294                }
17295            }
17296        }
17297    }
17298
17299    async fn load_decommission_durable_ilm_manifest_for_run(
17300        &self,
17301        source_pool_idx: usize,
17302        run_token: &str,
17303    ) -> Result<HashMap<String, DecommissionDurableIlmReceipt>> {
17304        let receipt_paths = self
17305            .list_decommission_durable_ilm_manifest_receipts_for_run(source_pool_idx, run_token)
17306            .await?;
17307        let manifest_path = decommission_durable_ilm_manifest_path(run_token);
17308        let data = read_config_limited_preserve_empty(
17309            self.pools[source_pool_idx].clone(),
17310            &manifest_path,
17311            DECOMMISSION_DURABLE_ILM_MANIFEST_MAX_SIZE,
17312        )
17313        .await
17314        .map_err(|err| {
17315            Error::other(format!(
17316                "failed to read durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}"
17317            ))
17318        })?;
17319        DecommissionDurableIlmManifest::decode(&data, run_token, &receipt_paths).map_err(|err| {
17320            Error::other(format!(
17321                "durable ILM expected manifest `{manifest_path}` in source pool {source_pool_idx} is invalid: {err}"
17322            ))
17323        })?;
17324
17325        let mut receipts = HashMap::with_capacity(receipt_paths.len());
17326        for receipt_path in receipt_paths {
17327            let receipt = self
17328                .read_decommission_durable_ilm_receipt(source_pool_idx, &receipt_path)
17329                .await?;
17330            if receipts.insert(receipt_path.clone(), receipt).is_some() {
17331                return Err(Error::other(format!(
17332                    "durable ILM expected manifest contains duplicate receipt path `{receipt_path}`"
17333                )));
17334            }
17335        }
17336        Ok(receipts)
17337    }
17338
17339    #[cfg(all(test, feature = "test-util"))]
17340    async fn persist_decommission_durable_ilm_receipt(
17341        &self,
17342        source_pool_idx: usize,
17343        target_pool_idx: usize,
17344        receipt: &DecommissionDurableIlmReceipt,
17345    ) -> Result<()> {
17346        let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
17347        self.persist_decommission_durable_ilm_receipt_for_run(target_pool_idx, receipt, &run_token)
17348            .await
17349    }
17350
17351    async fn persist_decommission_durable_ilm_receipt_for_run(
17352        &self,
17353        target_pool_idx: usize,
17354        receipt: &DecommissionDurableIlmReceipt,
17355        run_token: &str,
17356    ) -> Result<()> {
17357        let receipt_path = decommission_durable_ilm_receipt_path(run_token, &receipt.source_path, &receipt.id_kind, &receipt.id);
17358        let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
17359        let mut attempt = 1;
17360        loop {
17361            let (merged, http_preconditions) = match read_config_limited_preserve_empty_with_metadata(
17362                self.pools[target_pool_idx].clone(),
17363                &receipt_path,
17364                DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
17365            )
17366            .await
17367            {
17368                Ok((existing_data, metadata)) => {
17369                    let existing = DecommissionDurableIlmReceipt::decode(&existing_data).map_err(|err| {
17370                        Error::other(format!(
17371                            "durable ILM decommission receipt `{receipt_path}` in pool {target_pool_idx} for {} is invalid: {err}",
17372                            locator.context()
17373                        ))
17374                    })?;
17375                    Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &existing)?;
17376                    let merged = merge_decommission_durable_ilm_receipts(&existing, receipt)?;
17377                    if merged == existing {
17378                        return Ok(());
17379                    }
17380                    let etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| {
17381                        Error::other(format!(
17382                            "durable ILM decommission receipt `{receipt_path}` in pool {target_pool_idx} is missing an ETag"
17383                        ))
17384                    })?;
17385                    (
17386                        merged,
17387                        HTTPPreconditions {
17388                            if_match: Some(etag),
17389                            ..Default::default()
17390                        },
17391                    )
17392                }
17393                Err(err)
17394                    if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17395                        || is_err_object_not_found(&err)
17396                        || is_err_version_not_found(&err) =>
17397                {
17398                    (
17399                        receipt.clone(),
17400                        HTTPPreconditions {
17401                            if_none_match: Some("*".to_string()),
17402                            ..Default::default()
17403                        },
17404                    )
17405                }
17406                Err(err) => {
17407                    return Err(Error::other(format!(
17408                        "failed to read durable ILM decommission receipt `{receipt_path}` from pool {target_pool_idx} for {}: {err}",
17409                        locator.context()
17410                    )));
17411                }
17412            };
17413            let encoded = merged.encode().map_err(|err| {
17414                Error::other(format!(
17415                    "failed to encode durable ILM decommission receipt `{receipt_path}` for source path `{}` {}: {err}",
17416                    receipt.source_path,
17417                    receipt.context()
17418                ))
17419            })?;
17420            match save_config_with_opts(
17421                self.pools[target_pool_idx].clone(),
17422                &receipt_path,
17423                encoded,
17424                &ObjectOptions {
17425                    max_parity: true,
17426                    write_completion: crate::object_api::WriteCompletion::TailDrained,
17427                    http_preconditions: Some(http_preconditions),
17428                    ..Default::default()
17429                },
17430            )
17431            .await
17432            {
17433                Ok(()) => return Ok(()),
17434                Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => {
17435                    attempt += 1;
17436                }
17437                Err(Error::PreconditionFailed) => {
17438                    return Err(Error::other(format!(
17439                        "failed to persist durable ILM decommission receipt `{receipt_path}` for {} after concurrent updates",
17440                        locator.context()
17441                    )));
17442                }
17443                Err(err) => {
17444                    return Err(Error::other(format!(
17445                        "failed to persist durable ILM decommission receipt `{receipt_path}` for {}: {err}",
17446                        locator.context()
17447                    )));
17448                }
17449            }
17450        }
17451    }
17452
17453    fn validate_decommission_durable_ilm_receipt_locator(
17454        receipt_path: &str,
17455        locator: &DecommissionDurableIlmReceiptLocator,
17456        receipt: &DecommissionDurableIlmReceipt,
17457    ) -> Result<()> {
17458        if locator.source_path != receipt.source_path || locator.id_kind != receipt.id_kind || locator.id != receipt.id {
17459            return Err(Error::other(format!(
17460                "durable ILM decommission receipt path `{receipt_path}` identity {} does not match receipt {}",
17461                locator.context(),
17462                receipt.context()
17463            )));
17464        }
17465        Ok(())
17466    }
17467
17468    async fn read_decommission_durable_ilm_receipt(
17469        &self,
17470        receipt_pool_idx: usize,
17471        receipt_path: &str,
17472    ) -> Result<DecommissionDurableIlmReceipt> {
17473        let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?;
17474        let data = read_config_limited_preserve_empty(
17475            self.pools[receipt_pool_idx].clone(),
17476            receipt_path,
17477            DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
17478        )
17479        .await
17480        .map_err(|err| {
17481            Error::other(format!(
17482                "failed to read durable ILM decommission receipt `{receipt_path}` from pool {receipt_pool_idx} for {}: {err}",
17483                locator.context()
17484            ))
17485        })?;
17486        let receipt = DecommissionDurableIlmReceipt::decode(&data).map_err(|err| {
17487            Error::other(format!(
17488                "durable ILM decommission receipt `{receipt_path}` in pool {receipt_pool_idx} for {} is invalid: {err}",
17489                locator.context()
17490            ))
17491        })?;
17492        Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?;
17493        Ok(receipt)
17494    }
17495
17496    async fn load_decommission_durable_ilm_terminal_receipt_for_run(
17497        &self,
17498        source_pool_idx: usize,
17499        path: &str,
17500        source_record: &ValidatedDurableIlmRecord,
17501        run_token: &str,
17502    ) -> Result<Option<DecommissionDurableIlmReceipt>> {
17503        let receipt_path = decommission_durable_ilm_receipt_path(run_token, path, source_record.id_kind, &source_record.id);
17504        let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
17505        let mut proof = None::<DecommissionDurableIlmReceipt>;
17506        let mut nonterminal_receipt_found = false;
17507        for pool_idx in 0..self.pools.len() {
17508            if pool_idx == source_pool_idx {
17509                continue;
17510            }
17511            let data = match read_config_limited_preserve_empty(
17512                self.pools[pool_idx].clone(),
17513                &receipt_path,
17514                DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
17515            )
17516            .await
17517            {
17518                Ok(data) => data,
17519                Err(err)
17520                    if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17521                        || is_err_object_not_found(&err)
17522                        || is_err_version_not_found(&err) =>
17523                {
17524                    continue;
17525                }
17526                Err(err) => {
17527                    return Err(Error::other(format!(
17528                        "failed to read terminal durable ILM decommission receipt `{receipt_path}` from pool {pool_idx} for {}: {err}",
17529                        source_record.context()
17530                    )));
17531                }
17532            };
17533            let receipt = DecommissionDurableIlmReceipt::decode(&data).map_err(|err| {
17534                Error::other(format!(
17535                    "terminal durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} for {} is invalid: {err}",
17536                    source_record.context()
17537                ))
17538            })?;
17539            Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &receipt)?;
17540            if receipt.namespace != source_record.namespace
17541                || receipt.id_kind != source_record.id_kind
17542                || receipt.id != source_record.id
17543            {
17544                return Err(Error::other(format!(
17545                    "terminal durable ILM decommission receipt identity mismatch at path `{path}` {}; receipt {}",
17546                    source_record.context(),
17547                    receipt.context()
17548                )));
17549            }
17550            if let Some(terminal_checkpoint) = receipt.terminal_checkpoint.as_ref() {
17551                if !source_record.checkpoint.is_predecessor_of_terminal(terminal_checkpoint) {
17552                    return Err(Error::other_with_context(
17553                        "terminal durable ILM decommission receipt does not cover source",
17554                        format!("path `{path}` {}", source_record.context()),
17555                    ));
17556                }
17557                proof = Some(match proof {
17558                    Some(existing) => merge_decommission_durable_ilm_receipts(&existing, &receipt)?,
17559                    None => receipt,
17560                });
17561            } else {
17562                source_record
17563                    .checkpoint
17564                    .validate_successor(&receipt.checkpoint)
17565                    .map_err(|err| {
17566                        Error::other_with_context(
17567                            "durable ILM decommission receipt does not cover source",
17568                            format!("path `{path}` {}: {err}", source_record.context()),
17569                        )
17570                    })?;
17571                nonterminal_receipt_found = true;
17572            }
17573        }
17574        // A terminal receipt on one target must not hide another target copy
17575        // whose receipt was installed later and has not reached terminal yet.
17576        // Returning no proof makes recovery advance every outstanding copy
17577        // before source cleanup can treat the operation as complete.
17578        if nonterminal_receipt_found { Ok(None) } else { Ok(proof) }
17579    }
17580
17581    async fn verify_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> {
17582        let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
17583        self.verify_decommission_durable_ilm_receipts_for_run(source_pool_idx, &run_token)
17584            .await
17585    }
17586
17587    async fn verify_decommission_durable_ilm_receipts_for_run(&self, source_pool_idx: usize, run_token: &str) -> Result<()> {
17588        let expected_receipts = self
17589            .load_decommission_durable_ilm_manifest_for_run(source_pool_idx, run_token)
17590            .await?;
17591        let receipt_paths = self
17592            .list_decommission_durable_ilm_receipts_for_run(source_pool_idx, run_token)
17593            .await?;
17594        let present_receipt_paths = receipt_paths
17595            .iter()
17596            .map(|(_, receipt_path)| receipt_path.as_str())
17597            .collect::<HashSet<_>>();
17598        for (expected_path, expected) in &expected_receipts {
17599            if !present_receipt_paths.contains(expected_path.as_str()) {
17600                return Err(Error::other(format!(
17601                    "durable ILM decommission receipt is missing at `{expected_path}` for source path `{}` {}",
17602                    expected.source_path,
17603                    expected.context()
17604                )));
17605            }
17606        }
17607
17608        for (receipt_pool_idx, receipt_path) in receipt_paths {
17609            let expected = expected_receipts.get(&receipt_path).ok_or_else(|| {
17610                Error::other(format!(
17611                    "durable ILM decommission receipt `{receipt_path}` in pool {receipt_pool_idx} is absent from the expected manifest"
17612                ))
17613            })?;
17614            let receipt = self
17615                .read_decommission_durable_ilm_receipt(receipt_pool_idx, &receipt_path)
17616                .await?;
17617            if receipt.source_path != expected.source_path
17618                || receipt.namespace != expected.namespace
17619                || receipt.id_kind != expected.id_kind
17620                || receipt.id != expected.id
17621            {
17622                return Err(Error::other(format!(
17623                    "durable ILM decommission receipt identity mismatch at `{receipt_path}` for source path `{}` {}; decoded {}",
17624                    expected.source_path,
17625                    expected.context(),
17626                    receipt.context()
17627                )));
17628            }
17629            expected.checkpoint.validate_successor(&receipt.checkpoint).map_err(|err| {
17630                Error::other(format!(
17631                    "durable ILM decommission receipt generation mismatch at `{receipt_path}` for source path `{}` {}: {err}",
17632                    expected.source_path,
17633                    expected.context()
17634                ))
17635            })?;
17636            match (&expected.terminal_checkpoint, &receipt.terminal_checkpoint) {
17637                (Some(expected_terminal), Some(receipt_terminal)) => {
17638                    expected_terminal.validate_successor(receipt_terminal).map_err(|err| {
17639                        Error::other(format!(
17640                            "durable ILM decommission terminal receipt generation mismatch at `{receipt_path}` for source path `{}` {}: {err}",
17641                            expected.source_path,
17642                            expected.context()
17643                        ))
17644                    })?;
17645                }
17646                (Some(_), None) => {
17647                    return Err(Error::other(format!(
17648                        "durable ILM decommission terminal receipt is missing at `{receipt_path}` for source path `{}` {}",
17649                        expected.source_path,
17650                        expected.context()
17651                    )));
17652                }
17653                (None, _) => {}
17654            }
17655            let namespace = classify_durable_ilm_record(&receipt.source_path)?
17656                .ok_or_else(|| Error::other(format!("path `{}` is not a durable ILM record", receipt.source_path)))?;
17657            let target = self
17658                .load_decommissioned_durable_ilm_target(
17659                    source_pool_idx,
17660                    &receipt.source_path,
17661                    namespace.max_record_size,
17662                    &receipt.context(),
17663                )
17664                .await?;
17665            if let Some((_, target)) = target {
17666                let target_record = validate_durable_ilm_record(&receipt.source_path, &target).map_err(|err| {
17667                    Error::other(format!(
17668                        "target durable ILM record is invalid at path `{}` {}: {err}",
17669                        receipt.source_path,
17670                        receipt.context()
17671                    ))
17672                })?;
17673                let identity_matches = target_record.namespace == receipt.namespace
17674                    && target_record.id_kind == receipt.id_kind
17675                    && target_record.id == receipt.id;
17676                let reused_manual_scope = receipt.terminal_checkpoint.is_some()
17677                    && matches!(
17678                        (&receipt.checkpoint, &target_record.checkpoint),
17679                        (
17680                            DurableIlmRecordCheckpoint::ManualTransitionScope { .. },
17681                            DurableIlmRecordCheckpoint::ManualTransitionScope { .. }
17682                        )
17683                    );
17684                if !identity_matches && !reused_manual_scope {
17685                    return Err(Error::other(format!(
17686                        "target durable ILM record identity mismatch at path `{}` {}; decoded {}",
17687                        receipt.source_path,
17688                        receipt.context(),
17689                        target_record.context()
17690                    )));
17691                }
17692                if identity_matches {
17693                    receipt
17694                        .terminal_checkpoint
17695                        .as_ref()
17696                        .unwrap_or(&receipt.checkpoint)
17697                        .validate_successor(&target_record.checkpoint)
17698                        .map_err(|err| {
17699                            Error::other(format!(
17700                                "target durable ILM record generation mismatch at path `{}` {}: {err}",
17701                                receipt.source_path,
17702                                receipt.context()
17703                            ))
17704                        })?;
17705                }
17706            } else if receipt.terminal_checkpoint.is_none() {
17707                return Err(Error::other(format!(
17708                    "target durable ILM record is missing at path `{}` {} without a recovery terminal checkpoint",
17709                    receipt.source_path,
17710                    receipt.context()
17711                )));
17712            }
17713        }
17714        Ok(())
17715    }
17716
17717    async fn advance_durable_ilm_decommission_receipt(
17718        &self,
17719        pool_idx: usize,
17720        receipt_path: &str,
17721        record: &ValidatedDurableIlmRecord,
17722        fleet_topology_generation: Option<&str>,
17723        terminal: bool,
17724    ) -> Result<bool> {
17725        let stage = if terminal { "terminal" } else { "progress" };
17726        let locator = parse_decommission_durable_ilm_receipt_path(receipt_path)?;
17727        let mut attempt = 1;
17728        loop {
17729            let (receipt_data, metadata) = match read_config_limited_preserve_empty_with_metadata(
17730                self.pools[pool_idx].clone(),
17731                receipt_path,
17732                DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
17733            )
17734            .await
17735            {
17736                Ok(receipt) => receipt,
17737                Err(err)
17738                    if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17739                        || is_err_object_not_found(&err)
17740                        || is_err_version_not_found(&err) =>
17741                {
17742                    return Ok(false);
17743                }
17744                Err(err) => {
17745                    return Err(Error::other(format!(
17746                        "failed to read durable ILM decommission receipt `{receipt_path}` from pool {pool_idx} for {}: {err}",
17747                        locator.context()
17748                    )));
17749                }
17750            };
17751            let mut receipt = DecommissionDurableIlmReceipt::decode(&receipt_data).map_err(|err| {
17752                Error::other(format!(
17753                    "durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} for {} is invalid: {err}",
17754                    locator.context()
17755                ))
17756            })?;
17757            Self::validate_decommission_durable_ilm_receipt_locator(receipt_path, &locator, &receipt)?;
17758            if receipt.fleet_topology_generation.as_deref() != fleet_topology_generation {
17759                return Err(Error::other_with_context(
17760                    "durable ILM decommission receipt fleet topology mismatch",
17761                    format!("path `{}` {}", receipt.source_path, receipt.context()),
17762                ));
17763            }
17764            receipt.checkpoint.validate_successor(&record.checkpoint).map_err(|err| {
17765                Error::other(format!(
17766                    "{stage} durable ILM record generation mismatch at path `{}` {}: {err}",
17767                    receipt.source_path,
17768                    receipt.context()
17769                ))
17770            })?;
17771
17772            if terminal {
17773                if let Some(existing) = &receipt.terminal_checkpoint {
17774                    if existing == &record.checkpoint || record.checkpoint.validate_successor(existing).is_ok() {
17775                        return Ok(true);
17776                    }
17777                    existing.validate_successor(&record.checkpoint).map_err(|err| {
17778                        Error::other(format!(
17779                            "terminal durable ILM record checkpoint conflicts at path `{}` {}: {err}",
17780                            receipt.source_path,
17781                            receipt.context()
17782                        ))
17783                    })?;
17784                }
17785                receipt.terminal_checkpoint = Some(record.checkpoint.clone());
17786            } else {
17787                if let Some(existing) = &receipt.terminal_checkpoint {
17788                    if existing == &record.checkpoint {
17789                        return Ok(true);
17790                    }
17791                    existing.validate_successor(&record.checkpoint).map_err(|err| {
17792                        Error::other(format!(
17793                            "progress durable ILM record conflicts with terminal checkpoint at path `{}` {}: {err}",
17794                            receipt.source_path,
17795                            receipt.context()
17796                        ))
17797                    })?;
17798                    receipt.terminal_checkpoint = None;
17799                }
17800                if receipt.checkpoint == record.checkpoint {
17801                    return Ok(true);
17802                }
17803                receipt.checkpoint = record.checkpoint.clone();
17804            }
17805
17806            let etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| {
17807                Error::other(format!(
17808                    "durable ILM decommission receipt `{receipt_path}` in pool {pool_idx} is missing an ETag"
17809                ))
17810            })?;
17811            let encoded = receipt.encode()?;
17812            match save_config_with_opts(
17813                self.pools[pool_idx].clone(),
17814                receipt_path,
17815                encoded,
17816                &ObjectOptions {
17817                    max_parity: true,
17818                    write_completion: crate::object_api::WriteCompletion::TailDrained,
17819                    http_preconditions: Some(HTTPPreconditions {
17820                        if_match: Some(etag),
17821                        ..Default::default()
17822                    }),
17823                    ..Default::default()
17824                },
17825            )
17826            .await
17827            {
17828                Ok(()) => return Ok(true),
17829                Err(Error::PreconditionFailed) if attempt < DECOMMISSION_DURABLE_ILM_RECEIPT_CAS_ATTEMPTS => {
17830                    attempt += 1;
17831                    continue;
17832                }
17833                Err(Error::PreconditionFailed) => {
17834                    return Err(Error::other(format!(
17835                        "failed to persist {stage} durable ILM decommission receipt `{receipt_path}` for {} after concurrent updates",
17836                        locator.context()
17837                    )));
17838                }
17839                Err(err) => {
17840                    return Err(Error::other(format!(
17841                        "failed to persist {stage} durable ILM decommission receipt `{receipt_path}` for {}: {err}",
17842                        locator.context()
17843                    )));
17844                }
17845            }
17846        }
17847    }
17848
17849    async fn advance_durable_ilm_decommission_receipts(
17850        &self,
17851        path: &str,
17852        data: &[u8],
17853        terminal: bool,
17854    ) -> Result<Option<Vec<usize>>> {
17855        let active_runs = {
17856            let pool_meta = self.pool_meta.read().await;
17857            pool_meta
17858                .pools
17859                .iter()
17860                .enumerate()
17861                .filter_map(|(pool_idx, pool)| {
17862                    pool.decommission
17863                        .as_ref()
17864                        .filter(|info| info.has_decommission_state() && !info.complete)
17865                        .and_then(|info| info.start_time)
17866                        .map(|start_time| (pool_idx, decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time)))
17867                })
17868                .collect::<Vec<_>>()
17869        };
17870        if active_runs.is_empty() {
17871            return Ok(None);
17872        }
17873
17874        let stage = if terminal { "terminal" } else { "progress" };
17875        let record = validate_durable_ilm_record(path, data)
17876            .map_err(|err| Error::other(format!("{stage} durable ILM record is invalid at path `{path}`: {err}")))?;
17877        let fleet_topology_generation = durable_ilm_v6_topology_generation(path, data)?;
17878        let active_source_pool_indices = active_runs.iter().map(|(pool_idx, _)| *pool_idx).collect::<Vec<_>>();
17879        let mut terminal_target_pool_indices = Vec::new();
17880        for (source_pool_idx, run_token) in active_runs {
17881            let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, record.id_kind, &record.id);
17882            let mut receipt_found = false;
17883            for pool_idx in 0..self.pools.len() {
17884                if pool_idx != source_pool_idx {
17885                    let found = self
17886                        .advance_durable_ilm_decommission_receipt(
17887                            pool_idx,
17888                            &receipt_path,
17889                            &record,
17890                            fleet_topology_generation.as_deref(),
17891                            terminal,
17892                        )
17893                        .await?;
17894                    receipt_found |= found;
17895                    if terminal
17896                        && found
17897                        && !active_source_pool_indices.contains(&pool_idx)
17898                        && !terminal_target_pool_indices.contains(&pool_idx)
17899                    {
17900                        terminal_target_pool_indices.push(pool_idx);
17901                    }
17902                }
17903            }
17904            if terminal && !receipt_found {
17905                return Err(Error::other(format!(
17906                    "terminal durable ILM record at path `{path}` {} is retained until its decommission receipt is committed",
17907                    record.context()
17908                )));
17909            }
17910        }
17911        Ok(Some(terminal_target_pool_indices))
17912    }
17913
17914    /// Resolve the exact receipt-bearing target copies on which a v6 dispatch
17915    /// manifest may advance while a decommission reservation is active.
17916    ///
17917    /// This is intentionally not a general capacity bypass. The target copy
17918    /// must still be covered by the active source reservation and its durable
17919    /// receipt, the ETag must be the caller's exact CAS generation, and the
17920    /// replacement must be a byte-non-growing adjacent manifest checkpoint.
17921    pub(crate) async fn decommission_durable_ilm_checkpoint_targets(
17922        &self,
17923        path: &str,
17924        next_data: &[u8],
17925        expected_etag: &str,
17926    ) -> Result<Option<Vec<DecommissionDurableIlmCheckpointTarget>>> {
17927        let active_runs = {
17928            let pool_meta = self.pool_meta.read().await;
17929            let mut active_runs = Vec::new();
17930            for (source_pool_index, pool) in pool_meta.pools.iter().enumerate() {
17931                let Some(info) = pool
17932                    .decommission
17933                    .as_ref()
17934                    .filter(|info| info.has_decommission_state() && !info.complete)
17935                else {
17936                    continue;
17937                };
17938                let Some(start_time) = info.start_time else {
17939                    continue;
17940                };
17941                let reservation = info.capacity_reservation.clone().ok_or_else(|| {
17942                    decommission_capacity_blocked_error(format!(
17943                        "active decommission source pool {source_pool_index} has no reservation for durable ILM checkpoint"
17944                    ))
17945                })?;
17946                if !reservation.active() {
17947                    return Err(decommission_capacity_blocked_error(format!(
17948                        "active decommission source pool {source_pool_index} has a released reservation for durable ILM checkpoint"
17949                    )));
17950                }
17951                active_runs.push((
17952                    source_pool_index,
17953                    decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time),
17954                    reservation,
17955                ));
17956            }
17957            active_runs
17958        };
17959        if active_runs.is_empty() {
17960            return Ok(None);
17961        }
17962
17963        let next_record = validate_durable_ilm_record(path, next_data)?;
17964        if next_record.namespace != TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.name {
17965            return Ok(None);
17966        }
17967        let next_fleet_topology_generation = durable_ilm_v6_topology_generation(path, next_data)?;
17968
17969        let mut targets = Vec::<DecommissionDurableIlmCheckpointTarget>::new();
17970        let mut missing_source_receipts = 0usize;
17971        let mut stale_target_etag_mismatch = false;
17972        for (source_pool_index, run_token, reservation) in active_runs {
17973            let receipt_path = decommission_durable_ilm_receipt_path(&run_token, path, next_record.id_kind, &next_record.id);
17974            let mut source_receipt_found = false;
17975            for allocation in &reservation.targets {
17976                let receipt_data = match read_config_limited_preserve_empty(
17977                    self.pools[allocation.pool_index].clone(),
17978                    &receipt_path,
17979                    DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
17980                )
17981                .await
17982                {
17983                    Ok(data) => data,
17984                    Err(err)
17985                        if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
17986                            || is_err_object_not_found(&err)
17987                            || is_err_version_not_found(&err) =>
17988                    {
17989                        continue;
17990                    }
17991                    Err(err) => return Err(err),
17992                };
17993                let receipt = DecommissionDurableIlmReceipt::decode(&receipt_data)?;
17994                let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
17995                Self::validate_decommission_durable_ilm_receipt_locator(&receipt_path, &locator, &receipt)?;
17996                if receipt.source_path != path
17997                    || receipt.namespace != next_record.namespace
17998                    || receipt.id_kind != next_record.id_kind
17999                    || receipt.id != next_record.id
18000                {
18001                    return Err(Error::other_with_context(
18002                        "durable ILM checkpoint receipt identity does not authorize source",
18003                        format!("path `{path}` {}", next_record.context()),
18004                    ));
18005                }
18006                if receipt.fleet_topology_generation != next_fleet_topology_generation {
18007                    return Err(Error::other_with_context(
18008                        "durable ILM checkpoint receipt fleet topology does not authorize source",
18009                        format!("path `{path}` {}", next_record.context()),
18010                    ));
18011                }
18012                receipt
18013                    .checkpoint
18014                    .validate_successor(&next_record.checkpoint)
18015                    .map_err(|err| {
18016                        Error::other_with_context(
18017                            "durable ILM checkpoint receipt is not a predecessor of the requested generation",
18018                            format!(
18019                                "target pool {}, path `{path}` {}; receipt checkpoint {:?}, requested checkpoint {:?}: {err}",
18020                                allocation.pool_index,
18021                                next_record.context(),
18022                                receipt.checkpoint,
18023                                next_record.checkpoint
18024                            ),
18025                        )
18026                    })?;
18027
18028                let (target_data, metadata) = read_config_limited_preserve_empty_with_metadata(
18029                    self.pools[allocation.pool_index].clone(),
18030                    path,
18031                    TIER_DELETE_DISPATCH_MANIFEST_NAMESPACE.max_record_size,
18032                )
18033                .await?;
18034                let target_record = validate_durable_ilm_record(path, &target_data)?;
18035                if target_record.namespace != next_record.namespace
18036                    || target_record.id_kind != next_record.id_kind
18037                    || target_record.id != next_record.id
18038                {
18039                    return Err(Error::other_with_context(
18040                        "durable ILM checkpoint target identity does not match source",
18041                        format!("path `{path}` {}", next_record.context()),
18042                    ));
18043                }
18044                let already_committed = target_data.as_slice() == next_data;
18045                let target_etag = metadata.etag.filter(|etag| !etag.trim().is_empty()).ok_or_else(|| {
18046                    Error::other_with_context("durable ILM checkpoint target is missing an ETag", format!("path `{path}`"))
18047                })?;
18048                if already_committed {
18049                    receipt
18050                        .checkpoint
18051                        .validate_successor(&target_record.checkpoint)
18052                        .map_err(|err| {
18053                            Error::other_with_context(
18054                                "durable ILM checkpoint receipt is not a predecessor of the committed target generation",
18055                                format!("path `{path}` {}: {err}", next_record.context()),
18056                            )
18057                        })?;
18058                } else {
18059                    if target_record.checkpoint != receipt.checkpoint {
18060                        return Err(Error::other_with_context(
18061                            "durable ILM checkpoint target is not the receipt generation",
18062                            format!("path `{path}` {}", next_record.context()),
18063                        ));
18064                    }
18065                    target_record
18066                        .checkpoint
18067                        .validate_successor(&next_record.checkpoint)
18068                        .map_err(|err| {
18069                            Error::other_with_context(
18070                                "durable ILM checkpoint target is not a predecessor of the requested generation",
18071                                format!("path `{path}` {}: {err}", next_record.context()),
18072                            )
18073                        })?;
18074                    if next_data.len() > target_data.len() {
18075                        return Err(decommission_capacity_blocked_error(format!(
18076                            "durable ILM checkpoint update at `{path}` grows from {} to {} bytes",
18077                            target_data.len(),
18078                            next_data.len()
18079                        )));
18080                    }
18081                    stale_target_etag_mismatch |= target_etag != expected_etag;
18082                }
18083
18084                source_receipt_found = true;
18085                if let Some(existing) = targets
18086                    .iter_mut()
18087                    .find(|target| target.target_pool_index == allocation.pool_index)
18088                {
18089                    if existing.already_committed != already_committed {
18090                        return Err(Error::other_with_context(
18091                            "durable ILM checkpoint target state changed during authorization",
18092                            format!("path `{path}`"),
18093                        ));
18094                    }
18095                    continue;
18096                }
18097                let base_owner = DecommissionCapacityOwner {
18098                    source_pool_index,
18099                    operation_id: reservation.operation_id,
18100                    generation: reservation.generation,
18101                    owner_nonce: reservation.owner_nonce,
18102                    mutation_id: None,
18103                };
18104                let mutation_id = decommission_capacity_mutation_id(
18105                    base_owner,
18106                    RUSTFS_META_BUCKET,
18107                    path,
18108                    Some(next_record.checkpoint.content_sha256()),
18109                    false,
18110                    None,
18111                );
18112                targets.push(DecommissionDurableIlmCheckpointTarget {
18113                    source_pool_index,
18114                    target_pool_index: allocation.pool_index,
18115                    capacity_owner: base_owner.with_mutation_id(mutation_id),
18116                    already_committed,
18117                    target_etag: Some(target_etag),
18118                });
18119            }
18120            if !source_receipt_found {
18121                missing_source_receipts = missing_source_receipts.saturating_add(1);
18122            }
18123        }
18124
18125        if missing_source_receipts > 0 {
18126            return Err(decommission_capacity_blocked_error(format!(
18127                "durable ILM checkpoint at `{path}` is missing receipt coverage for {missing_source_receipts} active source(s)"
18128            )));
18129        }
18130        if targets.is_empty() {
18131            return Err(decommission_capacity_blocked_error(format!(
18132                "durable ILM checkpoint at `{path}` has no receipt-bearing reservation target"
18133            )));
18134        }
18135        if stale_target_etag_mismatch && !targets.iter().any(|target| target.already_committed) {
18136            return Err(Error::PreconditionFailed);
18137        }
18138        // After a partial multi-target commit, an aggregate read may return
18139        // the ETag of the already-advanced target while another authorized
18140        // target still has the predecessor ETag. The exact committed bytes
18141        // plus every target's receipt/checkpoint proof authorize repairing
18142        // that predecessor with its own target-local CAS. Without an exact
18143        // committed target, retain the caller-ETag requirement above.
18144        targets.sort_unstable_by_key(|target| target.target_pool_index);
18145        Ok(Some(targets))
18146    }
18147
18148    pub(crate) async fn record_durable_ilm_decommission_progress(&self, path: &str, data: &[u8]) -> Result<()> {
18149        self.advance_durable_ilm_decommission_receipts(path, data, false)
18150            .await
18151            .map(|_| ())
18152    }
18153
18154    pub(crate) async fn record_durable_ilm_decommission_terminal(&self, path: &str, data: &[u8]) -> Result<()> {
18155        self.record_durable_ilm_decommission_terminal_target_pools(path, data)
18156            .await
18157            .map(|_| ())
18158    }
18159
18160    /// Record terminal proof and return its non-source receipt pools for targeted cleanup.
18161    pub(crate) async fn record_durable_ilm_decommission_terminal_target_pools(
18162        &self,
18163        path: &str,
18164        data: &[u8],
18165    ) -> Result<Option<Vec<usize>>> {
18166        self.advance_durable_ilm_decommission_receipts(path, data, true).await
18167    }
18168
18169    /// Return true only when `data` is the exact copy still owned by an active
18170    /// decommission source and a target-side terminal receipt authorizes that
18171    /// source's later verified cleanup. Lifecycle recovery may then regard the
18172    /// logical record as terminal without deleting the source checkpoint.
18173    pub(crate) async fn durable_ilm_terminal_receipt_covers_active_source(&self, path: &str, data: &[u8]) -> Result<bool> {
18174        let namespace = classify_durable_ilm_record(path)?
18175            .ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{path}`")))?;
18176        let source_record = validate_durable_ilm_record(path, data)?;
18177        let active_runs = {
18178            let pool_meta = self.pool_meta.read().await;
18179            pool_meta
18180                .pools
18181                .iter()
18182                .enumerate()
18183                .filter_map(|(source_pool_index, pool)| {
18184                    pool.decommission
18185                        .as_ref()
18186                        .filter(|info| info.has_decommission_state() && !info.complete)
18187                        .and_then(|info| info.start_time)
18188                        .map(|start_time| {
18189                            (source_pool_index, decommission_durable_ilm_receipt_run_token(&pool.cmd_line, start_time))
18190                        })
18191                })
18192                .collect::<Vec<_>>()
18193        };
18194        let mut covered_active_source = false;
18195        for (source_pool_index, run_token) in active_runs {
18196            let source_data =
18197                match read_config_limited_preserve_empty(self.pools[source_pool_index].clone(), path, namespace.max_record_size)
18198                    .await
18199                {
18200                    Ok(source_data) => source_data,
18201                    Err(err)
18202                        if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
18203                            || is_err_object_not_found(&err)
18204                            || is_err_version_not_found(&err) =>
18205                    {
18206                        continue;
18207                    }
18208                    Err(err) => return Err(err),
18209                };
18210            if source_data.as_slice() != data {
18211                continue;
18212            }
18213            if self
18214                .load_decommission_durable_ilm_terminal_receipt_for_run(source_pool_index, path, &source_record, &run_token)
18215                .await?
18216                .is_some()
18217            {
18218                covered_active_source = true;
18219            } else {
18220                // Every active source that still stores this exact generation
18221                // needs complete terminal receipt coverage. One covered source
18222                // cannot authorize lifecycle recovery to skip another.
18223                return Ok(false);
18224            }
18225        }
18226        Ok(covered_active_source)
18227    }
18228
18229    async fn cleanup_decommission_durable_ilm_receipts(&self, source_pool_idx: usize) -> Result<()> {
18230        for (pool_idx, receipt_path) in self.list_decommission_durable_ilm_receipts(source_pool_idx).await? {
18231            match delete_config(self.pools[pool_idx].clone(), &receipt_path).await {
18232                Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {}
18233                Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
18234                Err(err) => {
18235                    return Err(Error::other(format!(
18236                        "failed to clean durable ILM decommission receipt `{receipt_path}` from pool {pool_idx}: {err}"
18237                    )));
18238                }
18239            }
18240        }
18241        for receipt_path in self.list_decommission_durable_ilm_manifest_receipts(source_pool_idx).await? {
18242            match delete_config(self.pools[source_pool_idx].clone(), &receipt_path).await {
18243                Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {}
18244                Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
18245                Err(err) => {
18246                    return Err(Error::other(format!(
18247                        "failed to clean durable ILM expected manifest receipt `{receipt_path}` from source pool {source_pool_idx}: {err}"
18248                    )));
18249                }
18250            }
18251        }
18252        let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
18253        let manifest_path = decommission_durable_ilm_manifest_path(&run_token);
18254        match delete_config(self.pools[source_pool_idx].clone(), &manifest_path).await {
18255            Ok(()) | Err(Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound) => {}
18256            Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => {}
18257            Err(err) => {
18258                return Err(Error::other(format!(
18259                    "failed to clean durable ILM expected manifest `{manifest_path}` from source pool {source_pool_idx}: {err}"
18260                )));
18261            }
18262        }
18263        Ok(())
18264    }
18265
18266    #[cfg(all(test, feature = "test-util"))]
18267    async fn verify_and_cleanup_decommissioned_durable_ilm_record(
18268        &self,
18269        source_pool_idx: usize,
18270        source_set: Arc<SetDisks>,
18271        path: &str,
18272    ) -> Result<()> {
18273        let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
18274        self.verify_and_cleanup_decommissioned_durable_ilm_record_for_run(source_pool_idx, source_set, path, &run_token)
18275            .await
18276    }
18277
18278    async fn verify_and_cleanup_decommissioned_durable_ilm_record_for_run(
18279        &self,
18280        source_pool_idx: usize,
18281        source_set: Arc<SetDisks>,
18282        path: &str,
18283        run_token: &str,
18284    ) -> Result<()> {
18285        let namespace = classify_durable_ilm_record(path)?
18286            .ok_or_else(|| Error::other(format!("path `{path}` is not a durable ILM record")))?;
18287        let source_versions = source_set
18288            .load_file_info_versions_exact(RUSTFS_META_BUCKET, path)
18289            .await
18290            .map_err(|err| Error::other(format!("failed to load source durable ILM versions at path `{path}`: {err}")))?
18291            .ok_or_else(|| Error::other(format!("source durable ILM record is missing at path `{path}`")))?;
18292        let source = read_config_limited_preserve_empty(source_set.clone(), path, namespace.max_record_size)
18293            .await
18294            .map_err(|err| Error::other(format!("failed to read source durable ILM record at path `{path}`: {err}")))?;
18295        let source_record = validate_durable_ilm_record(path, &source)
18296            .map_err(|err| Error::other(format!("source durable ILM record is invalid at path `{path}`: {err}")))?;
18297        let source_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &source)?;
18298        let target = self
18299            .load_decommissioned_durable_ilm_target(source_pool_idx, path, namespace.max_record_size, &source_record.context())
18300            .await?;
18301        let manifest_receipt = if let Some((target_pool_idx, target)) = target {
18302            let target_record = validate_decommission_durable_ilm_copy(path, &source_record, &target)?;
18303            let target_fleet_topology_generation = durable_ilm_v6_topology_generation(path, &target)?;
18304            if target_fleet_topology_generation != source_fleet_topology_generation {
18305                return Err(Error::other_with_context(
18306                    "target durable ILM fleet topology generation differs from source",
18307                    format!("path `{path}` {}", source_record.context()),
18308                ));
18309            }
18310            let receipt = DecommissionDurableIlmReceipt::new(path, &target_record, target_fleet_topology_generation);
18311            self.persist_decommission_durable_ilm_receipt_for_run(target_pool_idx, &receipt, run_token)
18312                .await?;
18313            receipt
18314        } else {
18315            self.load_decommission_durable_ilm_terminal_receipt_for_run(source_pool_idx, path, &source_record, run_token)
18316                .await?
18317                .ok_or_else(|| {
18318                    Error::other(format!(
18319                        "target durable ILM record is missing at path `{path}` {} without a matching terminal receipt",
18320                        source_record.context()
18321                    ))
18322                })?
18323        };
18324        if manifest_receipt.fleet_topology_generation != source_fleet_topology_generation {
18325            return Err(Error::other_with_context(
18326                "terminal durable ILM receipt fleet topology generation does not cover source",
18327                format!("path `{path}` {}", source_record.context()),
18328            ));
18329        }
18330        let fleet_proof = if let Some(expected_generation) = source_fleet_topology_generation.as_deref() {
18331            let proof = acquire_tier_delete_journal_fleet_proof()
18332                .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable for source cleanup"))?;
18333            if tier_delete_journal_topology_generation(&proof) != expected_generation
18334                || !tier_delete_journal_fleet_proof_matches(&proof)
18335            {
18336                return Err(Error::other("tier delete journal v6 fleet generation changed before source cleanup"));
18337            }
18338            Some(proof)
18339        } else {
18340            None
18341        };
18342        self.persist_decommission_durable_ilm_receipt_for_run(source_pool_idx, &manifest_receipt, run_token)
18343            .await?;
18344
18345        if fleet_proof
18346            .as_ref()
18347            .is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof))
18348        {
18349            return Err(Error::other("tier delete journal v6 fleet proof changed before source cleanup"));
18350        }
18351
18352        let cleanup_result = data_movement::cleanup_source_entry_if_unchanged(
18353            source_set,
18354            RUSTFS_META_BUCKET,
18355            path,
18356            &source_versions,
18357            &[],
18358            data_movement::SourceCleanupBucketFence::default(),
18359            "decommission durable ILM final sweep",
18360        )
18361        .await
18362        .map_err(|err| {
18363            Error::other(format!(
18364                "source durable ILM cleanup failed at path `{path}` {}: {err}",
18365                source_record.context()
18366            ))
18367        });
18368        let cleanup_result = resolve_decommission_entry_cleanup_delete_result(cleanup_result, RUSTFS_META_BUCKET, path);
18369        if fleet_proof
18370            .as_ref()
18371            .is_some_and(|proof| !tier_delete_journal_fleet_proof_matches(proof))
18372        {
18373            return Err(Error::other(
18374                "tier delete journal v6 fleet proof changed during source cleanup; exact source outcome requires verification",
18375            ));
18376        }
18377        cleanup_result
18378    }
18379
18380    #[cfg(all(test, feature = "test-util"))]
18381    pub(crate) async fn verify_and_cleanup_decommissioned_durable_ilm_record_for_test(
18382        &self,
18383        source_pool_idx: usize,
18384        source_set: Arc<SetDisks>,
18385        path: &str,
18386    ) -> Result<()> {
18387        self.verify_and_cleanup_decommissioned_durable_ilm_record(source_pool_idx, source_set, path)
18388            .await
18389    }
18390
18391    #[cfg(all(test, feature = "test-util"))]
18392    pub(crate) async fn decommission_durable_ilm_receipt_count_for_test(&self, source_pool_idx: usize) -> Result<usize> {
18393        Ok(self.list_decommission_durable_ilm_receipts(source_pool_idx).await?.len())
18394    }
18395
18396    #[cfg(all(test, feature = "test-util"))]
18397    pub(crate) async fn decommission_durable_ilm_receipt_paths_for_test(
18398        &self,
18399        source_pool_idx: usize,
18400    ) -> Result<Vec<(usize, String)>> {
18401        self.list_decommission_durable_ilm_receipts(source_pool_idx).await
18402    }
18403
18404    #[cfg(all(test, feature = "test-util"))]
18405    pub(crate) async fn persist_decommission_durable_ilm_receipt_for_test(
18406        &self,
18407        source_pool_idx: usize,
18408        target_pool_idx: usize,
18409        source_path: &str,
18410        record: &ValidatedDurableIlmRecord,
18411        terminal: bool,
18412    ) -> Result<String> {
18413        let fleet_topology_generation = match read_config_limited_preserve_empty(
18414            self.pools[target_pool_idx].clone(),
18415            source_path,
18416            classify_durable_ilm_record(source_path)?
18417                .ok_or_else(|| Error::other_with_context("path is not a durable ILM record", format!("path `{source_path}`")))?
18418                .max_record_size,
18419        )
18420        .await
18421        {
18422            Ok(target_data) => durable_ilm_v6_topology_generation(source_path, &target_data)?,
18423            Err(err)
18424                if matches!(&err, Error::ConfigNotFound | Error::FileNotFound | Error::FileVersionNotFound)
18425                    || is_err_object_not_found(&err)
18426                    || is_err_version_not_found(&err) =>
18427            {
18428                None
18429            }
18430            Err(err) => return Err(err),
18431        };
18432        let mut receipt = DecommissionDurableIlmReceipt::new(source_path, record, fleet_topology_generation);
18433        if terminal {
18434            receipt.terminal_checkpoint = Some(record.checkpoint.clone());
18435        }
18436        self.persist_decommission_durable_ilm_receipt(source_pool_idx, target_pool_idx, &receipt)
18437            .await?;
18438        self.decommission_durable_ilm_receipt_path_for_test(source_pool_idx, source_path, record)
18439            .await
18440    }
18441
18442    #[cfg(all(test, feature = "test-util"))]
18443    pub(crate) async fn decommission_durable_ilm_receipt_path_for_test(
18444        &self,
18445        source_pool_idx: usize,
18446        source_path: &str,
18447        record: &ValidatedDurableIlmRecord,
18448    ) -> Result<String> {
18449        let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
18450        Ok(decommission_durable_ilm_receipt_path(&run_token, source_path, record.id_kind, &record.id))
18451    }
18452
18453    #[cfg(all(test, feature = "test-util"))]
18454    pub(crate) async fn persist_decommission_durable_ilm_manifest_for_test(&self, source_pool_idx: usize) -> Result<()> {
18455        self.persist_decommission_durable_ilm_manifest(source_pool_idx).await
18456    }
18457
18458    #[cfg(all(test, feature = "test-util"))]
18459    pub(crate) async fn cleanup_decommission_durable_ilm_receipts_for_test(&self, source_pool_idx: usize) -> Result<()> {
18460        self.cleanup_decommission_durable_ilm_receipts(source_pool_idx).await
18461    }
18462
18463    async fn check_after_decommission(
18464        self: &Arc<Self>,
18465        idx: usize,
18466        rx: &CancellationToken,
18467        generation: OffsetDateTime,
18468    ) -> Result<Vec<DecommissionUnresolvedEntry>> {
18469        let run_token = self.durable_ilm_receipt_run_token_for_generation(idx, generation).await?;
18470        let operation_gate = self.ctx.data_movement_operation_gate();
18471        self.run_guarded_decommission_side_effect(rx, &operation_gate, || {
18472            self.check_after_decommission_unfenced(idx, generation, run_token)
18473        })
18474        .await
18475    }
18476
18477    async fn check_after_decommission_unfenced(
18478        self: &Arc<Self>,
18479        idx: usize,
18480        generation: OffsetDateTime,
18481        run_token: String,
18482    ) -> Result<Vec<DecommissionUnresolvedEntry>> {
18483        let unresolved_entries = {
18484            let pool_meta = self.pool_meta.read().await;
18485            ensure_decommission_generation(&pool_meta, idx, generation)?;
18486            let info = pool_meta.pools[idx]
18487                .decommission
18488                .as_ref()
18489                .ok_or_else(|| decommission_metadata_not_initialized_error("verify unresolved decommission entries"))?;
18490            if info
18491                .unresolved_entries
18492                .iter()
18493                .any(|entry| entry.pool_index != idx || entry.source_generation != generation)
18494            {
18495                return Err(Error::other(format!(
18496                    "failed to verify decommission for pool {idx}: unresolved listing ledger contains a different pool or generation"
18497                )));
18498            }
18499            info.unresolved_entries.clone()
18500        };
18501        let unresolved_entries_by_identity = Arc::new(
18502            unresolved_entries
18503                .iter()
18504                .map(|entry| (decommission_unresolved_entry_identity(entry), entry.clone()))
18505                .collect::<HashMap<_, _>>(),
18506        );
18507        let resolved_unresolved_entries = Arc::new(tokio::sync::Mutex::new(Vec::new()));
18508        let buckets = self.get_buckets_to_decommission().await?;
18509        let pool = self.pools[idx].clone();
18510
18511        self.ensure_decommission_multipart_uploads_drained(idx, &pool, &buckets)
18512            .await?;
18513
18514        for (set_index, set) in pool.disk_set.iter().enumerate() {
18515            let require_all_disks = unresolved_entries.iter().any(|entry| entry.set_index == set_index);
18516            for bucket_info in &buckets {
18517                let mut lifecycle_config = None;
18518                let mut object_lock_config = None;
18519                let mut replication_configured = false;
18520                if bucket_info.name != RUSTFS_META_BUCKET {
18521                    let expiry_configs = get_expiry_configs(self, &bucket_info.name).await?;
18522                    lifecycle_config = expiry_configs.lifecycle.map(|config| (*config).clone());
18523                    object_lock_config = expiry_configs.object_lock.map(|config| (*config).clone());
18524                    replication_configured = resolve_decommission_optional_bucket_config_result(
18525                        &bucket_info.name,
18526                        "replication",
18527                        metadata_sys::get_replication_config_in(&self.ctx, &bucket_info.name).await,
18528                    )?
18529                    .is_some();
18530                }
18531
18532                let versions_found = Arc::new(AtomicUsize::new(0));
18533                let entry_error = Arc::new(tokio::sync::Mutex::new(None::<Error>));
18534                let first_remaining_path = Arc::new(tokio::sync::Mutex::new(None::<String>));
18535                let callback_rx = CancellationToken::new();
18536                let versions_found_cb = versions_found.clone();
18537                let entry_error_cb = entry_error.clone();
18538                let first_remaining_path_cb = first_remaining_path.clone();
18539                let bucket_name = bucket_info.name.clone();
18540                let lifecycle_config_cb = lifecycle_config.clone();
18541                let object_lock_config_cb = object_lock_config.clone();
18542                let replication_configured_cb = replication_configured;
18543                let store = Arc::clone(self);
18544                let source_set = set.clone();
18545                let callback_rx_cb = callback_rx.clone();
18546                let unresolved_entries_by_identity_cb = unresolved_entries_by_identity.clone();
18547                let resolved_unresolved_entries_cb = resolved_unresolved_entries.clone();
18548                let run_token_cb = run_token.clone();
18549
18550                let callback: ListCallback = Arc::new(move |entry: MetaCacheEntry| {
18551                    let versions_found = versions_found_cb.clone();
18552                    let entry_error = entry_error_cb.clone();
18553                    let first_remaining_path = first_remaining_path_cb.clone();
18554                    let bucket_name = bucket_name.clone();
18555                    let lifecycle_config = lifecycle_config_cb.clone();
18556                    let object_lock_config = object_lock_config_cb.clone();
18557                    let replication_configured = replication_configured_cb;
18558                    let store = Arc::clone(&store);
18559                    let source_set = source_set.clone();
18560                    let callback_rx = callback_rx_cb.clone();
18561                    let unresolved_entries_by_identity = unresolved_entries_by_identity_cb.clone();
18562                    let resolved_unresolved_entries = resolved_unresolved_entries_cb.clone();
18563                    let run_token = run_token_cb.clone();
18564                    Box::pin(async move {
18565                        if callback_rx.is_cancelled() {
18566                            return;
18567                        }
18568
18569                        if !entry.is_object() {
18570                            return;
18571                        }
18572
18573                        if is_decommission_set_local_usage_cache(&bucket_name, &entry.name) {
18574                            return;
18575                        }
18576
18577                        let durable_ilm_record = if bucket_name == RUSTFS_META_BUCKET {
18578                            match classify_durable_ilm_record(&entry.name) {
18579                                Ok(record) => record,
18580                                Err(err) => {
18581                                    let mut first_err = entry_error.lock().await;
18582                                    if first_err.is_none() {
18583                                        *first_err = Some(with_decommission_entry_context(
18584                                            "check_after_decommission.durable_ilm_namespace",
18585                                            &bucket_name,
18586                                            &entry.name,
18587                                            err,
18588                                        ));
18589                                        callback_rx.cancel();
18590                                    }
18591                                    return;
18592                                }
18593                            }
18594                        } else {
18595                            None
18596                        };
18597
18598                        if durable_ilm_record.is_some() {
18599                            if let Err(err) = store
18600                                .verify_and_cleanup_decommissioned_durable_ilm_record_for_run(
18601                                    idx,
18602                                    source_set,
18603                                    &entry.name,
18604                                    &run_token,
18605                                )
18606                                .await
18607                            {
18608                                let mut first_err = entry_error.lock().await;
18609                                if first_err.is_none() {
18610                                    *first_err = Some(err);
18611                                    callback_rx.cancel();
18612                                }
18613                            }
18614                            return;
18615                        }
18616
18617                        let mut fivs = match load_decommission_entry_exact_versions(
18618                            &source_set,
18619                            &entry,
18620                            &bucket_name,
18621                            "check_after_decommission.file_info_versions",
18622                        )
18623                        .await
18624                        {
18625                            Ok(fivs) => fivs,
18626                            Err(err) => {
18627                                let mut first_err = entry_error.lock().await;
18628                                if first_err.is_none() {
18629                                    *first_err = Some(err);
18630                                    callback_rx.cancel();
18631                                }
18632                                return;
18633                            }
18634                        };
18635
18636                        fivs.versions
18637                            .sort_by_key(|version| (version.mod_time.is_none(), std::cmp::Reverse(version.mod_time)));
18638
18639                        let mut remaining = 0;
18640                        let mut expired = 0;
18641                        for version in fivs.versions.iter().chain(fivs.free_versions.iter()) {
18642                            if version.tier_free_version() {
18643                                remaining += 1;
18644                                debug!(
18645                                    event = EVENT_DECOMMISSION_ENTRY,
18646                                    component = LOG_COMPONENT_ECSTORE,
18647                                    subsystem = LOG_SUBSYSTEM_POOLS,
18648                                    pool_index = idx,
18649                                    bucket = %bucket_name,
18650                                    object = %entry.name,
18651                                    version_id = ?version.version_id,
18652                                    reason = DECOMMISSION_FREE_VERSION_SWEEP_REASON,
18653                                    state = "free_version_retained",
18654                                    "Decommission final sweep retained a free version"
18655                                );
18656                                continue;
18657                            }
18658                            let skip_lifecycle = match should_skip_lifecycle_for_data_movement(
18659                                Arc::clone(&store),
18660                                &bucket_name,
18661                                version,
18662                                lifecycle_config.as_ref(),
18663                                object_lock_config.as_ref(),
18664                                false,
18665                                &LcEventSrc::Decom,
18666                                None,
18667                            )
18668                            .await
18669                            {
18670                                Ok(skip_lifecycle) => skip_lifecycle,
18671                                Err(err) => {
18672                                    let mut first_err = entry_error.lock().await;
18673                                    if first_err.is_none() {
18674                                        *first_err = Some(err);
18675                                        callback_rx.cancel();
18676                                    }
18677                                    return;
18678                                }
18679                            };
18680                            if skip_lifecycle {
18681                                expired += 1;
18682                                continue;
18683                            }
18684                            let remaining_versions = decommission_remaining_version_count(&fivs.versions, expired);
18685                            if should_skip_decommission_delete_marker(version, remaining_versions, replication_configured) {
18686                                continue;
18687                            }
18688                            remaining += 1;
18689                        }
18690
18691                        if remaining > 0 {
18692                            let mut first_path = first_remaining_path.lock().await;
18693                            if first_path.is_none() {
18694                                *first_path = Some(format!("{bucket_name}/{}", entry.name));
18695                            }
18696                        } else {
18697                            let identity = (set_index, bucket_name.clone(), entry.name.clone());
18698                            if let Some(unresolved_entry) = unresolved_entries_by_identity.get(&identity) {
18699                                let mut resolved = resolved_unresolved_entries.lock().await;
18700                                if !resolved.contains(unresolved_entry) {
18701                                    resolved.push(unresolved_entry.clone());
18702                                }
18703                            }
18704                        }
18705
18706                        versions_found.fetch_add(remaining, Ordering::Relaxed);
18707                    })
18708                });
18709
18710                let list_result = set
18711                    .list_objects_to_decommission(
18712                        self.clone(),
18713                        callback_rx,
18714                        bucket_info.clone(),
18715                        callback,
18716                        entry_error.clone(),
18717                        idx,
18718                        set_index,
18719                        generation,
18720                        require_all_disks,
18721                    )
18722                    .await;
18723                let entry_error = entry_error.lock().await.clone();
18724                resolve_decommission_check_after_list_result(list_result, entry_error)?;
18725
18726                let versions_found = versions_found.load(Ordering::Relaxed);
18727                if versions_found > 0 {
18728                    let first_remaining_path = first_remaining_path
18729                        .lock()
18730                        .await
18731                        .clone()
18732                        .unwrap_or_else(|| format!("{}/<unknown>", bucket_info.name));
18733                    return Err(Error::other(format!(
18734                        "at least {versions_found} object(s)/version(s) were found in bucket `{}` after decommissioning; first remaining path `{first_remaining_path}`",
18735                        bucket_info.name,
18736                    )));
18737                }
18738            }
18739        }
18740
18741        let mut verified_unresolved_entries = resolved_unresolved_entries.lock().await.clone();
18742        for entry in &unresolved_entries {
18743            let set = pool.disk_set.get(entry.set_index).ok_or_else(|| {
18744                Error::other(format!(
18745                    "failed to verify decommission for pool {idx}: unresolved listing entry references missing set {}",
18746                    entry.set_index
18747                ))
18748            })?;
18749            if set.decommission_unresolved_entry_absent_on_all_disks(idx, entry).await?
18750                && !verified_unresolved_entries.contains(entry)
18751            {
18752                verified_unresolved_entries.push(entry.clone());
18753            }
18754            if !verified_unresolved_entries.contains(entry) {
18755                return Err(Error::other(format!(
18756                    "failed to verify decommission for pool {idx}: unresolved listing entry {}/{} in set {} was neither re-observed as resolved nor absent on every source disk",
18757                    entry.bucket, entry.object, entry.set_index
18758                )));
18759            }
18760        }
18761
18762        self.persist_decommission_durable_ilm_manifest_for_run(idx, &run_token)
18763            .await?;
18764        self.verify_decommission_durable_ilm_receipts_for_run(idx, &run_token).await?;
18765
18766        Ok(verified_unresolved_entries)
18767    }
18768
18769    async fn ensure_decommission_multipart_uploads_drained(
18770        &self,
18771        idx: usize,
18772        pool: &Sets,
18773        buckets: &[DecomBucketInfo],
18774    ) -> Result<()> {
18775        let mut bucket_names = buckets
18776            .iter()
18777            .filter(|bucket| bucket.name != RUSTFS_META_BUCKET)
18778            .map(|bucket| bucket.name.as_str())
18779            .collect::<Vec<_>>();
18780        bucket_names.sort_unstable();
18781        bucket_names.dedup();
18782
18783        // Take one bucket fence at a time so cross-bucket COPY cannot form an
18784        // ABBA cycle. Suspension prevents new source uploads after each fence.
18785        for bucket in bucket_names {
18786            let lifecycle_guard = self.acquire_bucket_lifecycle_write_lock(bucket).await?;
18787            if lifecycle_guard.is_lock_lost() {
18788                return Err(Error::other(format!(
18789                    "decommission multipart drain lost the bucket lifecycle fence for `{bucket}`"
18790                )));
18791            }
18792            for set in &pool.disk_set {
18793                if let Some(upload_path) = set.first_multipart_upload_path_for_decommission(bucket).await? {
18794                    return Err(Error::other(format!(
18795                        "pool {idx} still contains multipart upload `{upload_path}` for bucket `{bucket}`; resolve it before retrying decommission"
18796                    )));
18797                }
18798            }
18799        }
18800
18801        Ok(())
18802    }
18803
18804    #[cfg(test)]
18805    pub(crate) async fn ensure_decommission_multipart_uploads_drained_for_test(self: &Arc<Self>, idx: usize) -> Result<()> {
18806        let buckets = self.get_buckets_to_decommission().await?;
18807        let pool = self.pools[idx].clone();
18808        self.ensure_decommission_multipart_uploads_drained(idx, pool.as_ref(), &buckets)
18809            .await
18810    }
18811    #[cfg(all(test, feature = "test-util"))]
18812    pub(crate) async fn check_after_decommission_for_test(self: &Arc<Self>, idx: usize) -> Result<()> {
18813        let generation = self.active_decommission_generation(idx).await?;
18814        self.check_after_decommission(idx, &CancellationToken::new(), generation)
18815            .await
18816            .map(|_| ())
18817    }
18818
18819    #[tracing::instrument(skip(self, rd))]
18820    async fn decommission_object(
18821        self: Arc<Self>,
18822        pool_idx: usize,
18823        bucket: String,
18824        rd: GetObjectReader,
18825        expected_bucket_incarnation_id: Option<uuid::Uuid>,
18826        capacity_owner: Option<DecommissionCapacityOwner>,
18827    ) -> Result<()> {
18828        warn!("decommission_object: start {} {}", &bucket, &rd.object_info.name);
18829        let object_name = rd.object_info.name.clone();
18830        let mut migration = tokio::task::JoinSet::new();
18831        migration.spawn(data_movement::migrate_decommission_object(
18832            self,
18833            pool_idx,
18834            bucket.clone(),
18835            rd,
18836            expected_bucket_incarnation_id,
18837            "decommission_object",
18838            capacity_owner,
18839        ));
18840        let result = migration
18841            .join_next()
18842            .await
18843            .ok_or_else(|| Error::other("decommission migration task was not started"))?
18844            .map_err(|err| Error::other(format!("decommission migration task join error: {err}")))?;
18845        if result.is_ok() {
18846            warn!("decommission_object: migrated {} {}", &bucket, &object_name);
18847        }
18848        result
18849    }
18850}
18851
18852#[cfg(test)]
18853async fn persist_v3_pool_meta_for_test(store: &Arc<ECStore>) {
18854    let mut meta = PoolMeta::default();
18855    meta.load_no_lock_from_replicas(store.pools.clone())
18856        .await
18857        .expect("the baseline pool metadata should be readable before V3 migration");
18858    meta.version = POOL_META_GENERATION_VERSION;
18859    let mut write_state = store.pool_meta_save_gate.lock().await.clone();
18860    temp_env::async_with_vars(
18861        [
18862            (rustfs_config::ENV_POOL_META_V3_WRITE, Some("true")),
18863            (rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED, Some("true")),
18864        ],
18865        async {
18866            meta.save_no_lock_observing(store.pools.clone(), &mut write_state)
18867                .await
18868                .expect("the baseline metadata should be upgraded to a durable V3 replica");
18869        },
18870    )
18871    .await;
18872
18873    let mut loaded = PoolMeta::default();
18874    loaded
18875        .load_no_lock_from_replicas(store.pools.clone())
18876        .await
18877        .expect("the upgraded V3 pool metadata should be readable");
18878    assert_eq!(loaded.version, POOL_META_GENERATION_VERSION);
18879    store.pool_meta.write().await.version = POOL_META_GENERATION_VERSION;
18880}
18881
18882#[cfg(test)]
18883#[allow(clippy::items_after_test_module)]
18884mod tests {
18885    use super::*;
18886    use crate::bucket::replication::{ReplicationState, ReplicationStatusType};
18887    use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
18888    use crate::storage_api_contracts::multipart::MultipartOperations as _;
18889    use serde::Serialize;
18890
18891    #[test]
18892    fn decommission_set_local_usage_cache_classification_is_exact() {
18893        for object in [
18894            "buckets/.usage-cache.bin",
18895            "buckets/.usage-cache.bin.bkp",
18896            "buckets/photos/.usage-cache.bin",
18897            "buckets/photos/.usage-cache.bin.bkp",
18898        ] {
18899            assert!(is_decommission_set_local_usage_cache(RUSTFS_META_BUCKET, object), "{object}");
18900            assert!(!is_decommission_set_local_usage_cache("user-bucket", object), "{object}");
18901        }
18902        for object in [
18903            "buckets/.usage.v2.json",
18904            "buckets/.usage.v2.json.bkp",
18905            "buckets/.usage-cache.bin.extra",
18906            "buckets/.usage-cache.bin.bkp.extra",
18907            "buckets/prefix.usage-cache.bin",
18908            "buckets/photos/.usage-cache.bin.bkp.bkp",
18909            "buckets/photos/nested/.usage-cache.bin",
18910            "buckets//.usage-cache.bin",
18911            "buckets/../.usage-cache.bin",
18912            "buckets/./.usage-cache.bin",
18913            "buckets/.usage-cache.bin/child",
18914            "config/.usage-cache.bin",
18915            "buckets-other/.usage-cache.bin",
18916            ".usage-cache.bin",
18917        ] {
18918            assert!(!is_decommission_set_local_usage_cache(RUSTFS_META_BUCKET, object), "{object}");
18919        }
18920    }
18921
18922    #[tokio::test]
18923    #[serial_test::serial]
18924    async fn decommission_keeps_set_local_usage_caches_out_of_target_capacity() {
18925        use crate::object_api::PutObjReader;
18926        use tokio::io::AsyncReadExt as _;
18927
18928        // Keep the scenario's large setup and migration futures off the test
18929        // future so ordinary metadata I/O retains the default thread stack.
18930        let (_temp_dirs, store, _other_store) =
18931            Box::pin(crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None)).await;
18932        let user_bucket = "decommission-usage-cache-control";
18933        Box::pin(store.make_bucket(user_bucket, &MakeBucketOptions::default()))
18934            .await
18935            .expect("create the ordinary-object control bucket");
18936        let incarnation = Box::pin(store.bucket_incarnation_id(user_bucket))
18937            .await
18938            .expect("control bucket incarnation");
18939        let source_time = OffsetDateTime::now_utc();
18940        let cache_objects = [
18941            "buckets/.usage-cache.bin",
18942            "buckets/.usage-cache.bin.bkp",
18943            "buckets/photos/.usage-cache.bin",
18944            "buckets/photos/.usage-cache.bin.bkp",
18945        ];
18946        let conflict_object = "buckets/.usage-cache.bin.conflict";
18947        let source_read_failure_object = "buckets/.usage-cache.bin.read-error";
18948        let source_body = b"source set cache";
18949        let target_body = b"independent older target set cache";
18950        for object in cache_objects.into_iter().chain([conflict_object]) {
18951            for (pool_index, body, mod_time) in [
18952                (0, source_body.as_slice(), source_time),
18953                (1, target_body.as_slice(), source_time - Duration::seconds(1)),
18954            ] {
18955                // Scanner cache persistence writes directly to its own set.
18956                store.pools[pool_index]
18957                    .get_disks_by_key(object)
18958                    .put_object(
18959                        RUSTFS_META_BUCKET,
18960                        object,
18961                        &mut PutObjReader::from_vec(body.to_vec()),
18962                        &ObjectOptions {
18963                            mod_time: Some(mod_time),
18964                            ..Default::default()
18965                        },
18966                    )
18967                    .await
18968                    .expect("seed distinct native set-local objects");
18969            }
18970        }
18971        let controls = [
18972            (RUSTFS_META_BUCKET, "buckets/.usage.v2.json"),
18973            (RUSTFS_META_BUCKET, "buckets/photos/.usage-cache.bin.extra"),
18974            (user_bucket, "ordinary-object"),
18975            (user_bucket, "buckets/.usage-cache.bin"),
18976        ];
18977        for (bucket, object) in controls {
18978            store.pools[0]
18979                .put_object(
18980                    bucket,
18981                    object,
18982                    &mut PutObjReader::from_vec(b"ordinary object contents".to_vec()),
18983                    &ObjectOptions {
18984                        expected_bucket_incarnation_id: (bucket == user_bucket).then_some(incarnation),
18985                        ..Default::default()
18986                    },
18987                )
18988                .await
18989                .expect("seed a control that must migrate");
18990        }
18991        store.pools[0]
18992            .put_object(
18993                RUSTFS_META_BUCKET,
18994                source_read_failure_object,
18995                &mut PutObjReader::from_vec(source_body.to_vec()),
18996                &ObjectOptions::default(),
18997            )
18998            .await
18999            .expect("seed a similarly named object whose source read will fail");
19000
19001        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19002        set_decommission_capacity_info_overrides_for_test(
19003            store.id,
19004            vec![vec![
19005                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 16_384, 16_384),
19006                DecommissionPoolCapacityInfo::for_test(1, layout, 131_072, 131_072, 0),
19007            ]],
19008        );
19009        Box::pin(store.save_current_pool_meta_for_decommission_start(&[0], Vec::new()))
19010            .await
19011            .expect("activate the decommission capacity reservation");
19012
19013        for object in cache_objects {
19014            Box::pin(store.decommission_entry_for_test(
19015                0,
19016                MetaCacheEntry {
19017                    name: object.to_string(),
19018                    ..Default::default()
19019                },
19020                RUSTFS_META_BUCKET.to_string(),
19021                store.pools[0].get_disks_by_key(object),
19022            ))
19023            .await
19024            .expect("set-local cache must not enter cross-pool migration");
19025            for (pool_index, expected) in [(0, source_body.as_slice()), (1, target_body.as_slice())] {
19026                let mut reader = store.pools[pool_index]
19027                    .get_disks_by_key(object)
19028                    .get_object_reader(RUSTFS_META_BUCKET, object, None, HeaderMap::new(), &ObjectOptions::default())
19029                    .await
19030                    .expect("each set must retain its own cache");
19031                let mut actual = Vec::new();
19032                reader.stream.read_to_end(&mut actual).await.expect("read retained cache");
19033                assert_eq!(actual, expected, "pool {pool_index}, {object}");
19034            }
19035            let meta = store.pool_meta.read().await;
19036            let info = meta.pools[0].decommission.as_ref().expect("decommission progress");
19037            assert_eq!((info.items_decommissioned, info.items_decommission_failed), (0, 0));
19038            assert_eq!((info.bytes_done, info.bytes_failed), (0, 0));
19039            let reservation = info.capacity_reservation.as_ref().expect("capacity reservation");
19040            assert_eq!(reservation.pending_target_physical_bytes, 0);
19041            assert_eq!(reservation.consumed_target_physical_bytes, 0);
19042            assert!(reservation.targets.iter().all(|target| target.pending_mutation_id.is_none()));
19043        }
19044        let mut persisted = PoolMeta::default();
19045        Box::pin(persisted.load_no_lock_from_replicas(store.pools.clone()))
19046            .await
19047            .expect("reload durable capacity intents after cache entries");
19048        let reservation = persisted.pools[0]
19049            .decommission
19050            .as_ref()
19051            .and_then(|info| info.capacity_reservation.as_ref())
19052            .expect("durable reservation");
19053        assert_eq!(reservation.pending_target_physical_bytes, 0);
19054        assert!(reservation.targets.iter().all(|target| target.pending_mutation_id.is_none()));
19055
19056        let injected_reads = Arc::new(AtomicUsize::new(0));
19057        let observed_reads = Arc::clone(&injected_reads);
19058        let read_fault = DecommissionTestFaultGuard::install(Arc::new(move |stage, bucket, object, _, success| {
19059            if stage == "object_read" && bucket == RUSTFS_META_BUCKET && object == source_read_failure_object && success {
19060                observed_reads.fetch_add(1, Ordering::SeqCst);
19061                return true;
19062            }
19063            false
19064        }));
19065        Box::pin(store.decommission_entry_for_test(
19066            0,
19067            MetaCacheEntry {
19068                name: source_read_failure_object.to_string(),
19069                ..Default::default()
19070            },
19071            RUSTFS_META_BUCKET.to_string(),
19072            store.pools[0].get_disks_by_key(source_read_failure_object),
19073        ))
19074        .await
19075        .expect("entry must record the non-NotFound source read failure");
19076        drop(read_fault);
19077        assert_eq!(injected_reads.load(Ordering::SeqCst), DECOMMISSION_VERSION_COPY_ATTEMPTS);
19078        {
19079            let meta = store.pool_meta.read().await;
19080            let info = meta.pools[0].decommission.as_ref().expect("source read failure progress");
19081            assert_eq!((info.items_decommissioned, info.items_decommission_failed), (0, 1));
19082            assert_eq!(info.bytes_failed, source_body.len());
19083            assert_eq!(
19084                info.capacity_reservation
19085                    .as_ref()
19086                    .expect("reservation")
19087                    .pending_target_physical_bytes,
19088                0
19089            );
19090        }
19091        let mut retained = store.pools[0]
19092            .get_object_reader(
19093                RUSTFS_META_BUCKET,
19094                source_read_failure_object,
19095                None,
19096                HeaderMap::new(),
19097                &ObjectOptions::default(),
19098            )
19099            .await
19100            .expect("source read failure must retain the source");
19101        let mut retained_body = Vec::new();
19102        retained
19103            .stream
19104            .read_to_end(&mut retained_body)
19105            .await
19106            .expect("read retained source");
19107        assert_eq!(retained_body, source_body);
19108        drop(retained);
19109        let target_err = store.pools[1]
19110            .get_object_info(RUSTFS_META_BUCKET, source_read_failure_object, &ObjectOptions::default())
19111            .await
19112            .expect_err("failed source read must not create a target object");
19113        assert!(is_err_object_not_found(&target_err), "unexpected target state: {target_err:?}");
19114
19115        for (bucket, object) in controls {
19116            Box::pin(store.decommission_entry_for_test(
19117                0,
19118                MetaCacheEntry {
19119                    name: object.to_string(),
19120                    ..Default::default()
19121                },
19122                bucket.to_string(),
19123                store.pools[0].get_disks_by_key(object),
19124            ))
19125            .await
19126            .expect("ordinary and similarly named objects must migrate");
19127            let mut reader = store.pools[1]
19128                .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
19129                .await
19130                .expect("control must exist on the target");
19131            let mut actual = Vec::new();
19132            reader.stream.read_to_end(&mut actual).await.expect("read migrated control");
19133            assert_eq!(actual, b"ordinary object contents", "{bucket}/{object}");
19134            let err = store.pools[0]
19135                .get_object_info(bucket, object, &ObjectOptions::default())
19136                .await
19137                .expect_err("migrated control must be removed from the source");
19138            assert!(is_err_object_not_found(&err), "{bucket}/{object}: {err:?}");
19139        }
19140
19141        Box::pin(store.decommission_entry_for_test(
19142            0,
19143            MetaCacheEntry {
19144                name: conflict_object.to_string(),
19145                ..Default::default()
19146            },
19147            RUSTFS_META_BUCKET.to_string(),
19148            store.pools[0].get_disks_by_key(conflict_object),
19149        ))
19150        .await
19151        .expect("entry must record a real conditional-copy failure");
19152        let meta = store.pool_meta.read().await;
19153        let info = meta.pools[0].decommission.as_ref().expect("final progress");
19154        assert_eq!(info.items_decommissioned, controls.len());
19155        assert_eq!(
19156            info.items_decommission_failed, 2,
19157            "similar names must not hide read or migration failures"
19158        );
19159        assert_eq!(info.bytes_failed, source_body.len() * 2);
19160        drop(meta);
19161        for (pool_index, expected) in [(0, source_body.as_slice()), (1, target_body.as_slice())] {
19162            let mut reader = store.pools[pool_index]
19163                .get_object_reader(RUSTFS_META_BUCKET, conflict_object, None, HeaderMap::new(), &ObjectOptions::default())
19164                .await
19165                .expect("failed migration must preserve both objects");
19166            let mut actual = Vec::new();
19167            reader.stream.read_to_end(&mut actual).await.expect("read conflict object");
19168            assert_eq!(actual, expected);
19169        }
19170    }
19171
19172    #[tokio::test]
19173    #[serial_test::serial]
19174    async fn decommission_final_sweep_excludes_only_set_local_usage_caches() {
19175        use crate::object_api::PutObjReader;
19176
19177        let (_temp_dirs, store, _other_store) =
19178            crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
19179        for object in [
19180            "buckets/.usage-cache.bin",
19181            "buckets/.usage-cache.bin.bkp",
19182            "buckets/photos/.usage-cache.bin",
19183            "buckets/photos/.usage-cache.bin.bkp",
19184        ] {
19185            store.pools[0]
19186                .get_disks_by_key(object)
19187                .put_object(
19188                    RUSTFS_META_BUCKET,
19189                    object,
19190                    &mut PutObjReader::from_vec(b"set-local cache".to_vec()),
19191                    &ObjectOptions::default(),
19192                )
19193                .await
19194                .expect("seed each supported set-local cache path");
19195        }
19196        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19197        set_decommission_capacity_info_overrides_for_test(
19198            store.id,
19199            vec![vec![
19200                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 16_384, 16_384),
19201                DecommissionPoolCapacityInfo::for_test(1, layout, 131_072, 131_072, 0),
19202            ]],
19203        );
19204        store
19205            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19206            .await
19207            .expect("activate the final-sweep generation");
19208        let generation = store.active_decommission_generation(0).await.expect("active generation");
19209        store
19210            .check_after_decommission(0, &CancellationToken::new(), generation)
19211            .await
19212            .expect("the four set-local cache forms must not block the final sweep");
19213
19214        for object in [
19215            "buckets/.usage-cache.bin.extra",
19216            "buckets/photos/.usage-cache.bin.bkp.extra",
19217            "buckets/.usage.v2.json",
19218        ] {
19219            let source_set = store.pools[0].get_disks_by_key(object);
19220            source_set
19221                .put_object(
19222                    RUSTFS_META_BUCKET,
19223                    object,
19224                    &mut PutObjReader::from_vec(b"unmigrated ordinary metadata".to_vec()),
19225                    &ObjectOptions::default(),
19226                )
19227                .await
19228                .expect("seed ordinary metadata that must prevent completion");
19229            let err = store
19230                .check_after_decommission(0, &CancellationToken::new(), generation)
19231                .await
19232                .expect_err("a remaining similar name or global usage snapshot must block completion");
19233            assert!(err.to_string().contains("after decommissioning"), "unexpected final-sweep error: {err:?}");
19234            assert!(err.to_string().contains(object), "the final sweep must identify {object}: {err:?}");
19235            source_set
19236                .delete_object(RUSTFS_META_BUCKET, object, ObjectOptions::default())
19237                .await
19238                .expect("remove only the ordinary-metadata control before the next sweep");
19239        }
19240        store
19241            .check_after_decommission(0, &CancellationToken::new(), generation)
19242            .await
19243            .expect("only the four set-local caches remain after removing the controls");
19244    }
19245
19246    #[test]
19247    fn pool_activation_fleet_proof_error_classifier_matches_only_retryable_proof_failures() {
19248        assert!(is_pool_activation_fleet_proof_error(&Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED)));
19249        assert!(is_pool_activation_fleet_proof_error(&Error::other(POOL_ACTIVATION_FLEET_PROOF_EXPIRED)));
19250        let wrapped = format!("rebalance meta save failed during start_rebalance: {POOL_ACTIVATION_FLEET_PROOF_EXPIRED}");
19251        assert!(is_pool_activation_fleet_proof_error(&Error::other(wrapped)));
19252        assert!(!is_pool_activation_fleet_proof_error(&Error::ConfigNotFound));
19253    }
19254
19255    #[tokio::test]
19256    #[serial_test::serial]
19257    async fn decommission_v1_start_preflights_reject_before_metadata_writes() {
19258        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
19259        let mut v1_meta = store.pool_meta.read().await.clone();
19260        v1_meta.version = POOL_META_V1_VERSION;
19261        let v1_data = pool_meta_v1_replica_test_data(&v1_meta);
19262        for pool in &store.pools {
19263            save_config_with_opts(
19264                pool.clone(),
19265                POOL_META_NAME,
19266                v1_data.clone(),
19267                &ObjectOptions {
19268                    max_parity: true,
19269                    ..Default::default()
19270                },
19271            )
19272            .await
19273            .expect("the V1 baseline should be persisted to every pool");
19274        }
19275        let baseline = load_pool_meta_replicas(store.pools.clone(), true)
19276            .await
19277            .expect("baseline pool metadata should be readable");
19278        assert_eq!(baseline.meta.version, POOL_META_V1_VERSION);
19279        *store.pool_meta.write().await = baseline.meta.clone();
19280        let start_probe = PoolActivationStartProbe::install(PoolActivationStartKind::Decommission);
19281        let err = store
19282            .start_decommission(vec![0])
19283            .await
19284            .expect_err("the initial V1 start preflight must reject before side effects");
19285        assert!(matches!(err, Error::InvalidArgument(..)));
19286        assert!(
19287            !start_probe.preflight_side_effect_was_attempted(),
19288            "V1 rejection must precede bucket listing, healing, and metadata-bucket creation"
19289        );
19290        assert!(
19291            !start_probe.activation_was_attempted(),
19292            "V1 rejection must not enter the authoritative activation save"
19293        );
19294        let after_early_rejection = load_pool_meta_replicas(store.pools.clone(), true)
19295            .await
19296            .expect("early rejection must leave durable pool metadata readable");
19297        assert_eq!(after_early_rejection.canonical, baseline.canonical);
19298        assert!(
19299            store
19300                .pool_meta
19301                .read()
19302                .await
19303                .pools
19304                .iter()
19305                .all(|pool| pool.decommission.is_none())
19306        );
19307        assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
19308        store
19309            .ensure_pool_meta_side_effects_safe("V1 start preflight")
19310            .await
19311            .expect("a deterministic start rejection must not latch recovery");
19312        drop(start_probe);
19313
19314        let err = store
19315            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19316            .await
19317            .expect_err("the authoritative V1 start preflight must reject before saving");
19318        assert!(matches!(err, Error::InvalidArgument(..)));
19319
19320        let after_authoritative_rejection = load_pool_meta_replicas(store.pools.clone(), true)
19321            .await
19322            .expect("authoritative rejection must leave durable pool metadata readable");
19323        assert_eq!(after_authoritative_rejection.canonical, baseline.canonical);
19324        assert!(
19325            after_authoritative_rejection
19326                .meta
19327                .pools
19328                .iter()
19329                .all(|pool| pool.decommission.is_none())
19330        );
19331        assert!(
19332            store
19333                .pool_meta
19334                .read()
19335                .await
19336                .pools
19337                .iter()
19338                .all(|pool| pool.decommission.is_none())
19339        );
19340        assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
19341        store
19342            .ensure_pool_meta_side_effects_safe("authoritative V1 start preflight")
19343            .await
19344            .expect("an authoritative capability rejection must not latch recovery");
19345    }
19346
19347    #[tokio::test]
19348    #[serial_test::serial]
19349    async fn decommission_activation_fence_loss_after_durable_save_blocks_publication() {
19350        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
19351        crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
19352        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19353        set_decommission_capacity_info_overrides_for_test(
19354            store.id,
19355            vec![vec![
19356                DecommissionPoolCapacityInfo::for_test(0, layout, 60, 60, 0),
19357                DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100),
19358            ]],
19359        );
19360        let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
19361        let start_store = Arc::clone(&store);
19362        let mut start_task = tokio::spawn(async move {
19363            start_store
19364                .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19365                .await
19366        });
19367
19368        tokio::select! {
19369            result = &mut start_task => panic!("activation finished before the durable-save barrier: {result:?}"),
19370            () = barrier.wait_until_paused() => {}
19371        }
19372        barrier.release_after_fence_loss();
19373        let err = tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
19374            .await
19375            .expect("decommission activation should stop after losing its fence")
19376            .expect("decommission activation task should not panic")
19377            .expect_err("post-durable-save fence loss must reject in-memory publication");
19378        assert!(err.to_string().contains("activation lock lost"));
19379
19380        let local = store.pool_meta.read().await;
19381        assert!(
19382            !pool_meta_has_active_decommission(&local),
19383            "the activation must not publish after its durable fence is lost"
19384        );
19385        drop(local);
19386        store
19387            .ensure_pool_meta_side_effects_safe("post-durable-save activation fence loss")
19388            .await
19389            .expect_err("the undisarmed transaction must latch the sticky pool metadata gate");
19390
19391        let mut persisted = PoolMeta::default();
19392        persisted
19393            .load_no_lock_from_replicas(vec![store.pools[0].clone()])
19394            .await
19395            .expect("the durable replica should remain readable for recovery");
19396        assert!(
19397            pool_meta_has_active_decommission(&persisted),
19398            "the test must lose the fence only after one durable replica commit"
19399        );
19400    }
19401
19402    #[tokio::test]
19403    #[serial_test::serial]
19404    async fn decommission_activation_adopts_canonical_commit_after_replica_failure() {
19405        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
19406        crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
19407        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19408        set_decommission_capacity_info_overrides_for_test(
19409            store.id,
19410            vec![vec![
19411                DecommissionPoolCapacityInfo::for_test(0, layout, 60, 60, 0),
19412                DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100),
19413            ]],
19414        );
19415        let barrier = PoolActivationDurableSaveBarrier::install(&store.pools[0]);
19416        let start_store = Arc::clone(&store);
19417        let mut start_task = tokio::spawn(async move {
19418            start_store
19419                .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19420                .await
19421        });
19422
19423        tokio::select! {
19424            result = &mut start_task => panic!("activation finished before the durable-save barrier: {result:?}"),
19425            () = barrier.wait_until_paused() => {}
19426        }
19427        let mut replica_disks = Vec::new();
19428        for set in &store.pools[1].disk_set {
19429            let mut disks = set.disks.write().await;
19430            let saved = std::mem::take(&mut *disks);
19431            *disks = vec![None; saved.len()];
19432            replica_disks.push(saved);
19433        }
19434        barrier.release_without_fence_loss();
19435
19436        tokio::time::timeout(std::time::Duration::from_secs(30), start_task)
19437            .await
19438            .expect("decommission activation should finish after its canonical commit")
19439            .expect("decommission activation task should not panic")
19440            .expect("a replica save failure must not report the committed activation as failed");
19441
19442        for (set, disks) in store.pools[1].disk_set.iter().zip(replica_disks) {
19443            *set.disks.write().await = disks;
19444        }
19445
19446        let local = store.pool_meta.read().await;
19447        assert!(pool_meta_has_active_decommission(&local));
19448        drop(local);
19449
19450        let mut canonical = PoolMeta::default();
19451        canonical
19452            .load_no_lock_from_replicas(vec![store.pools[0].clone()])
19453            .await
19454            .expect("the canonical committed decommission metadata should remain readable");
19455        assert!(pool_meta_has_active_decommission(&canonical));
19456
19457        let mut replica = PoolMeta::default();
19458        replica
19459            .load_no_lock_from_replicas(vec![store.pools[1].clone()])
19460            .await
19461            .expect("the stale replica metadata should remain readable after disks recover");
19462        assert!(!pool_meta_has_active_decommission(&replica));
19463
19464        let mut reloaded = PoolMeta::default();
19465        let replica_state = reloaded
19466            .load_no_lock_from_replicas(store.pools.clone())
19467            .await
19468            .expect("the canonical commit should remain reloadable after a replica recovers stale");
19469        assert!(pool_meta_has_active_decommission(&reloaded));
19470        assert!(replica_state.needs_repair);
19471        assert!(replica_state.repair_write_safe);
19472
19473        let worker_cancel = CancellationToken::new();
19474        let index_cancelers = store
19475            .reserve_decommission_routines(&worker_cancel, &[0])
19476            .await
19477            .expect("the committed activation should admit its decommission worker");
19478        drop(spawn_decommission_index_cancelers(
19479            Arc::clone(&store),
19480            worker_cancel.clone(),
19481            index_cancelers,
19482            Arc::new(Semaphore::new(decommission_entry_concurrency_limit())),
19483        ));
19484        let admitted_cancel = store.decommission_cancelers.read().await[0]
19485            .clone()
19486            .expect("the admitted decommission worker should have a cancellation token");
19487        assert!(!admitted_cancel.is_cancelled());
19488        worker_cancel.cancel();
19489        assert!(admitted_cancel.is_cancelled());
19490    }
19491
19492    #[tokio::test]
19493    #[serial_test::serial(pool_meta_version_env)]
19494    async fn decommission_capacity_reservation_serializes_two_nodes_without_overselling() {
19495        let (_temp_dirs, first_node, second_node) =
19496            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19497        persist_v3_pool_meta_for_test(&first_node).await;
19498        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19499        let capacity_snapshot = vec![
19500            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
19501            DecommissionPoolCapacityInfo::for_test(1, layout, 0, 30, 30),
19502            DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0),
19503        ];
19504        set_decommission_capacity_info_overrides_for_test(first_node.id, vec![capacity_snapshot.clone()]);
19505        set_decommission_capacity_info_overrides_for_test(second_node.id, vec![capacity_snapshot]);
19506
19507        let barrier =
19508            PutObjectCommitBarrier::install(RUSTFS_META_BUCKET, POOL_META_NAME, PutObjectCommitPause::BeforeQuotaRename);
19509        let first_store = Arc::clone(&first_node);
19510        let first = tokio::spawn(async move {
19511            first_store
19512                .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19513                .await
19514        });
19515        barrier.wait_until_paused().await;
19516
19517        let second_probe = PoolActivationStartProbe::install(PoolActivationStartKind::Decommission);
19518        let second_store = Arc::clone(&second_node);
19519        let second = tokio::spawn(async move {
19520            second_store
19521                .save_current_pool_meta_for_decommission_start(&[1], Vec::new())
19522                .await
19523        });
19524        second_probe.wait_until_attempted().await;
19525        assert!(
19526            !second.is_finished(),
19527            "the competing node must wait behind the distributed activation fence"
19528        );
19529
19530        drop(barrier);
19531        first
19532            .await
19533            .expect("first-node activation should not panic")
19534            .expect("first-node reservation should fit");
19535        let mut started_v3 = PoolMeta::default();
19536        started_v3
19537            .load_no_lock_from_replicas(first_node.pools.clone())
19538            .await
19539            .expect("the started capacity reservation should reload from V3 replicas");
19540        assert_eq!(started_v3.version, POOL_META_GENERATION_VERSION);
19541        let err = second
19542            .await
19543            .expect("second-node activation should not panic")
19544            .expect_err("the second reservation must observe and reject the committed first reservation");
19545        assert!(
19546            err.to_string().contains("requires 60 bytes, but 40 bytes are available"),
19547            "the second reservation must report the committed capacity rejection, got {err:?}"
19548        );
19549
19550        let mut persisted = PoolMeta::default();
19551        persisted
19552            .load_no_lock_from_replicas(first_node.pools.clone())
19553            .await
19554            .expect("the winning capacity reservation should remain readable");
19555        let reservation = persisted.pools[0]
19556            .decommission
19557            .as_ref()
19558            .and_then(|info| info.capacity_reservation.as_ref())
19559            .expect("the winning operation must retain its durable reservation");
19560        assert_eq!(
19561            reservation.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
19562            "an all-v4 proof must persist the per-target lock model through V3 replicas"
19563        );
19564        assert_eq!(reservation.peak_physical_bytes, 60);
19565        assert!(persisted.pools[1].decommission.is_none());
19566    }
19567
19568    #[tokio::test]
19569    #[serial_test::serial]
19570    async fn decommission_start_without_dedicated_v4_proof_persists_the_legacy_lock_model() {
19571        let (_temp_dirs, store, _other_store) =
19572            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19573        crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
19574        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19575        set_decommission_capacity_info_overrides_for_test(
19576            store.id,
19577            vec![vec![
19578                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
19579                DecommissionPoolCapacityInfo::for_test(1, layout, 0, 10, 10),
19580                DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
19581            ]],
19582        );
19583        let _proof_guard = crate::services::notification_sys::without_decommission_target_fence_fleet_proof_for_test();
19584
19585        store
19586            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19587            .await
19588            .expect("a mixed-version fleet should retain the compatible global lock model");
19589        let reservation = store.pool_meta.read().await.pools[0]
19590            .decommission
19591            .as_ref()
19592            .and_then(|info| info.capacity_reservation.as_ref())
19593            .expect("proofless start should persist a capacity reservation")
19594            .clone();
19595        assert_eq!(reservation.model_version, DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION);
19596    }
19597
19598    #[tokio::test]
19599    #[serial_test::serial]
19600    async fn decommission_cancel_replans_target_fences_after_a_concurrent_v2_start() {
19601        let (_temp_dirs, store, _other_store) =
19602            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19603        crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
19604        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19605        set_decommission_capacity_info_overrides_for_test(
19606            store.id,
19607            vec![vec![
19608                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
19609                DecommissionPoolCapacityInfo::for_test(1, layout, 0, 10, 10),
19610                DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
19611            ]],
19612        );
19613        let barrier = super::DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19614        barrier.pause_cancel_before_start();
19615        let cancel_store = Arc::clone(&store);
19616        let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
19617        barrier.wait_until_cancel_before_start().await;
19618
19619        store
19620            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19621            .await
19622            .expect("the concurrent start should persist a v2 reservation");
19623        let target_pool_index = store.pool_meta.read().await.pools[0]
19624            .decommission
19625            .as_ref()
19626            .and_then(|info| info.capacity_reservation.as_ref())
19627            .map(|reservation| {
19628                assert_eq!(reservation.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION);
19629                reservation.targets[0].pool_index
19630            })
19631            .expect("the concurrent start should publish its target cohort");
19632        let target_lock = store.pools[0]
19633            .new_ns_lock(
19634                RUSTFS_META_BUCKET,
19635                &format!("{}/{target_pool_index}", super::DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX),
19636            )
19637            .await
19638            .expect("create the cancel/start target-fence probe");
19639        let target_guard = target_lock
19640            .get_write_lock(std::time::Duration::from_secs(30))
19641            .await
19642            .expect("hold the newly started target fence");
19643
19644        barrier.release_cancel_before_start();
19645        tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel)
19646            .await
19647            .expect_err("cancel must wait for the target fence selected by the concurrent start");
19648        assert!(
19649            store.pool_meta.read().await.pools[0]
19650                .decommission
19651                .as_ref()
19652                .and_then(|info| info.capacity_reservation.as_ref())
19653                .is_some_and(DecommissionCapacityReservation::active),
19654            "cancel must not release the new reservation before fencing its target"
19655        );
19656
19657        drop(target_guard);
19658        tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
19659            .await
19660            .expect("cancel should finish after the target fence is released")
19661            .expect("cancel task should not panic")
19662            .expect("cancel should publish the terminal state");
19663        drop(barrier);
19664
19665        let pool_meta = store.pool_meta.read().await;
19666        let info = pool_meta.pools[0]
19667            .decommission
19668            .as_ref()
19669            .expect("canceled decommission metadata should remain present");
19670        assert!(info.canceled);
19671        let reservation = info
19672            .capacity_reservation
19673            .as_ref()
19674            .expect("canceled capacity accounting should remain inspectable");
19675        assert!(!reservation.active());
19676        assert_eq!(reservation.pending_target_physical_bytes, 0);
19677        assert_eq!(reservation.inflight_target_physical_bytes, 0);
19678    }
19679
19680    async fn start_target_fenced_cancel_test(
19681        store: &Arc<ECStore>,
19682        first_target_free: usize,
19683    ) -> (DecommissionCapacityTerminalFencePlan, DecommissionCanceler) {
19684        crate::services::rebalance::promote_test_pool_meta_to_v2(store).await;
19685        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
19686        set_decommission_capacity_info_overrides_for_test(
19687            store.id,
19688            vec![vec![
19689                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
19690                DecommissionPoolCapacityInfo::for_test(1, layout, first_target_free, 10, 10 - first_target_free),
19691                DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
19692            ]],
19693        );
19694        store
19695            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
19696            .await
19697            .expect("persist an active target-fenced decommission");
19698        let plan = decommission_capacity_terminal_fence_plan(&*store.pool_meta.read().await, 0)
19699            .expect("active decommission should have a valid fence plan")
19700            .expect("active decommission should retain its reservation");
19701        assert_eq!(plan.model_version, DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION);
19702        let canceler = DecommissionCanceler::new(CancellationToken::new());
19703        *store.decommission_cancelers.write().await = vec![Some(canceler.clone()), None, None];
19704        (plan, canceler)
19705    }
19706
19707    #[tokio::test]
19708    #[serial_test::serial]
19709    async fn decommission_cancel_waits_for_target_contention_past_one_lock_timeout() {
19710        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("5"))], async {
19711            let (_temp_dirs, store, _other_store) =
19712                crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19713            let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
19714            assert_eq!(plan.target_pool_indices, vec![2]);
19715            let target_lock = store.pools[0]
19716                .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
19717                .await
19718                .expect("create the active migration target gate");
19719            let target_guard = target_lock
19720                .get_write_lock(std::time::Duration::from_secs(30))
19721                .await
19722                .expect("hold the target gate across the first cancel acquisition timeout");
19723            let movement_gate = store.ctx.data_movement_operation_gate();
19724            let movement_guard = movement_gate.read().await;
19725            let cancel_store = Arc::clone(&store);
19726            let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
19727
19728            tokio::time::timeout(get_lock_acquire_timeout() + std::time::Duration::from_secs(1), &mut cancel)
19729                .await
19730                .expect_err("one target-lock timeout must not end a legitimate cancel while its retry budget remains");
19731            assert!(!canceler.is_cancelled(), "cancel must not signal its worker before durable fencing");
19732            let mut durable = PoolMeta::default();
19733            durable
19734                .load_no_lock_from_replicas(store.pools.clone())
19735                .await
19736                .expect("the active reservation must remain readable while cancel waits");
19737            assert_eq!(
19738                decommission_capacity_terminal_fence_plan(&durable, 0).expect("read the durable active plan"),
19739                Some(plan),
19740                "a failed target acquisition must not release or replace the durable reservation"
19741            );
19742            assert!(!durable.pools[0].decommission.as_ref().expect("active decommission").canceled);
19743
19744            drop(target_guard);
19745            tokio::time::timeout(std::time::Duration::from_secs(30), canceler.token().cancelled())
19746                .await
19747                .expect("cancel should persist and signal its worker after target contention is released");
19748            durable
19749                .load_no_lock_from_replicas(store.pools.clone())
19750                .await
19751                .expect("reload the committed cancellation from native replicas");
19752            let info = durable.pools[0]
19753                .decommission
19754                .as_ref()
19755                .expect("canceled state must remain durable");
19756            assert!(info.canceled);
19757            assert!(!info.complete && !info.failed);
19758            let reservation = info
19759                .capacity_reservation
19760                .as_ref()
19761                .expect("terminal capacity accounting must remain inspectable");
19762            assert!(!reservation.active());
19763            assert_eq!(reservation.pending_target_physical_bytes, 0);
19764            assert_eq!(reservation.inflight_target_physical_bytes, 0);
19765            tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel)
19766                .await
19767                .expect_err("the runtime-fenced cancel must wait for in-flight movement after signaling");
19768            drop(movement_guard);
19769            tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
19770                .await
19771                .expect("cancel should return after in-flight movement quiesces")
19772                .expect("cancel task should not panic")
19773                .expect("the same cancel request should finish its durable terminal transition");
19774        })
19775        .await;
19776    }
19777
19778    #[tokio::test]
19779    #[serial_test::serial]
19780    async fn decommission_cancel_target_contention_exhausts_bounded_attempts_without_committing() {
19781        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
19782            let (_temp_dirs, store, _other_store) =
19783                crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19784            let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
19785            let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19786            let target_lock = store.pools[0]
19787                .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
19788                .await
19789                .expect("create the persistently contended target gate");
19790            let target_guard = target_lock
19791                .get_write_lock(std::time::Duration::from_secs(30))
19792                .await
19793                .expect("hold the target gate for every cancel attempt");
19794
19795            let err = tokio::time::timeout(std::time::Duration::from_secs(30), store.decommission_cancel(0))
19796                .await
19797                .expect("target contention must not retry indefinitely")
19798                .expect_err("exhausting the acquisition budget must not report cancellation success");
19799            match err {
19800                Error::Lock(rustfs_lock::LockError::Timeout { resource, timeout }) => {
19801                    assert_eq!(resource, ".rustfs.sys/decommission/capacity-target/2@latest");
19802                    assert_eq!(timeout, std::time::Duration::from_secs(1));
19803                }
19804                other => panic!("expected the final typed target-lock timeout, got {other:?}"),
19805            }
19806            assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 3);
19807            assert!(!canceler.is_cancelled());
19808            assert!(canceler.is_active());
19809            let mut durable = PoolMeta::default();
19810            durable
19811                .load_no_lock_from_replicas(store.pools.clone())
19812                .await
19813                .expect("reload the reservation after the canceled request exhausted its budget");
19814            assert_eq!(
19815                decommission_capacity_terminal_fence_plan(&durable, 0).expect("the active plan should remain valid"),
19816                Some(plan)
19817            );
19818            assert!(!durable.pools[0].decommission.as_ref().expect("active decommission").canceled);
19819            drop(target_guard);
19820        })
19821        .await;
19822    }
19823
19824    #[tokio::test]
19825    #[serial_test::serial]
19826    async fn decommission_cancel_target_retry_releases_partial_cohort_and_rejects_remote_replacement() {
19827        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
19828            let (_temp_dirs, store, other_store) =
19829                crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19830            let (plan, _old_canceler) = start_target_fenced_cancel_test(&store, 8).await;
19831            assert_eq!(plan.target_pool_indices, vec![1, 2]);
19832            let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19833            barrier.pause_cancel_target_timeout();
19834            let target_lock = store.pools[0]
19835                .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
19836                .await
19837                .expect("create the second target gate");
19838            let target_guard = target_lock
19839                .get_write_lock(std::time::Duration::from_secs(30))
19840                .await
19841                .expect("block cancellation after it acquires the first target");
19842            let cancel_store = Arc::clone(&store);
19843            let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
19844            barrier.wait_until_cancel_target_timeout().await;
19845
19846            let first_target_lock = store.pools[0]
19847                .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/1"))
19848                .await
19849                .expect("create the partial-cohort release probe");
19850            let first_target_guard = first_target_lock
19851                .get_write_lock(std::time::Duration::from_secs(1))
19852                .await
19853                .expect("cancel must release its first target before retrying the blocked second target");
19854            drop(first_target_guard);
19855            drop(target_guard);
19856
19857            other_store
19858                .reload_pool_meta()
19859                .await
19860                .expect("load the active generation on the remote node");
19861            tokio::time::timeout(std::time::Duration::from_secs(30), other_store.decommission_cancel(0))
19862                .await
19863                .expect("remote cancel must persist pool metadata without re-entering its state locks")
19864                .expect("the remote node should cancel the old generation");
19865            other_store
19866                .clear_decommission(0)
19867                .await
19868                .expect("the remote node should clear the old generation");
19869            let (replacement_plan, replacement_canceler) = start_target_fenced_cancel_test(&other_store, 8).await;
19870            assert_ne!(replacement_plan.operation_id, plan.operation_id);
19871
19872            barrier.release_cancel_target_timeout();
19873            let err = tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
19874                .await
19875                .expect("the stale request must finish without retrying a replacement operation")
19876                .expect("the stale cancel task should not panic")
19877                .expect_err("the original request must not cancel a remotely replaced generation");
19878            assert!(err.to_string().contains("changed while retrying terminal fences"));
19879            assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
19880            assert!(!replacement_canceler.is_cancelled());
19881            let mut durable = PoolMeta::default();
19882            durable
19883                .load_no_lock_from_replicas(store.pools.clone())
19884                .await
19885                .expect("reload the replacement after the stale cancel returns");
19886            assert_eq!(
19887                decommission_capacity_terminal_fence_plan(&durable, 0).expect("the replacement must retain its valid plan"),
19888                Some(replacement_plan)
19889            );
19890            assert!(
19891                !durable.pools[0]
19892                    .decommission
19893                    .as_ref()
19894                    .expect("replacement decommission")
19895                    .canceled
19896            );
19897        })
19898        .await;
19899    }
19900
19901    #[tokio::test]
19902    #[serial_test::serial]
19903    async fn decommission_cancel_target_retry_survives_caller_abort_and_settles_inflight_mutation() {
19904        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
19905            let (_temp_dirs, store, _other_store) =
19906                crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19907            let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
19908            let owner = DecommissionCapacityOwner {
19909                source_pool_index: 0,
19910                operation_id: plan.operation_id,
19911                generation: plan.generation,
19912                owner_nonce: plan.owner_nonce,
19913                mutation_id: Some(uuid::Uuid::new_v4()),
19914            };
19915            let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
19916            let (release_tx, release_rx) = tokio::sync::oneshot::channel();
19917            let mutation_store = Arc::clone(&store);
19918            let mutation = tokio::spawn(async move {
19919                mutation_store
19920                    .run_decommission_capacity_admitted_mutation(2, Some(owner), Some(1), || async {
19921                        entered_tx
19922                            .send(())
19923                            .expect("the test should observe the admitted target mutation");
19924                        release_rx.await.expect("the test should release the in-flight mutation");
19925                        Ok(())
19926                    })
19927                    .await
19928            });
19929            tokio::time::timeout(std::time::Duration::from_secs(30), entered_rx)
19930                .await
19931                .expect("target mutation should reach its controlled I/O phase")
19932                .expect("target admission should succeed before cancel starts");
19933            let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19934            let cancel_store = Arc::clone(&store);
19935            let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
19936            barrier.wait_until_cancel_target_timeout().await;
19937            assert!(!canceler.is_cancelled(), "in-flight capacity settlement must precede the cancel signal");
19938            cancel.abort();
19939            assert!(cancel.await.expect_err("the RPC waiter should be aborted").is_cancelled());
19940
19941            release_tx
19942                .send(())
19943                .expect("the mutation must still be alive after the caller disconnects");
19944            tokio::time::timeout(std::time::Duration::from_secs(30), mutation)
19945                .await
19946                .expect("the in-flight mutation must be able to settle without a metadata lock cycle")
19947                .expect("the mutation task should not panic")
19948                .expect("the admitted mutation must settle before the target gate is released");
19949            tokio::time::timeout(std::time::Duration::from_secs(30), async {
19950                loop {
19951                    if store.pool_meta.read().await.pools[0]
19952                        .decommission
19953                        .as_ref()
19954                        .is_some_and(|info| info.canceled)
19955                    {
19956                        break;
19957                    }
19958                    tokio::time::sleep(std::time::Duration::from_millis(20)).await;
19959                }
19960            })
19961            .await
19962            .expect("the detached cancel transaction must finish after target contention clears");
19963            assert!(canceler.is_cancelled());
19964            let mut durable = PoolMeta::default();
19965            durable
19966                .load_no_lock_from_replicas(store.pools.clone())
19967                .await
19968                .expect("reload cancellation after the RPC waiter was dropped");
19969            let info = durable.pools[0].decommission.as_ref().expect("durable canceled decommission");
19970            assert!(info.canceled && !info.failed && !info.complete);
19971            let reservation = info
19972                .capacity_reservation
19973                .as_ref()
19974                .expect("inspect settled capacity accounting");
19975            assert!(!reservation.active());
19976            assert_eq!(reservation.pending_target_physical_bytes, 0);
19977            assert_eq!(reservation.inflight_target_physical_bytes, 0);
19978        })
19979        .await;
19980    }
19981
19982    #[tokio::test]
19983    #[serial_test::serial]
19984    async fn decommission_cancel_target_retry_does_not_replay_a_failed_durable_save() {
19985        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
19986            let (_temp_dirs, store, _other_store) =
19987                crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
19988            let (plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
19989            let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
19990            barrier.pause_cancel_target_timeout();
19991            let target_lock = store.pools[0]
19992                .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
19993                .await
19994                .expect("create the target gate preceding the failed save");
19995            let target_guard = target_lock
19996                .get_write_lock(std::time::Duration::from_secs(30))
19997                .await
19998                .expect("force one target acquisition retry before persistence");
19999            let save_calls = Arc::new(AtomicUsize::new(0));
20000            let calls = Arc::clone(&save_calls);
20001            let cancel_store = Arc::clone(&store);
20002            let owner = canceler.clone();
20003            let cancel = tokio::spawn(async move {
20004                cancel_store
20005                    .decommission_cancel_transaction(0, Some(owner), true, move |_, _| async move {
20006                        calls.fetch_add(1, Ordering::AcqRel);
20007                        Err(Error::Timeout)
20008                    })
20009                    .await
20010            });
20011            barrier.wait_until_cancel_target_timeout().await;
20012            assert_eq!(save_calls.load(Ordering::Acquire), 0, "persistence must wait for every target fence");
20013            drop(target_guard);
20014            barrier.release_cancel_target_timeout();
20015            let err = tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
20016                .await
20017                .expect("a persistence failure must end the cancellation transaction")
20018                .expect("the failed-save task should not panic")
20019                .expect_err("the injected durable-save failure must reach the caller");
20020            assert!(matches!(err, Error::Timeout));
20021            assert_eq!(save_calls.load(Ordering::Acquire), 1);
20022            assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
20023            assert!(!canceler.is_cancelled());
20024            assert!(canceler.is_active());
20025            store
20026                .ensure_pool_meta_side_effects_safe("verify ambiguous cancellation save blocks further writes")
20027                .await
20028                .expect_err("a failed durable save must retain the existing recovery gate");
20029            let mut durable = PoolMeta::default();
20030            durable
20031                .load_no_lock_from_replicas(store.pools.clone())
20032                .await
20033                .expect("the original reservation should remain readable after the injected save failure");
20034            assert_eq!(
20035                decommission_capacity_terminal_fence_plan(&durable, 0).expect("the old reservation should remain valid"),
20036                Some(plan)
20037            );
20038        })
20039        .await;
20040    }
20041
20042    #[tokio::test]
20043    #[serial_test::serial]
20044    async fn decommission_cancel_target_retry_converges_after_successive_contenders() {
20045        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
20046            let (_temp_dirs, store, _other_store) =
20047                crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
20048            let (_plan, canceler) = start_target_fenced_cancel_test(&store, 0).await;
20049            let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
20050            barrier.pause_cancel_target_timeout();
20051            let target_lock = store.pools[0]
20052                .new_ns_lock(RUSTFS_META_BUCKET, &format!("{DECOMMISSION_CAPACITY_TARGET_LOCK_PREFIX}/2"))
20053                .await
20054                .expect("create the repeatedly contended target gate");
20055            let first_guard = target_lock
20056                .get_write_lock(std::time::Duration::from_secs(30))
20057                .await
20058                .expect("the first contender should own the target gate");
20059            let cancel_store = Arc::clone(&store);
20060            let cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
20061            barrier.wait_until_cancel_target_timeout().await;
20062            assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 1);
20063            drop(first_guard);
20064            let second_guard = target_lock
20065                .get_write_lock(std::time::Duration::from_secs(30))
20066                .await
20067                .expect("a second contender should be able to acquire between cancel attempts");
20068            barrier.release_cancel_target_timeout();
20069            barrier.pause_cancel_target_timeout();
20070            barrier.wait_until_cancel_target_timeout().await;
20071            assert_eq!(barrier.state.cancel_target_timeouts.load(Ordering::Acquire), 2);
20072            assert!(!canceler.is_cancelled());
20073            drop(second_guard);
20074            barrier.release_cancel_target_timeout();
20075
20076            tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
20077                .await
20078                .expect("cancel should converge after the repeated contention ends")
20079                .expect("the cancel task should not panic")
20080                .expect("the third target acquisition should permit a durable cancellation");
20081            assert!(canceler.is_cancelled());
20082            let mut durable = PoolMeta::default();
20083            durable
20084                .load_no_lock_from_replicas(store.pools.clone())
20085                .await
20086                .expect("the successful retried cancellation must survive a native metadata reload");
20087            assert!(
20088                durable.pools[0]
20089                    .decommission
20090                    .as_ref()
20091                    .expect("canceled decommission")
20092                    .canceled
20093            );
20094            assert!(
20095                decommission_capacity_terminal_fence_plan(&durable, 0)
20096                    .expect("valid terminal metadata")
20097                    .is_none()
20098            );
20099        })
20100        .await;
20101    }
20102
20103    #[tokio::test]
20104    #[serial_test::serial]
20105    async fn stale_node_cancel_cannot_replace_a_new_durable_v2_operation() {
20106        let (_temp_dirs, first_node, stale_node) =
20107            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
20108        crate::services::rebalance::promote_test_pool_meta_to_v2(&first_node).await;
20109        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20110        set_decommission_capacity_info_overrides_for_test(
20111            first_node.id,
20112            vec![vec![
20113                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
20114                DecommissionPoolCapacityInfo::for_test(1, layout, 0, 10, 10),
20115                DecommissionPoolCapacityInfo::for_test(2, layout, 40, 40, 0),
20116            ]],
20117        );
20118
20119        first_node
20120            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20121            .await
20122            .expect("start the operation retained by the stale node");
20123        let stale_operation = first_node.pool_meta.read().await.clone();
20124        let stale_plan = decommission_capacity_terminal_fence_plan(&stale_operation, 0)
20125            .expect("the stale operation should have a valid terminal fence plan")
20126            .expect("the stale operation should have an active reservation");
20127
20128        tokio::time::timeout(std::time::Duration::from_secs(30), first_node.decommission_cancel(0))
20129            .await
20130            .expect("cancel must persist pool metadata without re-entering its state locks")
20131            .expect("cancel the first durable operation");
20132        first_node
20133            .clear_decommission(0)
20134            .await
20135            .expect("clear the first terminal operation");
20136        first_node
20137            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20138            .await
20139            .expect("start the replacement durable operation");
20140        let replacement_plan = {
20141            let pool_meta = first_node.pool_meta.read().await;
20142            decommission_capacity_terminal_fence_plan(&pool_meta, 0)
20143                .expect("the replacement operation should have a valid terminal fence plan")
20144                .expect("the replacement operation should have an active reservation")
20145        };
20146        assert_ne!(replacement_plan.operation_id, stale_plan.operation_id);
20147
20148        *stale_node.pool_meta.write().await = stale_operation;
20149        let err = stale_node
20150            .decommission_cancel(0)
20151            .await
20152            .expect_err("a stale local operation must not be merged over the durable replacement");
20153        assert!(err.to_string().contains("differs from the durable terminal fence plan"));
20154
20155        let mut durable = PoolMeta::default();
20156        durable
20157            .load_no_lock_from_replicas(first_node.pools.clone())
20158            .await
20159            .expect("the replacement operation should remain readable from durable replicas");
20160        assert_eq!(
20161            decommission_capacity_terminal_fence_plan(&durable, 0)
20162                .expect("the durable replacement should retain a valid terminal fence plan")
20163                .expect("the durable replacement reservation should remain active"),
20164            replacement_plan
20165        );
20166        let replacement = durable.pools[0]
20167            .decommission
20168            .as_ref()
20169            .expect("the durable replacement metadata should remain present");
20170        assert!(!replacement.canceled);
20171        assert!(
20172            replacement
20173                .capacity_reservation
20174                .as_ref()
20175                .is_some_and(DecommissionCapacityReservation::active)
20176        );
20177    }
20178
20179    #[tokio::test]
20180    #[serial_test::serial]
20181    async fn decommission_capacity_publication_allows_immediate_bucket_completion() {
20182        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
20183        crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
20184        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20185        let capacity = vec![
20186            DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80),
20187            DecommissionPoolCapacityInfo::for_test(1, layout, 200, 200, 0),
20188        ];
20189        set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity.clone(), capacity.clone(), capacity]);
20190        let bucket = DecomBucketInfo {
20191            name: "capacity-publication-bucket".to_string(),
20192            prefix: String::new(),
20193        };
20194        store
20195            .save_current_pool_meta_for_decommission_start(&[0], vec![bucket.clone()])
20196            .await
20197            .expect("activate a reservation and queue the regression bucket");
20198        let (owner, initial_update, generation) = {
20199            let mut meta = store.pool_meta.write().await;
20200            meta.count_item(0, 17, false);
20201            let reservation = meta.pools[0]
20202                .decommission
20203                .as_ref()
20204                .and_then(|info| info.capacity_reservation.as_ref())
20205                .expect("the source must own an active reservation");
20206            (
20207                DecommissionCapacityOwner {
20208                    source_pool_index: 0,
20209                    operation_id: reservation.operation_id,
20210                    generation: reservation.generation,
20211                    owner_nonce: reservation.owner_nonce,
20212                    mutation_id: Some(uuid::Uuid::new_v4()),
20213                },
20214                meta.pools[0].last_update,
20215                meta.pools[0]
20216                    .decommission
20217                    .as_ref()
20218                    .and_then(|info| info.start_time)
20219                    .expect("active worker generation"),
20220            )
20221        };
20222        store
20223            .run_decommission_capacity_admitted_mutation(1, Some(owner), Some(17), || async {
20224                let meta = store.pool_meta.read().await;
20225                assert!(
20226                    meta.pools[0].last_update > initial_update,
20227                    "durable capacity intent must publish its revision before target I/O"
20228                );
20229                assert_eq!(
20230                    meta.pools[0]
20231                        .decommission
20232                        .as_ref()
20233                        .expect("active source")
20234                        .items_decommissioned,
20235                    1
20236                );
20237                Ok(())
20238            })
20239            .await
20240            .expect("capacity intent and consumption must commit without a progress checkpoint");
20241        let persisted = load_pool_meta_replicas(store.pools.clone(), true)
20242            .await
20243            .expect("reload capacity consumption before bucket completion");
20244        {
20245            let meta = store.pool_meta.read().await;
20246            assert_eq!(meta.pools[0].last_update, persisted.meta.pools[0].last_update);
20247            let info = meta.pools[0].decommission.as_ref().expect("active source");
20248            assert_eq!(info.items_decommissioned, 1, "capacity publication must retain uncheckpointed progress");
20249            assert_eq!(
20250                info.progress_save_item_baseline, 0,
20251                "capacity-only saves do not checkpoint object counters"
20252            );
20253        }
20254        assert!(
20255            store
20256                .mark_decommission_bucket_done_and_save(0, generation, &bucket)
20257                .await
20258                .expect("bucket completion immediately after capacity persistence must not reject its own revision")
20259        );
20260        let persisted = load_pool_meta_replicas(store.pools.clone(), true)
20261            .await
20262            .expect("bucket completion must survive a metadata reload");
20263        let info = persisted.meta.pools[0]
20264            .decommission
20265            .as_ref()
20266            .expect("durable source progress");
20267        assert!(info.queued_buckets.is_empty());
20268        assert_eq!(info.decommissioned_buckets, vec![bucket.to_string()]);
20269        assert_eq!((info.items_decommissioned, info.bytes_done), (1, 17));
20270        let reservation = info.capacity_reservation.as_ref().expect("durable capacity consumption");
20271        assert_eq!(reservation.pending_target_physical_bytes, 0);
20272        assert_eq!(reservation.consumed_target_physical_bytes, 17);
20273    }
20274
20275    #[tokio::test]
20276    #[serial_test::serial]
20277    async fn decommission_bucket_completion_publishes_only_committed_progress() {
20278        for fault_stage in [
20279            "bucket_completion_before_save",
20280            "bucket_completion_after_save",
20281            "count_during_save",
20282        ] {
20283            let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
20284            let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20285            set_decommission_capacity_info_overrides_for_test(
20286                store.id,
20287                vec![vec![
20288                    DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80),
20289                    DecommissionPoolCapacityInfo::for_test(1, layout, 200, 200, 0),
20290                ]],
20291            );
20292            let bucket = DecomBucketInfo {
20293                name: format!("bucket-completion-{fault_stage}"),
20294                prefix: String::new(),
20295            };
20296            store
20297                .save_current_pool_meta_for_decommission_start(&[0], vec![bucket.clone()])
20298                .await
20299                .expect("activate the bucket completion fixture");
20300            let generation = {
20301                let mut meta = store.pool_meta.write().await;
20302                meta.count_item(0, 17, false);
20303                meta.pools[0]
20304                    .decommission
20305                    .as_ref()
20306                    .and_then(|info| info.start_time)
20307                    .expect("active generation")
20308            };
20309            let hook_store = Arc::clone(&store);
20310            let observed = Arc::new(AtomicBool::new(false));
20311            let hook_observed = Arc::clone(&observed);
20312            let expected_bucket = bucket.name.clone();
20313            let fault = DecommissionTestFaultGuard::install(Arc::new(move |stage, name, _, _, success| {
20314                if name != expected_bucket || !success {
20315                    return false;
20316                }
20317                if stage == fault_stage {
20318                    hook_observed.store(true, Ordering::SeqCst);
20319                    return true;
20320                }
20321                if fault_stage == "count_during_save" && stage == "bucket_completion_after_save" {
20322                    hook_observed.store(true, Ordering::SeqCst);
20323                    hook_store
20324                        .pool_meta
20325                        .try_write()
20326                        .expect("publication must not hold the local metadata lock over I/O")
20327                        .count_item(0, 23, false);
20328                }
20329                false
20330            }));
20331            let result = store.mark_decommission_bucket_done_and_save(0, generation, &bucket).await;
20332            assert!(
20333                observed.load(Ordering::SeqCst),
20334                "{fault_stage}: the production save boundary was not reached"
20335            );
20336            drop(fault);
20337            let persisted = load_pool_meta_replicas(store.pools.clone(), true)
20338                .await
20339                .expect("durable metadata remains readable");
20340            let durable = persisted.meta.pools[0].decommission.as_ref().expect("durable progress");
20341            let local = store.pool_meta.read().await;
20342            let live = local.pools[0].decommission.as_ref().expect("live progress");
20343            if fault_stage == "count_during_save" {
20344                assert!(result.expect("successful bucket completion"));
20345                assert_eq!((live.items_decommissioned, live.bytes_done, live.progress_save_item_baseline), (2, 40, 1));
20346                assert_eq!((durable.items_decommissioned, durable.bytes_done), (1, 17));
20347                assert!(live.queued_buckets.is_empty());
20348                assert_eq!(live.decommissioned_buckets, vec![bucket.to_string()]);
20349            } else {
20350                let err = result.expect_err("injected failure must reach the caller");
20351                assert!(err.to_string().contains(fault_stage));
20352                assert_eq!(live.queued_buckets, vec![bucket.to_string()]);
20353                assert!(live.decommissioned_buckets.is_empty());
20354                assert_eq!(live.progress_save_item_baseline, 0);
20355                if fault_stage == "bucket_completion_before_save" {
20356                    assert_eq!(durable.queued_buckets, vec![bucket.to_string()]);
20357                    assert!(durable.decommissioned_buckets.is_empty());
20358                } else {
20359                    assert!(durable.queued_buckets.is_empty());
20360                    assert_eq!(durable.decommissioned_buckets, vec![bucket.to_string()]);
20361                    assert_eq!((durable.items_decommissioned, durable.bytes_done), (1, 17));
20362                }
20363            }
20364            drop(local);
20365            if fault_stage == "bucket_completion_after_save" {
20366                store
20367                    .ensure_pool_meta_side_effects_safe("ambiguous bucket completion")
20368                    .await
20369                    .expect_err("commit without publication must require recovery before another write");
20370            } else {
20371                store
20372                    .ensure_pool_meta_side_effects_safe("bucket completion")
20373                    .await
20374                    .expect("no ambiguous commit");
20375            }
20376        }
20377    }
20378
20379    #[tokio::test]
20380    #[serial_test::serial]
20381    async fn legacy_capacity_model_holds_the_global_pool_meta_fence_through_target_io() {
20382        let (_temp_dirs, store, _other_store) =
20383            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
20384        crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
20385        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20386        set_decommission_capacity_info_overrides_for_test(
20387            store.id,
20388            vec![vec![
20389                DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80),
20390                DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
20391                DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0),
20392            ]],
20393        );
20394        let _proof_guard = crate::services::notification_sys::without_decommission_target_fence_fleet_proof_for_test();
20395        store
20396            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20397            .await
20398            .expect("activate a legacy reservation spanning two targets");
20399        let base_owner = {
20400            let pool_meta = store.pool_meta.read().await;
20401            let reservation = pool_meta.pools[0]
20402                .decommission
20403                .as_ref()
20404                .and_then(|info| info.capacity_reservation.as_ref())
20405                .expect("legacy capacity reservation should exist");
20406            assert_eq!(reservation.model_version, DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION);
20407            assert_eq!(reservation.targets.iter().map(|target| target.pool_index).collect::<Vec<_>>(), vec![1, 2]);
20408            DecommissionCapacityOwner {
20409                source_pool_index: 0,
20410                operation_id: reservation.operation_id,
20411                generation: reservation.generation,
20412                owner_nonce: reservation.owner_nonce,
20413                mutation_id: None,
20414            }
20415        };
20416
20417        let (first_entered_tx, first_entered_rx) = tokio::sync::oneshot::channel();
20418        let (first_release_tx, first_release_rx) = tokio::sync::oneshot::channel();
20419        let first_store = Arc::clone(&store);
20420        let first_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
20421        let first = tokio::spawn(async move {
20422            first_store
20423                .run_decommission_capacity_admitted_mutation_with_capacity_lease(1, Some(first_owner), Some(1), |_| async {
20424                    first_entered_tx.send(()).expect("first legacy mutation should be observed");
20425                    first_release_rx.await.expect("first legacy mutation should be released");
20426                    Ok(())
20427                })
20428                .await
20429        });
20430        tokio::time::timeout(std::time::Duration::from_secs(30), first_entered_rx)
20431            .await
20432            .expect("first legacy mutation should enter")
20433            .expect("first legacy mutation should report entry");
20434        assert!(
20435            store.pool_meta_save_gate.try_lock().is_err(),
20436            "legacy target I/O must retain the local pool metadata save gate"
20437        );
20438        let pool_meta_lock = store.pools[0]
20439            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
20440            .await
20441            .expect("create the legacy pool metadata lock probe");
20442        assert!(
20443            pool_meta_lock
20444                .get_write_lock_quiet(std::time::Duration::from_millis(300))
20445                .await
20446                .is_err(),
20447            "legacy target I/O must retain the distributed pool metadata write fence"
20448        );
20449
20450        let (second_entered_tx, mut second_entered_rx) = tokio::sync::oneshot::channel();
20451        let (second_release_tx, second_release_rx) = tokio::sync::oneshot::channel();
20452        let second_store = Arc::clone(&store);
20453        let second_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
20454        let second = tokio::spawn(async move {
20455            second_store
20456                .run_decommission_capacity_admitted_mutation_with_capacity_lease(2, Some(second_owner), Some(1), |_| async {
20457                    second_entered_tx.send(()).expect("second legacy mutation should be observed");
20458                    second_release_rx.await.expect("second legacy mutation should be released");
20459                    Ok(())
20460                })
20461                .await
20462        });
20463        tokio::time::timeout(std::time::Duration::from_millis(500), &mut second_entered_rx)
20464            .await
20465            .expect_err("a different target must still wait behind the legacy global fence");
20466
20467        first_release_tx.send(()).expect("release first legacy mutation");
20468        first
20469            .await
20470            .expect("first legacy mutation task should not panic")
20471            .expect("first legacy mutation should finalize");
20472        tokio::time::timeout(std::time::Duration::from_secs(30), &mut second_entered_rx)
20473            .await
20474            .expect("second legacy mutation should enter after the first finalizes")
20475            .expect("second legacy mutation should report entry");
20476        assert!(
20477            pool_meta_lock
20478                .get_write_lock_quiet(std::time::Duration::from_millis(300))
20479                .await
20480                .is_err(),
20481            "the second legacy target I/O must also retain the global fence"
20482        );
20483        second_release_tx.send(()).expect("release second legacy mutation");
20484        second
20485            .await
20486            .expect("second legacy mutation task should not panic")
20487            .expect("second legacy mutation should finalize");
20488
20489        let pool_meta_guard = pool_meta_lock
20490            .get_write_lock(std::time::Duration::from_secs(30))
20491            .await
20492            .expect("the global pool metadata fence should release after both target tails");
20493        drop(pool_meta_guard);
20494        let pool_meta = store.pool_meta.read().await;
20495        let reservation = pool_meta.pools[0]
20496            .decommission
20497            .as_ref()
20498            .and_then(|info| info.capacity_reservation.as_ref())
20499            .expect("legacy reservation should remain active");
20500        assert_eq!(reservation.pending_target_physical_bytes, 0);
20501        assert_eq!(reservation.consumed_target_physical_bytes, 2);
20502    }
20503
20504    #[tokio::test]
20505    #[serial_test::serial]
20506    async fn decommission_capacity_mutations_overlap_across_targets_but_serialize_per_target() {
20507        let (_temp_dirs, store, _other_store) =
20508            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
20509        crate::services::rebalance::promote_test_pool_meta_to_v2(&store).await;
20510        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20511        let capacity_snapshot = vec![
20512            DecommissionPoolCapacityInfo::for_test(0, layout, 20, 100, 80),
20513            DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
20514            DecommissionPoolCapacityInfo::for_test(2, layout, 100, 100, 0),
20515        ];
20516        set_decommission_capacity_info_overrides_for_test(store.id, vec![capacity_snapshot]);
20517        store
20518            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20519            .await
20520            .expect("activate a reservation spanning two targets");
20521
20522        let base_owner = {
20523            let pool_meta = store.pool_meta.read().await;
20524            let reservation = pool_meta.pools[0]
20525                .decommission
20526                .as_ref()
20527                .and_then(|info| info.capacity_reservation.as_ref())
20528                .expect("active capacity reservation should exist");
20529            assert_eq!(
20530                reservation.targets.iter().map(|target| target.pool_index).collect::<Vec<_>>(),
20531                vec![1, 2],
20532                "the fixture must reserve both target pools"
20533            );
20534            DecommissionCapacityOwner {
20535                source_pool_index: 0,
20536                operation_id: reservation.operation_id,
20537                generation: reservation.generation,
20538                owner_nonce: reservation.owner_nonce,
20539                mutation_id: None,
20540            }
20541        };
20542
20543        let (first_entered_tx, first_entered_rx) = tokio::sync::oneshot::channel();
20544        let (first_release_tx, first_release_rx) = tokio::sync::oneshot::channel();
20545        let first_store = Arc::clone(&store);
20546        let first_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
20547        let first = tokio::spawn(async move {
20548            first_store
20549                .run_decommission_capacity_admitted_mutation_with_capacity_lease(1, Some(first_owner), Some(1), |_| async {
20550                    first_entered_tx.send(()).expect("first target mutation should be observed");
20551                    first_release_rx.await.expect("first target mutation should be released");
20552                    Ok(())
20553                })
20554                .await
20555        });
20556        tokio::time::timeout(std::time::Duration::from_secs(30), first_entered_rx)
20557            .await
20558            .expect("first target mutation should enter without hanging")
20559            .expect("first target mutation should report entry");
20560
20561        let (second_entered_tx, second_entered_rx) = tokio::sync::oneshot::channel();
20562        let (second_release_tx, second_release_rx) = tokio::sync::oneshot::channel();
20563        let second_store = Arc::clone(&store);
20564        let second_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
20565        let second = tokio::spawn(async move {
20566            second_store
20567                .run_decommission_capacity_admitted_mutation_with_capacity_lease(2, Some(second_owner), Some(1), |_| async {
20568                    second_entered_tx.send(()).expect("second target mutation should be observed");
20569                    second_release_rx.await.expect("second target mutation should be released");
20570                    Ok(())
20571                })
20572                .await
20573        });
20574        tokio::time::timeout(std::time::Duration::from_secs(30), second_entered_rx)
20575            .await
20576            .expect("a different target mutation should overlap instead of waiting for the first")
20577            .expect("second target mutation should report entry");
20578
20579        let save_guard = store
20580            .pool_meta_save_gate
20581            .try_lock()
20582            .expect("target I/O must not retain the local pool metadata save gate");
20583        drop(save_guard);
20584        let pool_meta_lock = store.pools[0]
20585            .new_ns_lock(RUSTFS_META_BUCKET, POOL_META_NAME)
20586            .await
20587            .expect("create a pool metadata lock probe");
20588        let pool_meta_guard = pool_meta_lock
20589            .get_write_lock_quiet(std::time::Duration::from_secs(1))
20590            .await
20591            .expect("target I/O must not retain the distributed pool metadata write lock");
20592        drop(pool_meta_guard);
20593
20594        let blocked_owner = base_owner.with_mutation_id(uuid::Uuid::new_v4());
20595        let blocked = store
20596            .run_decommission_capacity_admitted_mutation(1, Some(blocked_owner), Some(1), || async { Ok(()) })
20597            .await
20598            .expect_err("a second mutation on the same target must wait behind its target gate");
20599        assert!(is_decommission_capacity_blocked_error(&blocked));
20600        assert!(
20601            is_decommission_capacity_target_gate_busy(&blocked),
20602            "same-target serialization must remain distinguishable from durable intent conflicts"
20603        );
20604        assert!(
20605            decommission_capacity_retry_kind(&blocked, 0).is_none(),
20606            "target contention must be retried inside the current version instead of replaying the entry"
20607        );
20608
20609        let cancel_store = Arc::clone(&store);
20610        let mut cancel = tokio::spawn(async move { cancel_store.decommission_cancel(0).await });
20611        tokio::time::timeout(std::time::Duration::from_millis(500), &mut cancel)
20612            .await
20613            .expect_err("terminal reservation release must wait for prepared target mutations");
20614        assert!(
20615            store.pool_meta.read().await.pools[0]
20616                .decommission
20617                .as_ref()
20618                .and_then(|info| info.capacity_reservation.as_ref())
20619                .is_some_and(DecommissionCapacityReservation::active),
20620            "cancellation must not release the reservation while target mutations are active"
20621        );
20622
20623        first_release_tx.send(()).expect("release first target mutation");
20624        second_release_tx.send(()).expect("release second target mutation");
20625        first
20626            .await
20627            .expect("first target mutation task should not panic")
20628            .expect("first target mutation should finalize");
20629        second
20630            .await
20631            .expect("second target mutation task should not panic")
20632            .expect("second target mutation should finalize");
20633        tokio::time::timeout(std::time::Duration::from_secs(30), cancel)
20634            .await
20635            .expect("cancel must finish after target mutations release their gates")
20636            .expect("decommission cancellation task should not panic")
20637            .expect("decommission cancellation should finish after target mutations");
20638
20639        let pool_meta = store.pool_meta.read().await;
20640        let reservation = pool_meta.pools[0]
20641            .decommission
20642            .as_ref()
20643            .and_then(|info| info.capacity_reservation.as_ref())
20644            .expect("released capacity reservation should remain available for accounting");
20645        assert!(!reservation.active());
20646        assert_eq!(reservation.release_reason.as_deref(), Some(DECOMMISSION_CAPACITY_RELEASE_CANCELED));
20647        assert_eq!(reservation.pending_target_physical_bytes, 0);
20648        assert_eq!(reservation.consumed_target_physical_bytes, 2);
20649        assert_eq!(
20650            reservation
20651                .targets
20652                .iter()
20653                .map(|target| (target.pool_index, target.consumed_physical_bytes))
20654                .collect::<Vec<_>>(),
20655            vec![(1, 1), (2, 1)]
20656        );
20657    }
20658
20659    #[tokio::test]
20660    #[serial_test::serial]
20661    async fn ordinary_write_capacity_fence_serializes_with_decommission_activation() {
20662        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
20663        let (entered_tx, entered_rx) = tokio::sync::oneshot::channel();
20664        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
20665        let write_store = Arc::clone(&store);
20666        let ordinary_write = tokio::spawn(async move {
20667            write_store
20668                .run_decommission_capacity_admitted_mutation(1, None, None, || async move {
20669                    entered_tx.send(()).expect("ordinary write admission should be observed");
20670                    release_rx.await.expect("ordinary write should be released");
20671                    Ok(())
20672                })
20673                .await
20674        });
20675        entered_rx
20676            .await
20677            .expect("ordinary write should hold the shared capacity fence");
20678
20679        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20680        set_decommission_capacity_info_overrides_for_test(
20681            store.id,
20682            vec![vec![
20683                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
20684                DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1),
20685            ]],
20686        );
20687        let activation_store = Arc::clone(&store);
20688        let activation = tokio::spawn(async move {
20689            activation_store
20690                .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20691                .await
20692        });
20693        tokio::task::yield_now().await;
20694        assert!(
20695            !activation.is_finished(),
20696            "activation must wait until the already-admitted ordinary write leaves the distributed capacity boundary"
20697        );
20698
20699        release_tx.send(()).expect("ordinary write should be released");
20700        ordinary_write
20701            .await
20702            .expect("ordinary write task should join")
20703            .expect("the pre-activation ordinary write should complete");
20704        let err = activation
20705            .await
20706            .expect("activation task should join")
20707            .expect_err("activation must recheck and reject capacity consumed by the ordinary write");
20708        assert!(err.to_string().contains("requires 60 bytes, but 59 bytes are available"));
20709    }
20710
20711    async fn single_pool_capacity_admission_test_store() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
20712        let (temp_dirs, store) =
20713            crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta::default()).await;
20714        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
20715        (temp_dirs, store)
20716    }
20717
20718    #[tokio::test]
20719    #[serial_test::serial]
20720    async fn single_pool_public_writes_skip_decommission_capacity_admission() {
20721        let (_temp_dirs, store) = single_pool_capacity_admission_test_store().await;
20722        let bucket = format!("single-pool-capacity-skip-{}", uuid::Uuid::new_v4());
20723        store
20724            .make_bucket(&bucket, &MakeBucketOptions::default())
20725            .await
20726            .expect("create single-pool bucket before blocking pool metadata writes");
20727        let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
20728        store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
20729        let write_gate = store.pool_meta_write_gate_status().await;
20730        assert!(!write_gate.writes_ready);
20731        assert!(write_gate.write_blocked);
20732        assert!(!write_gate.transaction_aborted);
20733
20734        let object = "ordinary-put.bin";
20735        let mut put_data = crate::object_api::PutObjReader::from_vec(b"ordinary single-pool body".to_vec());
20736        store
20737            .put_object(&bucket, object, &mut put_data, &ObjectOptions::default())
20738            .await
20739            .expect("single-pool ordinary PUT must not enter decommission capacity admission");
20740        store
20741            .get_object_info(&bucket, object, &ObjectOptions::default())
20742            .await
20743            .expect("single-pool ordinary PUT must remain readable");
20744
20745        let multipart_object = "ordinary-multipart.bin";
20746        let upload = store
20747            .new_multipart_upload(
20748                &bucket,
20749                multipart_object,
20750                &ObjectOptions {
20751                    expected_bucket_incarnation_id: Some(incarnation),
20752                    ..Default::default()
20753                },
20754            )
20755            .await
20756            .expect("single-pool MPU creation must not enter decommission capacity admission");
20757        let mut part_data = crate::object_api::PutObjReader::from_vec(b"single-pool multipart body".to_vec());
20758        let part = store
20759            .put_object_part(
20760                &bucket,
20761                multipart_object,
20762                &upload.upload_id,
20763                1,
20764                &mut part_data,
20765                &ObjectOptions {
20766                    expected_bucket_incarnation_id: Some(incarnation),
20767                    ..Default::default()
20768                },
20769            )
20770            .await
20771            .expect("single-pool UploadPart must not enter decommission capacity admission");
20772        store
20773            .clone()
20774            .complete_multipart_upload(
20775                &bucket,
20776                multipart_object,
20777                &upload.upload_id,
20778                vec![crate::storage_api_contracts::multipart::CompletePart {
20779                    part_num: part.part_num,
20780                    etag: part.etag,
20781                    ..Default::default()
20782                }],
20783                &ObjectOptions {
20784                    expected_bucket_incarnation_id: Some(incarnation),
20785                    ..Default::default()
20786                },
20787            )
20788            .await
20789            .expect("single-pool CompleteMultipartUpload must not enter decommission capacity admission");
20790        store
20791            .get_object_info(&bucket, multipart_object, &ObjectOptions::default())
20792            .await
20793            .expect("single-pool completed MPU must remain readable");
20794    }
20795
20796    #[tokio::test]
20797    #[serial_test::serial]
20798    async fn multipart_mutations_locate_later_upload_before_reserved_pool_admission() {
20799        let (_temp_dirs, store, _other_store) =
20800            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
20801        let bucket = format!("multipart-capacity-routing-{}", uuid::Uuid::new_v4());
20802        let object = "later-pool.bin";
20803        store
20804            .make_bucket(&bucket, &MakeBucketOptions::default())
20805            .await
20806            .expect("create multipart routing bucket");
20807        let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
20808        let bucket_guard = store
20809            .acquire_bucket_lifecycle_read_lock(&bucket)
20810            .await
20811            .expect("acquire multipart routing bucket lifecycle guard");
20812        let mut upload_opts = ObjectOptions {
20813            expected_bucket_incarnation_id: Some(incarnation),
20814            ..Default::default()
20815        };
20816        upload_opts.add_bucket_lifecycle_lock_guard(&bucket_guard);
20817        let upload = store.pools[1]
20818            .new_multipart_upload(&bucket, object, &upload_opts)
20819            .await
20820            .expect("seed an upload in the later pool");
20821
20822        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20823        set_decommission_capacity_info_overrides_for_test(
20824            store.id,
20825            vec![vec![
20826                DecommissionPoolCapacityInfo::for_test(0, layout, 20, 20, 0),
20827                DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
20828                DecommissionPoolCapacityInfo::for_test(2, layout, 0, 10, 10),
20829            ]],
20830        );
20831        store
20832            .save_current_pool_meta_for_decommission_start(&[2], Vec::new())
20833            .await
20834            .expect("reserve the first target pool");
20835        {
20836            let pool_meta = store.pool_meta.read().await;
20837            let targets = &pool_meta.pools[2]
20838                .decommission
20839                .as_ref()
20840                .and_then(|info| info.capacity_reservation.as_ref())
20841                .expect("source reservation should exist")
20842                .targets;
20843            assert_eq!(targets.len(), 1);
20844            assert_eq!(targets[0].pool_index, 0, "the first probed pool must be reserved");
20845        }
20846
20847        let mut data = crate::object_api::PutObjReader::from_vec(b"multipart body".to_vec());
20848        let part = store
20849            .put_object_part(&bucket, object, &upload.upload_id, 1, &mut data, &ObjectOptions::default())
20850            .await
20851            .expect("put-part must locate the later upload before capacity admission");
20852        store
20853            .clone()
20854            .complete_multipart_upload(
20855                &bucket,
20856                object,
20857                &upload.upload_id,
20858                vec![crate::storage_api_contracts::multipart::CompletePart {
20859                    part_num: part.part_num,
20860                    etag: part.etag,
20861                    ..Default::default()
20862                }],
20863                &ObjectOptions::default(),
20864            )
20865            .await
20866            .expect("complete must locate the later upload before capacity admission");
20867
20868        store.pools[1]
20869            .get_object_info(&bucket, object, &ObjectOptions::default())
20870            .await
20871            .expect("the later-pool multipart upload should commit in place");
20872        let first_pool_result = store.pools[0]
20873            .get_object_info(&bucket, object, &ObjectOptions::default())
20874            .await;
20875        assert!(
20876            first_pool_result
20877                .as_ref()
20878                .err()
20879                .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)),
20880            "the reserved first pool must not receive the multipart mutation"
20881        );
20882    }
20883
20884    #[tokio::test]
20885    #[serial_test::serial]
20886    async fn expired_decommission_capacity_lease_requires_recovery_before_entry() {
20887        let (_temp_dirs, store, _other_store) =
20888            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
20889        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20890        set_decommission_capacity_info_overrides_for_test(
20891            store.id,
20892            vec![vec![
20893                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 1, 1),
20894                DecommissionPoolCapacityInfo::for_test(1, layout, 2, 2, 0),
20895                DecommissionPoolCapacityInfo::for_test(2, layout, 2, 2, 0),
20896            ]],
20897        );
20898        store
20899            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20900            .await
20901            .expect("activate the expired lease reservation");
20902        let generation = store
20903            .active_decommission_generation(0)
20904            .await
20905            .expect("read the active decommission generation");
20906        {
20907            let mut pool_meta = store.pool_meta.write().await;
20908            let reservation = pool_meta.pools[0]
20909                .decommission
20910                .as_mut()
20911                .and_then(|info| info.capacity_reservation.as_mut())
20912                .expect("the expired lease reservation should exist");
20913            reservation.expires_at = OffsetDateTime::now_utc() - Duration::seconds(1);
20914        }
20915
20916        let err = store
20917            .decommission_capacity_owner_for_worker(0, generation)
20918            .await
20919            .expect_err("an expired capacity lease must not fall back to ownerless migration");
20920        assert!(is_decommission_capacity_blocked_error(&err));
20921    }
20922
20923    #[tokio::test]
20924    #[serial_test::serial]
20925    async fn expired_capacity_owner_nonce_rejects_stale_put_and_multipart_without_consuming() {
20926        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
20927        let bucket = format!("stale-capacity-owner-{}", uuid::Uuid::new_v4());
20928        let multipart_object = "stale-multipart.bin";
20929        store
20930            .make_bucket(&bucket, &MakeBucketOptions::default())
20931            .await
20932            .expect("create stale owner bucket");
20933        let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
20934        let bucket_guard = store
20935            .acquire_bucket_lifecycle_read_lock(&bucket)
20936            .await
20937            .expect("acquire stale owner bucket lifecycle guard");
20938        let mut upload_opts = ObjectOptions {
20939            expected_bucket_incarnation_id: Some(incarnation),
20940            ..Default::default()
20941        };
20942        upload_opts.add_bucket_lifecycle_lock_guard(&bucket_guard);
20943        let upload = store.pools[1]
20944            .new_multipart_upload(&bucket, multipart_object, &upload_opts)
20945            .await
20946            .expect("seed multipart upload before reserving the target");
20947
20948        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
20949        set_decommission_capacity_info_overrides_for_test(
20950            store.id,
20951            vec![vec![
20952                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
20953                DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
20954            ]],
20955        );
20956        store
20957            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
20958            .await
20959            .expect("activate target reservation");
20960
20961        let stale_owner = {
20962            let pool_meta = store.pool_meta.read().await;
20963            let reservation = pool_meta.pools[0]
20964                .decommission
20965                .as_ref()
20966                .and_then(|info| info.capacity_reservation.as_ref())
20967                .expect("active reservation should exist");
20968            DecommissionCapacityOwner {
20969                source_pool_index: 0,
20970                operation_id: reservation.operation_id,
20971                generation: reservation.generation,
20972                owner_nonce: reservation.owner_nonce,
20973                mutation_id: None,
20974            }
20975        };
20976        {
20977            let mut pool_meta = store.pool_meta.write().await;
20978            let reservation = pool_meta.pools[0]
20979                .decommission
20980                .as_mut()
20981                .and_then(|info| info.capacity_reservation.as_mut())
20982                .expect("active reservation should remain mutable");
20983            reservation.expires_at = OffsetDateTime::now_utc() - Duration::seconds(1);
20984        }
20985        store
20986            .save_current_pool_meta(&[0])
20987            .await
20988            .expect("persist the expired lease before recovery");
20989        store
20990            .pause_decommission_for_capacity(0, &decommission_capacity_blocked_error("test lease recovery"))
20991            .await
20992            .expect("pause recovery should renew the lease with a new owner nonce");
20993
20994        let (current_owner, before_progress) = {
20995            let pool_meta = store.pool_meta.read().await;
20996            let reservation = pool_meta.pools[0]
20997                .decommission
20998                .as_ref()
20999                .and_then(|info| info.capacity_reservation.as_ref())
21000                .expect("renewed reservation should exist");
21001            (
21002                DecommissionCapacityOwner {
21003                    source_pool_index: 0,
21004                    operation_id: reservation.operation_id,
21005                    generation: reservation.generation,
21006                    owner_nonce: reservation.owner_nonce,
21007                    mutation_id: None,
21008                },
21009                (
21010                    reservation.consumed_target_physical_bytes,
21011                    reservation.inflight_target_physical_bytes,
21012                    reservation.pending_target_physical_bytes,
21013                    reservation.observed_target_physical_bytes,
21014                ),
21015            )
21016        };
21017        assert_ne!(stale_owner.owner_nonce, current_owner.owner_nonce);
21018        {
21019            let pool_meta = store.pool_meta.read().await;
21020            ensure_decommission_target_owner_admission(&pool_meta, current_owner, 1, 1, OffsetDateTime::now_utc())
21021                .expect("the renewed owner should retain its reservation");
21022            assert!(
21023                ensure_decommission_target_owner_admission(&pool_meta, stale_owner, 1, 1, OffsetDateTime::now_utc(),).is_err(),
21024                "the expired owner token must be rejected after nonce rotation"
21025            );
21026        }
21027
21028        let stale_put_object = "stale-put.bin";
21029        let mut stale_put_opts = ObjectOptions {
21030            data_movement: true,
21031            version_id: Some(uuid::Uuid::new_v4().to_string()),
21032            ..ObjectOptions::with_capacity_expected_data_bytes(Some(9))
21033        };
21034        stale_owner.apply_to(&mut stale_put_opts);
21035        let mut stale_put_data = crate::object_api::PutObjReader::from_vec(b"stale put".to_vec());
21036        let put_err = store
21037            .put_object_for_data_movement(&bucket, stale_put_object, &mut stale_put_data, &stale_put_opts, None)
21038            .await
21039            .expect_err("stale owner PUT must fail before selecting a target");
21040        assert!(is_decommission_capacity_blocked_error(&put_err));
21041
21042        let mut stale_part_opts = ObjectOptions {
21043            data_movement: true,
21044            part_number: Some(1),
21045            expected_bucket_incarnation_id: Some(incarnation),
21046            ..Default::default()
21047        };
21048        stale_owner.apply_to(&mut stale_part_opts);
21049        let mut stale_part_data = crate::object_api::PutObjReader::from_vec(b"stale part".to_vec());
21050        let part_err = store
21051            .put_object_part_for_data_movement(
21052                1,
21053                &bucket,
21054                multipart_object,
21055                &upload.upload_id,
21056                &mut stale_part_data,
21057                &stale_part_opts,
21058            )
21059            .await
21060            .expect_err("stale owner multipart PUT must fail before mutation");
21061        assert!(is_decommission_capacity_blocked_error(&part_err));
21062
21063        let after_progress = {
21064            let pool_meta = store.pool_meta.read().await;
21065            let reservation = pool_meta.pools[0]
21066                .decommission
21067                .as_ref()
21068                .and_then(|info| info.capacity_reservation.as_ref())
21069                .expect("renewed reservation should remain active");
21070            (
21071                reservation.consumed_target_physical_bytes,
21072                reservation.inflight_target_physical_bytes,
21073                reservation.pending_target_physical_bytes,
21074                reservation.observed_target_physical_bytes,
21075            )
21076        };
21077        assert_eq!(after_progress, before_progress, "stale mutations must not consume the capacity ledger");
21078        let parts = store.pools[1]
21079            .list_object_parts(
21080                &bucket,
21081                multipart_object,
21082                &upload.upload_id,
21083                None,
21084                1_000,
21085                &ObjectOptions {
21086                    expected_bucket_incarnation_id: Some(incarnation),
21087                    ..Default::default()
21088                },
21089            )
21090            .await
21091            .expect("the seeded upload should remain readable");
21092        assert!(parts.parts.is_empty(), "the stale multipart write must not stage a part");
21093        assert!(
21094            store.pools[1]
21095                .get_object_info(&bucket, stale_put_object, &ObjectOptions::default())
21096                .await
21097                .is_err(),
21098            "the stale PUT must not create a target object"
21099        );
21100    }
21101
21102    #[tokio::test]
21103    #[serial_test::serial]
21104    async fn decommission_activation_lock_recheck_rejects_sudden_space_drop_without_persisting() {
21105        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
21106        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
21107        let source = DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30);
21108        let preflight = vec![source, DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0)];
21109        {
21110            let pool_meta = store.pool_meta.read().await;
21111            ensure_decommission_start_target_capacity(&pool_meta, &[0], &preflight, true)
21112                .expect("the pre-lock capacity snapshot should fit exactly");
21113        }
21114
21115        set_decommission_capacity_info_overrides_for_test(
21116            store.id,
21117            vec![vec![source, DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)]],
21118        );
21119        let err = store
21120            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
21121            .await
21122            .expect_err("the capacity snapshot inside the activation lock must be authoritative");
21123        assert!(err.to_string().contains("requires 60 bytes, but 59 bytes are available"));
21124
21125        let local = store.pool_meta.read().await;
21126        assert!(local.pools[0].decommission.is_none());
21127        drop(local);
21128        let mut persisted = PoolMeta::default();
21129        persisted
21130            .load_no_lock_from_replicas(store.pools.clone())
21131            .await
21132            .expect("the rejected activation must leave baseline metadata readable");
21133        assert!(persisted.pools[0].decommission.is_none());
21134    }
21135
21136    #[tokio::test]
21137    #[serial_test::serial]
21138    async fn decommission_worker_pauses_on_runtime_capacity_shortage_without_source_side_effect() {
21139        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
21140        let bucket = format!("capacity-blocked-{}", uuid::Uuid::new_v4());
21141        let object = "object.bin";
21142        store
21143            .make_bucket(&bucket, &MakeBucketOptions::default())
21144            .await
21145            .expect("the production-path bucket should be created");
21146        let mut source = crate::object_api::PutObjReader::from_vec(b"source remains authoritative".to_vec());
21147        store.pools[0]
21148            .put_object(&bucket, object, &mut source, &ObjectOptions::default())
21149            .await
21150            .expect("the source object should be written before decommission starts");
21151
21152        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
21153        let enough = vec![
21154            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
21155            DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
21156        ];
21157        set_decommission_capacity_info_overrides_for_test(store.id, vec![enough.clone()]);
21158        store
21159            .save_current_pool_meta_for_decommission_start(
21160                &[0],
21161                vec![DecomBucketInfo {
21162                    name: bucket.clone(),
21163                    prefix: String::new(),
21164                }],
21165            )
21166            .await
21167            .expect("the initial reservation should be activated");
21168
21169        let shortage = vec![enough[0], DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)];
21170        set_decommission_capacity_info_overrides_for_test(store.id, vec![shortage]);
21171        let canceler = DecommissionCanceler::new(CancellationToken::new());
21172        store.decommission_cancelers.write().await[0] = Some(canceler.clone());
21173        store
21174            .do_decommission_in_routine(canceler, 0, Arc::new(Semaphore::new(1)))
21175            .await
21176            .expect("runtime capacity shortage should pause the worker, not fail it");
21177
21178        store.pools[0]
21179            .get_object_info(&bucket, object, &ObjectOptions::default())
21180            .await
21181            .expect("capacity pause must not delete the source object");
21182        let target_result = store.pools[1]
21183            .get_object_info(&bucket, object, &ObjectOptions::default())
21184            .await;
21185        assert!(
21186            target_result
21187                .as_ref()
21188                .err()
21189                .is_some_and(|err| is_err_object_not_found(err) || is_err_version_not_found(err)),
21190            "capacity pause must happen before a target mutation"
21191        );
21192
21193        let local = store.pool_meta.read().await;
21194        let info = local.pools[0]
21195            .decommission
21196            .as_ref()
21197            .expect("the blocked decommission state should remain present");
21198        assert!(
21199            !info.complete && !info.failed && !info.canceled,
21200            "capacity-blocked state unexpectedly became terminal: complete={}, failed={}, canceled={}, blocked_reason={:?}",
21201            info.complete,
21202            info.failed,
21203            info.canceled,
21204            info.capacity_blocked_reason
21205        );
21206        assert_eq!(info.items_decommission_failed, 0);
21207        assert_eq!(info.bytes_failed, 0);
21208        assert!(info.capacity_blocked_reason.is_some());
21209        assert!(
21210            info.capacity_reservation
21211                .as_ref()
21212                .is_some_and(DecommissionCapacityReservation::active),
21213            "blocked work must retain and renew its reservation"
21214        );
21215        drop(local);
21216
21217        let mut persisted = PoolMeta::default();
21218        persisted
21219            .load_no_lock_from_replicas(store.pools.clone())
21220            .await
21221            .expect("the blocked state should be durable");
21222        let persisted_info = persisted.pools[0]
21223            .decommission
21224            .as_ref()
21225            .expect("the durable blocked state should remain nonterminal");
21226        assert!(!persisted_info.complete && !persisted_info.failed && !persisted_info.canceled);
21227        assert!(persisted_info.capacity_blocked_reason.is_some());
21228        assert!(
21229            persisted_info
21230                .capacity_reservation
21231                .as_ref()
21232                .is_some_and(DecommissionCapacityReservation::active)
21233        );
21234    }
21235
21236    #[tokio::test]
21237    #[serial_test::serial]
21238    async fn decommission_worker_rechecks_runtime_capacity_before_empty_background_completion() {
21239        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
21240        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
21241        let enough = vec![
21242            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
21243            DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
21244        ];
21245        set_decommission_capacity_info_overrides_for_test(store.id, vec![enough.clone()]);
21246        store
21247            .save_current_pool_meta_for_decommission_start(&[0], Vec::new())
21248            .await
21249            .expect("the initial reservation should be activated");
21250
21251        let shortage = vec![enough[0], DecommissionPoolCapacityInfo::for_test(1, layout, 59, 60, 1)];
21252        set_decommission_capacity_info_overrides_for_test(store.id, vec![shortage]);
21253        let canceler = DecommissionCanceler::new(CancellationToken::new());
21254        store.decommission_cancelers.write().await[0] = Some(canceler.clone());
21255        store
21256            .do_decommission_in_routine(canceler, 0, Arc::new(Semaphore::new(1)))
21257            .await
21258            .expect("runtime capacity shortage should pause the worker before background completion");
21259
21260        let local = store.pool_meta.read().await;
21261        let info = local.pools[0]
21262            .decommission
21263            .as_ref()
21264            .expect("the blocked decommission state should remain present");
21265        assert!(!info.complete && !info.failed && !info.canceled);
21266        assert!(info.capacity_blocked_reason.is_some());
21267        assert!(
21268            info.capacity_reservation
21269                .as_ref()
21270                .is_some_and(DecommissionCapacityReservation::active)
21271        );
21272    }
21273
21274    fn pool_meta_replica_test_meta(cmd_line: &str) -> PoolMeta {
21275        PoolMeta {
21276            version: POOL_META_VERSION,
21277            pools: vec![PoolStatus {
21278                id: 0,
21279                cmd_line: cmd_line.to_string(),
21280                last_update: OffsetDateTime::UNIX_EPOCH,
21281                decommission: None,
21282            }],
21283            dont_save: false,
21284        }
21285    }
21286
21287    fn pool_meta_replica_test_data(cmd_line: &str) -> Vec<u8> {
21288        pool_meta_replica_test_meta(cmd_line)
21289            .encode_config_data()
21290            .expect("pool metadata should encode")
21291    }
21292
21293    fn pool_meta_v1_replica_test_data(meta: &PoolMeta) -> Vec<u8> {
21294        let mut data = Vec::new();
21295        data.write_u16::<LittleEndian>(POOL_META_FORMAT)
21296            .expect("pool metadata format should encode");
21297        data.write_u16::<LittleEndian>(POOL_META_V1_VERSION)
21298            .expect("pool metadata version should encode");
21299        PersistedPoolMetaV1::from(meta)
21300            .serialize(&mut Serializer::new(&mut data))
21301            .expect("legacy pool metadata should encode");
21302        data
21303    }
21304
21305    fn pool_meta_persisted_v1_replica_test_data(cmd_line: &str) -> Vec<u8> {
21306        pool_meta_v1_replica_test_data(&pool_meta_replica_test_meta(cmd_line))
21307    }
21308
21309    fn pool_meta_legacy_v1_replica_test_data(cmd_line: &str) -> Vec<u8> {
21310        let mut data = Vec::new();
21311        data.write_u16::<LittleEndian>(POOL_META_FORMAT)
21312            .expect("pool metadata format should encode");
21313        data.write_u16::<LittleEndian>(POOL_META_V1_VERSION)
21314            .expect("pool metadata version should encode");
21315        LegacyPoolMeta {
21316            version: POOL_META_V1_VERSION,
21317            pools: vec![LegacyPoolStatus {
21318                id: 0,
21319                cmd_line: cmd_line.to_string(),
21320                last_update: OffsetDateTime::UNIX_EPOCH,
21321                decommission: None,
21322            }],
21323            dont_save: false,
21324        }
21325        .serialize(&mut Serializer::new(&mut data))
21326        .expect("legacy v1 pool metadata should encode");
21327        data
21328    }
21329
21330    #[test]
21331    fn pool_meta_replica_selection_falls_back_from_corrupt_first_copy() {
21332        let selection = select_pool_meta_replica(vec![
21333            PoolMetaReplica::Corrupt("truncated".to_string()),
21334            decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
21335        ])
21336        .expect("a validated backup replica should be selected");
21337
21338        assert!(selection.replica_state.needs_repair);
21339        assert!(selection.replica_state.repair_write_safe);
21340        assert_eq!(selection.meta.pools.len(), 1);
21341        assert_eq!(selection.meta.pools[0].cmd_line, "pool-0");
21342    }
21343
21344    #[test]
21345    fn pool_meta_replica_selection_rejects_incompatible_copy() {
21346        let valid = pool_meta_replica_test_data("pool-0");
21347        let mut incompatible = valid.clone();
21348        LittleEndian::write_u16(&mut incompatible[2..4], POOL_META_VERSION + 1);
21349
21350        let err = select_pool_meta_replica(vec![decode_pool_meta_replica(valid), decode_pool_meta_replica(incompatible)])
21351            .expect_err("an incompatible replica must block fallback and repair writes");
21352
21353        assert!(err.to_string().contains("pool 1 is incompatible"));
21354        assert!(err.to_string().contains("without overwriting it"));
21355    }
21356
21357    #[test]
21358    fn pool_meta_replica_selection_prefers_valid_canonical_first_copy() {
21359        let selection = select_pool_meta_replica(vec![
21360            decode_pool_meta_replica(pool_meta_replica_test_data("pool-canonical")),
21361            decode_pool_meta_replica(pool_meta_replica_test_data("pool-stale")),
21362        ])
21363        .expect("pool zero is the durable commit record when a later replica is stale");
21364
21365        assert_eq!(selection.meta.pools[0].cmd_line, "pool-canonical");
21366        assert!(selection.replica_state.needs_repair);
21367        assert!(selection.replica_state.repair_write_safe);
21368    }
21369
21370    #[test]
21371    fn pool_meta_replica_selection_rejects_divergent_backups_without_canonical() {
21372        let err = select_pool_meta_replica(vec![
21373            PoolMetaReplica::Corrupt("canonical unavailable".to_string()),
21374            decode_pool_meta_replica(pool_meta_replica_test_data("pool-old")),
21375            decode_pool_meta_replica(pool_meta_replica_test_data("pool-new")),
21376        ])
21377        .expect_err("divergent backups have no safe ordering when the canonical copy is unavailable");
21378
21379        assert!(err.to_string().contains("valid replicas in pools 1 and 2 diverge"));
21380    }
21381
21382    #[test]
21383    fn pool_meta_replica_selection_normalizes_equivalent_legacy_copy() {
21384        let selection = select_pool_meta_replica(vec![
21385            decode_pool_meta_replica(pool_meta_legacy_v1_replica_test_data("pool-0")),
21386            decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
21387        ])
21388        .expect("equivalent legacy and current encodings should be compatible");
21389
21390        assert!(selection.replica_state.needs_repair);
21391        assert!(selection.replica_state.repair_write_safe);
21392        assert_eq!(selection.meta.pools[0].cmd_line, "pool-0");
21393    }
21394
21395    #[test]
21396    fn pool_meta_v1_replica_migrates_to_v2_canonical_form() {
21397        let mut source = pool_meta_replica_test_meta("pool-0");
21398        source.pools[0].decommission = Some(PoolDecommissionInfo {
21399            start_time: Some(OffsetDateTime::UNIX_EPOCH),
21400            queued_buckets: vec!["bucket-a".to_string()],
21401            ..Default::default()
21402        });
21403        let data = pool_meta_v1_replica_test_data(&source);
21404        let PoolMetaReplica::Valid {
21405            raw, canonical, meta, ..
21406        } = decode_pool_meta_replica(data)
21407        else {
21408            panic!("v1 pool metadata should remain readable");
21409        };
21410
21411        assert_eq!(LittleEndian::read_u16(&raw[2..4]), POOL_META_V1_VERSION);
21412        assert_eq!(LittleEndian::read_u16(&canonical[2..4]), POOL_META_VERSION);
21413        assert_eq!(meta.version, POOL_META_V1_VERSION);
21414        let info = meta.pools[0]
21415            .decommission
21416            .as_ref()
21417            .expect("v1 decommission state should migrate");
21418        assert_eq!(info.queued_buckets, vec!["bucket-a".to_string()]);
21419        assert!(info.unresolved_entries.is_empty());
21420    }
21421
21422    #[test]
21423    fn pool_meta_legacy_v1_replica_migrates_to_v2_canonical_form() {
21424        let data = pool_meta_legacy_v1_replica_test_data("pool-0");
21425        let PoolMetaReplica::Valid {
21426            raw, canonical, meta, ..
21427        } = decode_pool_meta_replica(data)
21428        else {
21429            panic!("legacy v1 pool metadata should remain readable");
21430        };
21431
21432        assert_eq!(LittleEndian::read_u16(&raw[2..4]), POOL_META_V1_VERSION);
21433        assert_eq!(LittleEndian::read_u16(&canonical[2..4]), POOL_META_VERSION);
21434        assert_eq!(meta.version, POOL_META_V1_VERSION);
21435        assert_eq!(meta.pools[0].cmd_line, "pool-0");
21436    }
21437
21438    #[test]
21439    fn pool_meta_replica_selection_rejects_v1_v2_divergence() {
21440        let err = select_pool_meta_replica(vec![
21441            decode_pool_meta_replica(pool_meta_persisted_v1_replica_test_data("pool-old")),
21442            decode_pool_meta_replica(pool_meta_replica_test_data("pool-new")),
21443        ])
21444        .expect_err("different v1 and v2 snapshots must remain fail-closed");
21445
21446        assert!(err.to_string().contains("valid replicas in pools 0 and 1 diverge"));
21447    }
21448
21449    #[test]
21450    fn pool_meta_replica_rejects_header_payload_version_mismatch() {
21451        let mut v1_with_v2_header = pool_meta_persisted_v1_replica_test_data("pool-0");
21452        LittleEndian::write_u16(&mut v1_with_v2_header[2..4], POOL_META_VERSION);
21453        assert!(matches!(decode_pool_meta_replica(v1_with_v2_header), PoolMetaReplica::Corrupt(_)));
21454
21455        let mut v2_with_v1_header = pool_meta_replica_test_data("pool-0");
21456        LittleEndian::write_u16(&mut v2_with_v1_header[2..4], POOL_META_V1_VERSION);
21457        assert!(matches!(
21458            decode_pool_meta_replica(v2_with_v1_header),
21459            PoolMetaReplica::Corrupt(_) | PoolMetaReplica::Incompatible(_)
21460        ));
21461    }
21462
21463    #[test]
21464    fn pool_meta_v2_header_guards_v1_readers_from_tuple_extension() {
21465        let mut meta = pool_meta_replica_test_meta("pool-0");
21466        meta.pools[0].decommission = Some(PoolDecommissionInfo::default());
21467        let data = meta.encode_config_data().expect("v2 pool metadata should encode");
21468
21469        assert_eq!(LittleEndian::read_u16(&data[2..4]), POOL_META_VERSION);
21470        assert_ne!(POOL_META_VERSION, POOL_META_V1_VERSION);
21471        assert!(rmp_serde::from_slice::<PersistedPoolMetaV1>(&data[4..]).is_err());
21472    }
21473
21474    #[test]
21475    fn pool_meta_writer_stays_v1_until_v2_is_fleet_confirmed() {
21476        let mut meta = pool_meta_replica_test_meta("pool-0");
21477        meta.version = POOL_META_V1_VERSION;
21478
21479        let v1 = meta
21480            .encode_config_data_for_v2_gate(false)
21481            .expect("the default writer should preserve v1");
21482        let v2 = meta
21483            .encode_config_data_for_v2_gate(true)
21484            .expect("the confirmed writer should emit v2");
21485
21486        assert_eq!(LittleEndian::read_u16(&v1[2..4]), POOL_META_V1_VERSION);
21487        assert_eq!(LittleEndian::read_u16(&v2[2..4]), POOL_META_VERSION);
21488        assert!(rmp_serde::from_slice::<PersistedPoolMetaV1>(&v1[4..]).is_ok());
21489    }
21490
21491    #[test]
21492    fn pool_meta_v2_writer_requires_both_gates() {
21493        assert!(!pool_meta_v2_writer_enabled_for(false, false));
21494        assert!(!pool_meta_v2_writer_enabled_for(true, false));
21495        assert!(!pool_meta_v2_writer_enabled_for(false, true));
21496        assert!(pool_meta_v2_writer_enabled_for(true, true));
21497    }
21498
21499    #[test]
21500    fn pool_meta_v3_writer_requires_both_gates() {
21501        assert!(!pool_meta_v3_writer_enabled_for(false, false));
21502        assert!(!pool_meta_v3_writer_enabled_for(true, false));
21503        assert!(!pool_meta_v3_writer_enabled_for(false, true));
21504        assert!(pool_meta_v3_writer_enabled_for(true, true));
21505    }
21506
21507    #[test]
21508    fn decommission_ledger_persistence_requires_an_observed_or_confirmed_format() {
21509        for (version, v2_enabled, v3_enabled, expected) in [
21510            (POOL_META_V1_VERSION, false, false, false),
21511            (POOL_META_V1_VERSION, true, false, true),
21512            (POOL_META_V1_VERSION, false, true, true),
21513            (POOL_META_VERSION, false, false, true),
21514            (super::POOL_META_GENERATION_VERSION, false, false, true),
21515        ] {
21516            let result = super::ensure_decommission_ledger_persistence_supported_for(version, v2_enabled, v3_enabled);
21517            assert_eq!(
21518                result.is_ok(),
21519                expected,
21520                "unexpected capability result for pool metadata version {version}"
21521            );
21522        }
21523
21524        let half_confirmed_v2 = pool_meta_v2_writer_enabled_for(true, false);
21525        let half_confirmed_v3 = pool_meta_v3_writer_enabled_for(false, true);
21526        let err = super::ensure_decommission_ledger_persistence_supported_for(
21527            POOL_META_V1_VERSION,
21528            half_confirmed_v2,
21529            half_confirmed_v3,
21530        )
21531        .expect_err("half-enabled rollout gates must not admit decommission");
21532        assert!(matches!(err, Error::InvalidArgument(..)));
21533        assert!(err.to_string().contains("durable unresolved-entry recovery"));
21534        assert!(err.to_string().contains(rustfs_config::ENV_POOL_META_V3_WRITE));
21535        assert!(err.to_string().contains(rustfs_config::ENV_POOL_META_V3_FLEET_CONFIRMED));
21536    }
21537
21538    #[test]
21539    fn decommission_capacity_accepts_v3_only_writer_rollout() {
21540        assert!(decommission_capacity_writer_supported_for(
21541            POOL_META_V1_VERSION,
21542            false,
21543            pool_meta_v3_writer_enabled_for(true, true),
21544        ));
21545        assert!(!decommission_capacity_writer_supported_for(
21546            POOL_META_V1_VERSION,
21547            false,
21548            pool_meta_v3_writer_enabled_for(true, false),
21549        ));
21550    }
21551
21552    #[test]
21553    fn pool_meta_stale_write_rejection_metric_is_countable() {
21554        let recorder = metrics_util::debugging::DebuggingRecorder::new();
21555        let snapshotter = recorder.snapshotter();
21556        metrics::with_local_recorder(&recorder, || {
21557            record_pool_meta_stale_write_rejection("prepare_cas");
21558            record_pool_meta_stale_write_rejection("prepare_cas");
21559        });
21560
21561        let total = snapshotter
21562            .snapshot()
21563            .into_vec()
21564            .into_iter()
21565            .filter(|(composite, _, _, _)| composite.key().name() == METRIC_POOL_META_STALE_WRITE_REJECTIONS_TOTAL)
21566            .filter_map(|(_, _, _, value)| match value {
21567                metrics_util::debugging::DebugValue::Counter(value) => Some(value),
21568                _ => None,
21569            })
21570            .sum::<u64>();
21571        assert_eq!(total, 2);
21572    }
21573
21574    #[test]
21575    fn pool_meta_block_transition_is_counted_once_across_both_latches() {
21576        let recorder = metrics_util::debugging::DebuggingRecorder::new();
21577        let snapshotter = recorder.snapshotter();
21578        metrics::with_local_recorder(&recorder, || {
21579            let mut state = PoolMetaWriteState::default();
21580            let mut arm = state.arm_transaction();
21581            arm.phase = Some("commit_cas");
21582            drop(arm);
21583            let first = state
21584                .ensure_write_safe("first block")
21585                .unwrap_err()
21586                .pool_metadata_failure()
21587                .unwrap()
21588                .since;
21589            state.block_writes_after_fence_loss();
21590            state.block_writes();
21591            assert_eq!(
21592                state
21593                    .ensure_write_safe("later block")
21594                    .unwrap_err()
21595                    .pool_metadata_failure()
21596                    .unwrap()
21597                    .since,
21598                first
21599            );
21600        });
21601        let total = snapshotter
21602            .snapshot()
21603            .into_vec()
21604            .into_iter()
21605            .filter(|(key, _, _, _)| key.key().name() == "rustfs_pool_metadata_blocks_total")
21606            .filter_map(|(_, _, _, value)| match value {
21607                metrics_util::debugging::DebugValue::Counter(count) => Some(count),
21608                _ => None,
21609            })
21610            .sum::<u64>();
21611        assert_eq!(total, 1);
21612    }
21613
21614    fn pool_meta_v3_test_revision(cluster_id: uuid::Uuid, generation: u64, transaction_id: uuid::Uuid) -> PoolMetaRevision {
21615        PoolMetaRevision {
21616            version: POOL_META_GENERATION_VERSION,
21617            cluster_id: Some(cluster_id),
21618            epoch: POOL_META_INITIAL_EPOCH,
21619            generation,
21620            transaction_id: Some(transaction_id),
21621        }
21622    }
21623
21624    fn pool_meta_legacy_candidate(meta: PoolMeta) -> PoolMetaCommittedCandidate {
21625        PoolMetaCommittedCandidate {
21626            canonical: meta
21627                .encode_config_data_for_v2_gate(true)
21628                .expect("legacy pool metadata should encode"),
21629            revision: PoolMetaRevision::legacy(meta.version),
21630            meta,
21631        }
21632    }
21633
21634    #[test]
21635    fn pool_meta_v3_pending_prepare_recovers_previous_snapshot_after_restart() {
21636        let previous = pool_meta_replica_test_meta("pool-before");
21637        let mut next = pool_meta_replica_test_meta("pool-after");
21638        next.version = POOL_META_GENERATION_VERSION;
21639        let revision = pool_meta_v3_test_revision(uuid::Uuid::new_v4(), 1, uuid::Uuid::new_v4());
21640        let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&pool_meta_legacy_candidate(previous.clone())))
21641            .expect("pending generation should encode");
21642
21643        let selected = select_pool_meta_replica(vec![
21644            decode_pool_meta_replica(pending),
21645            decode_pool_meta_replica(
21646                previous
21647                    .encode_config_data_for_v2_gate(true)
21648                    .expect("previous snapshot should encode"),
21649            ),
21650        ])
21651        .expect("a prepare-only transaction should expose its committed predecessor");
21652
21653        assert_eq!(selected.meta.pools[0].cmd_line, "pool-before");
21654        assert_eq!(selected.revision.version, POOL_META_VERSION);
21655        assert!(selected.replica_state.needs_repair);
21656    }
21657
21658    #[test]
21659    fn pool_meta_v3_partial_commit_selects_new_generation_after_restart() {
21660        let previous = pool_meta_replica_test_meta("pool-before");
21661        let mut next = pool_meta_replica_test_meta("pool-after");
21662        next.version = POOL_META_GENERATION_VERSION;
21663        let revision = pool_meta_v3_test_revision(uuid::Uuid::new_v4(), 1, uuid::Uuid::new_v4());
21664        let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&pool_meta_legacy_candidate(previous)))
21665            .expect("pending generation should encode");
21666        let committed = encode_pool_meta_v3_envelope(&next, revision, true, None).expect("committed generation should encode");
21667
21668        let selected = select_pool_meta_replica(vec![decode_pool_meta_replica(committed), decode_pool_meta_replica(pending)])
21669            .expect("one committed replica should make the cross-pool transaction durable");
21670
21671        assert_eq!(selected.meta.pools[0].cmd_line, "pool-after");
21672        assert_eq!(selected.revision, revision);
21673        assert!(selected.replica_state.needs_repair);
21674    }
21675
21676    #[test]
21677    fn pool_meta_v3_uses_valid_committed_backup_but_rejects_same_generation_fork() {
21678        let cluster_id = uuid::Uuid::new_v4();
21679        let mut next = pool_meta_replica_test_meta("pool-after");
21680        next.version = POOL_META_GENERATION_VERSION;
21681        let revision = pool_meta_v3_test_revision(cluster_id, 7, uuid::Uuid::new_v4());
21682        let committed = encode_pool_meta_v3_envelope(&next, revision, true, None).expect("committed generation should encode");
21683        let selected = select_pool_meta_replica(vec![
21684            PoolMetaReplica::Corrupt("truncated".to_string()),
21685            decode_pool_meta_replica(committed.clone()),
21686        ])
21687        .expect("a corrupt first copy should fall back to a valid committed generation");
21688        assert_eq!(selected.revision, revision);
21689        assert!(selected.replica_state.needs_repair);
21690
21691        let fork_revision = pool_meta_v3_test_revision(cluster_id, 7, uuid::Uuid::new_v4());
21692        let fork = encode_pool_meta_v3_envelope(&next, fork_revision, true, None).expect("fork generation should encode");
21693        let err = select_pool_meta_replica(vec![decode_pool_meta_replica(committed), decode_pool_meta_replica(fork)])
21694            .expect_err("same-generation transactions must never be selected by pool order");
21695        assert!(err.to_string().contains("committed generation 7 diverges"));
21696    }
21697
21698    #[test]
21699    fn pool_meta_v3_migration_keeps_legacy_committed_until_commit_record_exists() {
21700        let previous = pool_meta_replica_test_meta("pool-before");
21701        let mut next = pool_meta_replica_test_meta("pool-after");
21702        next.version = POOL_META_GENERATION_VERSION;
21703        let revision = pool_meta_v3_test_revision(uuid::Uuid::new_v4(), 1, uuid::Uuid::new_v4());
21704        let pending = encode_pool_meta_v3_envelope(&next, revision, false, Some(&pool_meta_legacy_candidate(previous.clone())))
21705            .expect("pending migration should encode");
21706        let legacy = previous
21707            .encode_config_data_for_v2_gate(true)
21708            .expect("legacy snapshot should encode");
21709
21710        let prepared = select_pool_meta_replica(vec![decode_pool_meta_replica(pending), decode_pool_meta_replica(legacy)])
21711            .expect("prepared migration should retain the legacy commit");
21712        assert_eq!(prepared.meta.version, POOL_META_VERSION);
21713        assert_eq!(prepared.meta.pools[0].cmd_line, "pool-before");
21714
21715        let committed = encode_pool_meta_v3_envelope(&next, revision, true, None).expect("committed migration should encode");
21716        let migrated = select_pool_meta_replica(vec![decode_pool_meta_replica(committed)])
21717            .expect("committed migration should establish the V3 floor");
21718        assert_eq!(migrated.meta.version, POOL_META_GENERATION_VERSION);
21719        assert_eq!(migrated.meta.pools[0].cmd_line, "pool-after");
21720    }
21721
21722    #[test]
21723    fn pool_meta_v3_unknown_fields_remain_incompatible_not_silently_ignored() {
21724        #[derive(Serialize)]
21725        struct FuturePoolMetaV3 {
21726            version: u16,
21727            cluster_id: String,
21728            epoch: u64,
21729            generation: u64,
21730            transaction_id: String,
21731            committed: bool,
21732            pools: Vec<PersistedPoolStatus>,
21733            previous: Option<PersistedPoolMetaV3Previous>,
21734            future_guard: u64,
21735        }
21736
21737        let mut data = Vec::new();
21738        data.write_u16::<LittleEndian>(POOL_META_FORMAT)
21739            .expect("pool metadata format should encode");
21740        data.write_u16::<LittleEndian>(POOL_META_GENERATION_VERSION)
21741            .expect("pool metadata version should encode");
21742        FuturePoolMetaV3 {
21743            version: POOL_META_GENERATION_VERSION,
21744            cluster_id: uuid::Uuid::new_v4().to_string(),
21745            epoch: POOL_META_INITIAL_EPOCH,
21746            generation: 1,
21747            transaction_id: uuid::Uuid::new_v4().to_string(),
21748            committed: true,
21749            pools: Vec::new(),
21750            previous: None,
21751            future_guard: 1,
21752        }
21753        .serialize(&mut Serializer::new(&mut data))
21754        .expect("future V3 payload should encode");
21755
21756        assert!(matches!(decode_pool_meta_replica(data), PoolMetaReplica::Incompatible(_)));
21757    }
21758
21759    #[test]
21760    fn pool_meta_identity_classifies_corrupt_incompatible_and_divergent_replicas() {
21761        let cluster_id = uuid::Uuid::new_v4();
21762        let identity = PersistedPoolMetaIdentity {
21763            version: POOL_META_IDENTITY_VERSION,
21764            cluster_id,
21765            epoch: POOL_META_INITIAL_EPOCH,
21766            initialized: true,
21767            fresh_bootstrap_nonce: None,
21768        };
21769        let selected = select_pool_meta_identity(
21770            vec![
21771                PoolMetaIdentityRead {
21772                    replica: PoolMetaIdentityReplica::Corrupt("truncated".to_string()),
21773                    cas: PoolMetaCasToken::Existing("corrupt".to_string()),
21774                },
21775                PoolMetaIdentityRead {
21776                    replica: PoolMetaIdentityReplica::Valid(identity),
21777                    cas: PoolMetaCasToken::Existing("valid".to_string()),
21778                },
21779            ],
21780            cluster_id,
21781        )
21782        .expect("a verified identity backup should survive a corrupt first replica");
21783        assert_eq!(selected.identity, Some(identity));
21784        assert!(selected.needs_repair);
21785
21786        let incompatible = select_pool_meta_identity(
21787            vec![PoolMetaIdentityRead {
21788                replica: PoolMetaIdentityReplica::Incompatible("future version".to_string()),
21789                cas: PoolMetaCasToken::Existing("future".to_string()),
21790            }],
21791            cluster_id,
21792        )
21793        .expect_err("an incompatible identity must fail closed");
21794        assert!(incompatible.to_string().contains("incompatible"));
21795
21796        let divergent = select_pool_meta_identity(
21797            vec![
21798                PoolMetaIdentityRead {
21799                    replica: PoolMetaIdentityReplica::Valid(identity),
21800                    cas: PoolMetaCasToken::Existing("epoch-1".to_string()),
21801                },
21802                PoolMetaIdentityRead {
21803                    replica: PoolMetaIdentityReplica::Valid(PersistedPoolMetaIdentity {
21804                        epoch: POOL_META_INITIAL_EPOCH + 1,
21805                        ..identity
21806                    }),
21807                    cas: PoolMetaCasToken::Existing("epoch-2".to_string()),
21808                },
21809            ],
21810            cluster_id,
21811        )
21812        .expect_err("identity epoch divergence must require recovery");
21813        assert!(divergent.to_string().contains("recovery required"));
21814    }
21815
21816    #[test]
21817    fn pool_meta_v2_floor_is_sticky_after_observation() {
21818        let meta = pool_meta_replica_test_meta("pool-0");
21819        let data = meta
21820            .encode_config_data_for_v2_gate(false)
21821            .expect("an observed v2 snapshot must not be downgraded");
21822
21823        assert_eq!(LittleEndian::read_u16(&data[2..4]), POOL_META_VERSION);
21824    }
21825
21826    #[test]
21827    fn pool_meta_v1_writer_rejects_unresolved_ledger() {
21828        let mut meta = pool_meta_replica_test_meta("pool-0");
21829        meta.version = POOL_META_V1_VERSION;
21830        meta.pools[0].decommission = Some(PoolDecommissionInfo {
21831            start_time: Some(OffsetDateTime::UNIX_EPOCH),
21832            unresolved_entries: vec![DecommissionUnresolvedEntry {
21833                bucket: "bucket-a".to_string(),
21834                object: "object-a".to_string(),
21835                pool_index: 0,
21836                set_index: 0,
21837                source_generation: OffsetDateTime::UNIX_EPOCH,
21838                candidate_count: 1,
21839                disk_error_count: 1,
21840                observed_at: OffsetDateTime::UNIX_EPOCH,
21841                reason: "metadata_resolution_failed".to_string(),
21842            }],
21843            ..Default::default()
21844        });
21845
21846        let err = meta
21847            .encode_config_data_for_v2_gate(false)
21848            .expect_err("v1 must not drop the unresolved ledger");
21849        assert!(err.to_string().contains("pool metadata V2 is required"));
21850    }
21851
21852    #[test]
21853    fn pool_meta_v1_writer_rejects_capacity_reservations() {
21854        let mut meta = pool_meta_replica_test_meta("pool-0");
21855        meta.version = POOL_META_V1_VERSION;
21856        let reservation = build_decommission_capacity_reservation(
21857            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 1, parity: 0 }, 0, 10, 10),
21858            DecommissionErasureLayout { data: 1, parity: 0 },
21859            uuid::Uuid::new_v4(),
21860            1,
21861            OffsetDateTime::UNIX_EPOCH,
21862        )
21863        .expect("test reservation should be valid");
21864        meta.pools[0].decommission = Some(PoolDecommissionInfo {
21865            start_time: Some(OffsetDateTime::UNIX_EPOCH),
21866            capacity_reservation: Some(reservation),
21867            ..Default::default()
21868        });
21869
21870        let err = meta
21871            .encode_config_data_for_v2_gate(false)
21872            .expect_err("v1 must not drop a distributed capacity reservation");
21873        assert!(
21874            err.to_string()
21875                .contains("pool metadata V2 is required to persist decommission capacity reservations")
21876        );
21877    }
21878
21879    #[test]
21880    fn pool_meta_replica_selection_preserves_observed_v2_floor() {
21881        let selection = select_pool_meta_replica(vec![
21882            decode_pool_meta_replica(pool_meta_persisted_v1_replica_test_data("pool-0")),
21883            decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
21884        ])
21885        .expect("equivalent v1 and v2 replicas should converge");
21886
21887        assert_eq!(selection.meta.version, POOL_META_VERSION);
21888        let data = selection
21889            .meta
21890            .encode_config_data_for_v2_gate(false)
21891            .expect("the selected v2 floor must remain writable");
21892        assert_eq!(LittleEndian::read_u16(&data[2..4]), POOL_META_VERSION);
21893    }
21894
21895    #[test]
21896    fn pool_meta_replica_selection_distinguishes_absent_from_unrecoverable() {
21897        assert!(matches!(decode_pool_meta_replica(Vec::new()), PoolMetaReplica::Corrupt(_)));
21898
21899        let empty = select_pool_meta_replica(vec![PoolMetaReplica::Missing, PoolMetaReplica::Missing])
21900            .expect("all missing replicas should preserve new-deployment behavior");
21901        assert!(empty.meta.pools.is_empty());
21902        assert!(!empty.replica_state.needs_repair);
21903        assert!(empty.replica_state.repair_write_safe);
21904
21905        let err = select_pool_meta_replica(vec![
21906            PoolMetaReplica::Missing,
21907            PoolMetaReplica::Unreadable(Arc::new(Error::other("read quorum unavailable"))),
21908        ])
21909        .expect_err("an unreadable replica must not be treated as a new deployment");
21910        assert!(err.to_string().contains("no valid committed replica is available"));
21911        assert!(err.to_string().contains("pool 1 is unreadable"));
21912    }
21913
21914    #[test]
21915    fn pool_meta_replica_selection_blocks_repair_for_unreadable_copy() {
21916        let selection = select_pool_meta_replica(vec![
21917            decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
21918            PoolMetaReplica::Unreadable(Arc::new(Error::other("read quorum unavailable"))),
21919        ])
21920        .expect("a validated replica should remain usable while another copy is unreadable");
21921
21922        assert!(selection.replica_state.needs_repair);
21923        assert!(!selection.replica_state.repair_write_safe);
21924    }
21925
21926    #[test]
21927    fn pool_meta_write_state_remains_blocked_after_unreadable_replica() {
21928        let mut write_state = PoolMetaWriteState::default();
21929        write_state.observe_replicas(PoolMetaReplicaState {
21930            needs_repair: true,
21931            repair_write_safe: false,
21932        });
21933        write_state.observe_replicas(PoolMetaReplicaState {
21934            needs_repair: false,
21935            repair_write_safe: true,
21936        });
21937
21938        let err = write_state
21939            .ensure_write_safe("pool metadata save failed")
21940            .expect_err("a later clean read must not clear the startup write block");
21941        assert!(
21942            err.to_string()
21943                .contains("restart after all replicas are readable and consistent")
21944        );
21945    }
21946
21947    #[test]
21948    fn pool_meta_read_probe_does_not_latch_writer_state() {
21949        let write_state = PoolMetaWriteState::default();
21950        select_pool_meta_replicas_for_read_probe(
21951            &write_state,
21952            vec![PoolMetaReplica::Unreadable(Arc::new(Error::Timeout))],
21953            "capacity probe",
21954        )
21955        .expect_err("an unreadable probe replica must fail the current admission");
21956
21957        assert!(
21958            write_state.ensure_write_safe("ordinary object write").is_ok(),
21959            "a read-only capacity probe must not permanently latch the pool metadata writer"
21960        );
21961    }
21962
21963    #[test]
21964    fn pool_meta_preflight_drop_does_not_arm_but_started_drop_retains_first_failure() {
21965        let state = PoolMetaWriteState::default();
21966        drop(state.arm_transaction());
21967        state.ensure_write_safe("preflight retry").unwrap();
21968        let mut first = state.arm_transaction();
21969        first.phase = Some("identity_cas");
21970        let err = first.failure(Error::Timeout);
21971        drop(first);
21972        let context = state.ensure_write_safe("retry").unwrap_err();
21973        let first_failure = context.pool_metadata_failure().unwrap();
21974        assert_eq!(first_failure.kind, crate::error::PoolMetadataFailure::TransactionUnknown);
21975        assert_eq!(first_failure.phase, "identity_cas");
21976        assert!(matches!(first_failure.source.as_deref(), Some(Error::Timeout)));
21977        assert!(matches!(err.pool_metadata_failure().unwrap().source.as_deref(), Some(Error::Timeout)));
21978        let mut late = state.arm_transaction();
21979        late.phase = Some("commit_cas");
21980        drop(late);
21981        let later = state.ensure_write_safe("retry").unwrap_err();
21982        assert_eq!(later.pool_metadata_failure().unwrap().since, first_failure.since);
21983        assert_eq!(later.pool_metadata_failure().unwrap().phase, "identity_cas");
21984        assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
21985    }
21986
21987    #[tokio::test]
21988    #[serial_test::serial]
21989    async fn pool_meta_writer_retries_after_identity_read_failure_without_restart() {
21990        let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
21991        let mut disks = Vec::new();
21992        for set in &store.pools[1].disk_set {
21993            let mut guard = set.disks.write().await;
21994            let disk_count = guard.len();
21995            let original = std::mem::replace(&mut *guard, vec![None; disk_count]);
21996            disks.push((set.clone(), original));
21997        }
21998        let mut state = store.pool_meta_save_gate.lock().await;
21999        let err = store
22000            .acquire_pool_meta_write_guard(&mut state, "writer preflight")
22001            .await
22002            .unwrap_err();
22003        assert_eq!(
22004            err.pool_metadata_failure().unwrap().kind,
22005            crate::error::PoolMetadataFailure::ReadUnavailable
22006        );
22007        state.ensure_write_safe("retry").unwrap();
22008        for (set, original) in disks {
22009            *set.disks.write().await = original;
22010        }
22011        let (_guard, _) = store.acquire_pool_meta_write_guard(&mut state, "writer retry").await.unwrap();
22012    }
22013
22014    #[tokio::test]
22015    #[serial_test::serial]
22016    async fn pool_meta_runtime_recovery_publishes_durable_state_and_drains_old_workers() {
22017        let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
22018        let before = store.pool_meta.read().await.clone();
22019        let mut requested = before.clone();
22020        requested.pools[0].last_update += Duration::seconds(1);
22021        requested.version = POOL_META_GENERATION_VERSION;
22022        {
22023            let mut state = store.pool_meta_save_gate.lock().await;
22024            let (fence, _) = store.acquire_pool_meta_write_guard(&mut state, "test save").await.unwrap();
22025            let outcome = requested
22026                .save_no_lock_armed(store.pools.clone(), &mut state, fence.lock_lost_signal(), &[0])
22027                .await
22028                .unwrap();
22029            assert_eq!(outcome.committed.pools[0].last_update, requested.pools[0].last_update);
22030            drop(outcome);
22031        }
22032        assert!(!store.pool_meta_writes_ready().await);
22033        let worker = DecommissionCanceler::new(CancellationToken::new());
22034        store.decommission_cancelers.write().await[0] = Some(worker.clone());
22035        assert!(store.recover_pool_meta_transaction().await.is_err());
22036        assert!(worker.is_cancelled());
22037        assert!(!store.pool_meta_writes_ready().await);
22038        assert_eq!(store.pool_meta.read().await.pools[0].last_update, before.pools[0].last_update);
22039        worker.release();
22040        store.pool_meta.write().await.pools[0].last_update = requested.pools[0].last_update + Duration::seconds(10);
22041        // An outcome may outlive the save mutex. Its Drop must precede recovery.
22042        let outstanding = store.pool_meta_save_gate.lock().await.arm_transaction();
22043        assert!(!store.recover_pool_meta_transaction().await.unwrap());
22044        drop(outstanding);
22045        assert!(store.recover_pool_meta_transaction().await.unwrap());
22046        assert!(store.pool_meta_writes_ready().await);
22047        let recovered = store.pool_meta_write_gate_status().await;
22048        assert!(recovered.writes_ready);
22049        assert!(!recovered.check_timed_out);
22050        assert!(!recovered.write_blocked);
22051        assert!(!recovered.transaction_aborted);
22052        assert_eq!((recovered.reason, recovered.phase, recovered.since_unix_secs), (None, None, None));
22053        assert_eq!(store.pool_meta.read().await.pools[0].last_update, requested.pools[0].last_update);
22054        assert!(!store.recover_pool_meta_transaction().await.unwrap());
22055        store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
22056        assert!(!store.recover_pool_meta_transaction().await.unwrap());
22057        assert!(!store.pool_meta_writes_ready().await);
22058    }
22059
22060    #[tokio::test]
22061    #[serial_test::serial]
22062    async fn pool_meta_readiness_is_bounded_while_save_gate_is_held() {
22063        let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
22064        let _state = store.pool_meta_save_gate.lock().await;
22065        let result = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_status())
22066            .await
22067            .unwrap();
22068        assert!(matches!(result, Err(Error::Timeout)));
22069        let snapshot = tokio::time::timeout(std::time::Duration::from_secs(1), store.pool_meta_write_gate_status())
22070            .await
22071            .unwrap();
22072        assert!(!snapshot.writes_ready);
22073        assert!(snapshot.check_timed_out);
22074        assert!(!snapshot.write_blocked);
22075        assert!(!snapshot.transaction_aborted);
22076        assert_eq!((snapshot.reason, snapshot.phase, snapshot.since_unix_secs), (None, None, None));
22077    }
22078
22079    #[tokio::test]
22080    #[serial_test::serial]
22081    async fn pool_meta_cancelled_recovery_and_new_integrity_block_never_clear_original_gate() {
22082        let (_dirs, store, _peer) = crate::services::rebalance::test_two_pool_stores(None).await;
22083        let since = OffsetDateTime::from_unix_timestamp(1_700_000_000).unwrap();
22084        {
22085            let state = store.pool_meta_save_gate.lock().await;
22086            let mut arm = state.arm_transaction();
22087            arm.phase = Some("publication");
22088            drop(arm);
22089            // An old fixed timestamp detects polling-time resets without sleeping.
22090            state.transaction_failure.lock().unwrap().as_mut().unwrap().since = since;
22091            *state.block_started_at.lock().unwrap() = Some(since);
22092        }
22093        let original = store.pool_meta_write_gate_status().await;
22094        assert!(!original.writes_ready);
22095        assert!(!original.check_timed_out);
22096        assert!(!original.write_blocked);
22097        assert!(original.transaction_aborted);
22098        assert_eq!(original.reason, Some("transaction_unknown"));
22099        assert_eq!(original.phase, Some("publication"));
22100        assert_eq!(original.since_unix_secs, Some(since.unix_timestamp()));
22101        assert_eq!(store.pool_meta_write_gate_status().await, original);
22102        let publication_guard = store.pool_meta.write().await;
22103        let recovery = tokio::spawn({
22104            let store = store.clone();
22105            async move { store.recover_pool_meta_transaction().await }
22106        });
22107        tokio::time::timeout(std::time::Duration::from_secs(10), async {
22108            loop {
22109                if store.pool_meta_save_gate.try_lock().is_err() {
22110                    break;
22111                }
22112                tokio::task::yield_now().await;
22113            }
22114        })
22115        .await
22116        .unwrap();
22117        recovery.abort();
22118        assert!(recovery.await.unwrap_err().is_cancelled());
22119        drop(publication_guard);
22120        assert!(!store.pool_meta_writes_ready().await);
22121        assert_eq!(store.pool_meta_write_gate_status().await, original);
22122        let start_guard = store.start_gate.lock().await;
22123        let recovery = tokio::spawn({
22124            let store = store.clone();
22125            async move { store.recover_pool_meta_transaction().await }
22126        });
22127        store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
22128        drop(start_guard);
22129        assert!(!recovery.await.unwrap().unwrap());
22130        assert_eq!(
22131            store
22132                .pool_meta_write_status()
22133                .await
22134                .unwrap_err()
22135                .pool_metadata_failure()
22136                .unwrap()
22137                .kind,
22138            crate::error::PoolMetadataFailure::FenceLost
22139        );
22140        let fenced = store.pool_meta_write_gate_status().await;
22141        assert!(!fenced.writes_ready);
22142        assert!(fenced.write_blocked);
22143        assert_eq!(fenced.reason, Some("fence_lost"));
22144        assert_eq!(fenced.phase, Some("format_heal"));
22145        assert_eq!(fenced.since_unix_secs, original.since_unix_secs);
22146    }
22147
22148    #[test]
22149    fn pool_meta_read_probe_rejects_missing_runtime_metadata_without_latching() {
22150        let write_state = PoolMetaWriteState {
22151            expected_cluster_id: Some(uuid::Uuid::new_v4()),
22152            identity_initialized: Some(true),
22153            ..Default::default()
22154        };
22155        select_pool_meta_replicas_for_read_probe(&write_state, vec![PoolMetaReplica::Missing], "capacity probe")
22156            .expect_err("runtime metadata disappearance must reject the current probe");
22157        write_state
22158            .ensure_write_safe("ordinary object write")
22159            .expect("a missing-metadata probe must not permanently latch the writer");
22160    }
22161
22162    #[tokio::test]
22163    #[serial_test::serial]
22164    async fn pool_meta_read_guard_does_not_latch_after_unreadable_replica() {
22165        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
22166        let mut saved_disks = Vec::new();
22167        for set in &store.pools[1].disk_set {
22168            let mut disks = set.disks.write().await;
22169            let original = std::mem::take(&mut *disks);
22170            let disk_count = original.len();
22171            saved_disks.push((set.clone(), original));
22172            *disks = vec![None; disk_count];
22173        }
22174
22175        let write_state = store.pool_meta_save_gate.lock().await;
22176        store
22177            .acquire_pool_meta_read_guard(&write_state, "capacity probe")
22178            .await
22179            .expect_err("an unreadable metadata replica must reject this probe");
22180        write_state
22181            .ensure_write_safe("ordinary object write")
22182            .expect("a failed read-only probe must remain retryable");
22183
22184        for (set, disks) in saved_disks {
22185            *set.disks.write().await = disks;
22186        }
22187        store
22188            .acquire_pool_meta_read_guard(&write_state, "capacity probe retry")
22189            .await
22190            .expect("a read-only probe must succeed after the replica recovers");
22191    }
22192
22193    #[test]
22194    fn pool_meta_writer_read_failure_is_retryable_when_all_replicas_are_unreadable() {
22195        let replicas = vec![
22196            PoolMetaReplica::Unreadable(Arc::new(Error::other("pool 0 read quorum unavailable"))),
22197            PoolMetaReplica::Unreadable(Arc::new(Error::other("pool 1 read quorum unavailable"))),
22198        ];
22199        let mut write_state = PoolMetaWriteState::default();
22200
22201        let err = select_pool_meta_replicas_observing(&mut write_state, replicas)
22202            .expect_err("all unreadable replicas must fail selection");
22203        assert_eq!(
22204            err.pool_metadata_failure().unwrap().kind,
22205            crate::error::PoolMetadataFailure::ReadUnavailable
22206        );
22207        write_state
22208            .ensure_write_safe("pool metadata save failed")
22209            .expect("a read-only preflight has no unknown writes and must remain retryable");
22210    }
22211
22212    #[test]
22213    fn pool_meta_write_state_blocks_on_any_recovery_required_selection() {
22214        fn assert_selection_blocks(replicas: Vec<PoolMetaReplica>) {
22215            let mut write_state = PoolMetaWriteState::default();
22216            select_pool_meta_replicas_observing(&mut write_state, replicas)
22217                .expect_err("recovery-required replicas must fail selection");
22218            write_state
22219                .ensure_write_safe("pool metadata save failed")
22220                .expect_err("a recovery-required selection must latch the write block");
22221        }
22222
22223        assert_selection_blocks(vec![PoolMetaReplica::Corrupt("truncated".to_string())]);
22224        assert_selection_blocks(vec![PoolMetaReplica::Incompatible("future format".to_string())]);
22225        assert_selection_blocks(vec![
22226            PoolMetaReplica::Corrupt("canonical unavailable".to_string()),
22227            decode_pool_meta_replica(pool_meta_replica_test_data("pool-old")),
22228            decode_pool_meta_replica(pool_meta_replica_test_data("pool-new")),
22229        ]);
22230    }
22231
22232    #[test]
22233    fn pool_meta_replica_selection_rejects_same_version_tuple_extension() {
22234        #[derive(Serialize)]
22235        struct FuturePersistedPoolMeta {
22236            version: u16,
22237            pools: Vec<PersistedPoolStatus>,
22238            generation: u64,
22239        }
22240
22241        let mut data = Vec::new();
22242        data.write_u16::<LittleEndian>(POOL_META_FORMAT)
22243            .expect("pool metadata format should encode");
22244        data.write_u16::<LittleEndian>(POOL_META_VERSION)
22245            .expect("pool metadata version should encode");
22246        FuturePersistedPoolMeta {
22247            version: POOL_META_VERSION,
22248            pools: Vec::new(),
22249            generation: 2,
22250        }
22251        .serialize(&mut Serializer::new(&mut data))
22252        .expect("extended tuple pool metadata should encode");
22253
22254        let err = select_pool_meta_replica(vec![
22255            decode_pool_meta_replica(pool_meta_replica_test_data("pool-0")),
22256            decode_pool_meta_replica(data),
22257        ])
22258        .expect_err("same-version tuple extensions must block fallback repair writes");
22259
22260        assert!(err.to_string().contains("pool 1 is incompatible"));
22261        assert!(err.to_string().contains("version 2 tuple has unsupported field count"));
22262    }
22263
22264    #[test]
22265    fn pool_meta_replica_selection_falls_back_from_truncated_current_tuple() {
22266        let mut truncated = pool_meta_replica_test_data("pool-truncated");
22267        truncated.pop();
22268
22269        let selection = select_pool_meta_replica(vec![
22270            decode_pool_meta_replica(truncated),
22271            decode_pool_meta_replica(pool_meta_replica_test_data("pool-valid")),
22272        ])
22273        .expect("a truncated tuple should not block a validated backup replica");
22274
22275        assert!(selection.replica_state.needs_repair);
22276        assert!(selection.replica_state.repair_write_safe);
22277        assert_eq!(selection.meta.pools[0].cmd_line, "pool-valid");
22278    }
22279
22280    #[test]
22281    fn ensure_pool_not_left_in_cmdline_after_decommission_allows_active_pool() {
22282        assert!(ensure_pool_not_left_in_cmdline_after_decommission(0, "http://node{1...4}/disk{1...4}", false).is_ok());
22283    }
22284
22285    #[test]
22286    fn ensure_pool_not_left_in_cmdline_after_decommission_rejects_completed_pool() {
22287        let err = ensure_pool_not_left_in_cmdline_after_decommission(1, "http://node{1...4}/disk{1...4}", true)
22288            .expect_err("completed decommissioned pool should fail validation");
22289
22290        assert!(
22291            err.to_string()
22292                .contains("pool(2) = http://node{1...4}/disk{1...4} is decommissioned, please remove from server command line")
22293        );
22294    }
22295
22296    #[test]
22297    fn determine_decommission_final_state_marks_failures_and_cancellations() {
22298        assert_eq!(determine_decommission_final_state(0, false), DecommissionFinalState::Complete);
22299        assert_eq!(determine_decommission_final_state(1, false), DecommissionFinalState::Failed);
22300        assert_eq!(determine_decommission_final_state(0, true), DecommissionFinalState::Failed);
22301    }
22302
22303    #[test]
22304    fn lifecycle_action_removes_data_movement_version_rejects_delete_marker_action() {
22305        assert!(!lifecycle_action_removes_data_movement_version(IlmAction::DeleteAction));
22306    }
22307
22308    #[test]
22309    fn lifecycle_action_removes_data_movement_version_accepts_version_delete_actions() {
22310        assert!(lifecycle_action_removes_data_movement_version(IlmAction::DeleteVersionAction));
22311        assert!(lifecycle_action_removes_data_movement_version(IlmAction::DeleteAllVersionsAction));
22312        assert!(lifecycle_action_removes_data_movement_version(
22313            IlmAction::DelMarkerDeleteAllVersionsAction
22314        ));
22315    }
22316
22317    #[test]
22318    fn lifecycle_action_skips_heal_version_for_every_delete_action() {
22319        assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAction));
22320        assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteVersionAction));
22321        assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredAction));
22322        assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteRestoredVersionAction));
22323        assert!(lifecycle_action_skips_heal_version(IlmAction::DeleteAllVersionsAction));
22324        assert!(lifecycle_action_skips_heal_version(IlmAction::DelMarkerDeleteAllVersionsAction));
22325        assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionAction));
22326        assert!(!lifecycle_action_skips_heal_version(IlmAction::TransitionVersionAction));
22327        assert!(!lifecycle_action_skips_heal_version(IlmAction::NoneAction));
22328    }
22329
22330    #[test]
22331    fn resolve_data_movement_lifecycle_expiry_result_allows_dry_run_skip() {
22332        let skip = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, false, false)
22333            .expect("dry-run lifecycle evaluation should not require expiry enqueue");
22334
22335        assert!(skip);
22336    }
22337
22338    #[test]
22339    fn resolve_data_movement_lifecycle_expiry_result_rejects_apply_failure() {
22340        let err = resolve_data_movement_lifecycle_expiry_result(IlmAction::DeleteVersionAction, true, false)
22341            .expect_err("failed lifecycle expiry enqueue should not be treated as skipped");
22342
22343        assert!(err.to_string().contains("failed to apply lifecycle expiry action"));
22344    }
22345
22346    /// The capacity failure classification is the single place that decides
22347    /// between retrying, ignoring, and propagating a decommission failure.
22348    #[test]
22349    fn decommission_capacity_failure_classification_covers_each_class() {
22350        let gate_busy = decommission_capacity_blocked_error(format!(
22351            "{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}2{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}"
22352        ));
22353        assert_eq!(
22354            classify_decommission_capacity_failure(&gate_busy),
22355            DecommissionCapacityFailure::TargetGateBusy { target_pool_index: 2 }
22356        );
22357
22358        let intent_conflict =
22359            decommission_capacity_blocked_error("decommission target mutation has an unresolved target capacity intent");
22360        assert_eq!(
22361            classify_decommission_capacity_failure(&intent_conflict),
22362            DecommissionCapacityFailure::IntentConflict
22363        );
22364
22365        let missing_source = Error::VersionNotFound("bucket".to_string(), "object".to_string(), "version".to_string());
22366        assert_eq!(
22367            classify_decommission_capacity_failure(&missing_source),
22368            DecommissionCapacityFailure::BenignContention
22369        );
22370
22371        let overwrite = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "v1".to_string());
22372        assert_eq!(
22373            classify_decommission_capacity_failure(&overwrite),
22374            DecommissionCapacityFailure::BenignContention
22375        );
22376
22377        let invalid_upload = Error::InvalidUploadID("bucket".to_string(), "object".to_string(), "upload".to_string());
22378        assert_eq!(
22379            classify_decommission_capacity_failure(&invalid_upload),
22380            DecommissionCapacityFailure::BenignContention
22381        );
22382
22383        assert_eq!(
22384            classify_decommission_capacity_failure(&Error::SlowDown),
22385            DecommissionCapacityFailure::Fatal
22386        );
22387        assert_eq!(
22388            classify_decommission_capacity_failure(&decommission_capacity_blocked_error(
22389                "decommission target mutation reservation identity is stale"
22390            )),
22391            DecommissionCapacityFailure::Fatal
22392        );
22393    }
22394
22395    /// A stage wrapper must not hide the underlying capacity class.
22396    #[test]
22397    fn decommission_capacity_failure_classification_sees_through_a_stage_wrapper() {
22398        let gate_busy = decommission_capacity_blocked_error(format!(
22399            "{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}1{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}"
22400        ));
22401        let wrapped =
22402            data_movement::data_movement_stage_error_for_test("decommission_object", "put_object_part", "b", "o", gate_busy);
22403
22404        assert_eq!(
22405            classify_decommission_capacity_failure(&wrapped),
22406            DecommissionCapacityFailure::TargetGateBusy { target_pool_index: 1 }
22407        );
22408        assert!(matches!(
22409            resolve_decommission_entry_failure(wrapped),
22410            Ok(DecommissionEntryOutcome::Deferred(_))
22411        ));
22412    }
22413
22414    #[tokio::test]
22415    #[serial_test::serial]
22416    async fn decommission_deferred_directory_object_is_copied_and_removed() {
22417        use crate::object_api::PutObjReader;
22418        let (_temp_dirs, store, _other_store) =
22419            Box::pin(crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None)).await;
22420        let bucket = "deferred-directory-object";
22421        let object = "folder/";
22422        Box::pin(store.make_bucket(bucket, &MakeBucketOptions::default()))
22423            .await
22424            .expect("create bucket");
22425        let incarnation = store.bucket_incarnation_id(bucket).await.expect("bucket incarnation");
22426        Box::pin(store.pools[0].put_object(
22427            bucket,
22428            &encode_dir_object(object),
22429            &mut PutObjReader::from_vec(Vec::new()),
22430            &ObjectOptions {
22431                expected_bucket_incarnation_id: Some(incarnation),
22432                ..Default::default()
22433            },
22434        ))
22435        .await
22436        .expect("seed directory object");
22437        assert!(
22438            store.pools[0]
22439                .get_disks_by_key(&encode_dir_object(object))
22440                .load_file_info_versions_exact(bucket, object)
22441                .await
22442                .expect("read seeded directory object")
22443                .is_some()
22444        );
22445        let retry_object = "retry.bin";
22446        Box::pin(store.pools[0].put_object(
22447            bucket,
22448            retry_object,
22449            &mut PutObjReader::from_vec(b"retry contents".to_vec()),
22450            &ObjectOptions {
22451                expected_bucket_incarnation_id: Some(incarnation),
22452                ..Default::default()
22453            },
22454        ))
22455        .await
22456        .expect("seed retry object");
22457        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
22458        set_decommission_capacity_info_overrides_for_test(
22459            store.id,
22460            vec![vec![
22461                DecommissionPoolCapacityInfo::for_test(0, layout, 0, 16_384, 16_384),
22462                DecommissionPoolCapacityInfo::for_test(1, layout, 131_072, 131_072, 0),
22463            ]],
22464        );
22465        Box::pin(store.save_current_pool_meta_for_decommission_start(&[0], Vec::new()))
22466            .await
22467            .expect("activate reservation");
22468        let generation = store.active_decommission_generation(0).await.expect("active generation");
22469        let source_set = store.pools[0].get_disks_by_key(&encode_dir_object(object));
22470        let entry_error = Arc::new(tokio::sync::Mutex::new(None));
22471        let outcome = Box::pin(store.clone().replay_deferred_decommission_entry(
22472            CancellationToken::new(),
22473            0,
22474            0,
22475            generation,
22476            DeferredDecommissionEntry {
22477                name: object.into(),
22478                attempts: 2,
22479            },
22480            bucket.into(),
22481            source_set.clone(),
22482            None,
22483            None,
22484            None,
22485            Some(incarnation),
22486            Arc::new(AtomicUsize::new(0)),
22487            Arc::new(Semaphore::new(1)),
22488            entry_error.clone(),
22489        ))
22490        .await
22491        .expect("replay deferred directory object");
22492        assert!(matches!(outcome, DecommissionEntryOutcome::Complete));
22493        assert!(entry_error.lock().await.is_none());
22494        assert!(
22495            source_set
22496                .load_file_info_versions_exact(bucket, object)
22497                .await
22498                .expect("read exact source versions")
22499                .is_none(),
22500            "source object must be removed"
22501        );
22502        let target = store.pools[1]
22503            .get_disks_by_key(&encode_dir_object(object))
22504            .load_file_info_versions_exact(bucket, object)
22505            .await
22506            .expect("read exact target versions")
22507            .expect("directory object must reach target");
22508        assert_eq!(target.versions.len(), 1);
22509        assert_eq!(target.versions[0].size, 0);
22510        let faults = Arc::new(AtomicUsize::new(0));
22511        let observed = faults.clone();
22512        let _fault = DecommissionTestFaultGuard::install(Arc::new(move |stage, b, o, attempt, success| {
22513            if stage == "object_read" && b == bucket && o == retry_object && attempt == 1 && success {
22514                observed.fetch_add(1, Ordering::Relaxed);
22515                return true;
22516            }
22517            false
22518        }));
22519        let mut attempts = 2;
22520        let result = Box::pin(store.decommission_entry(
22521            CancellationToken::new(),
22522            0,
22523            generation,
22524            MetaCacheEntry {
22525                name: retry_object.into(),
22526                ..Default::default()
22527            },
22528            bucket.into(),
22529            store.pools[0].get_disks_by_key(retry_object),
22530            None,
22531            None,
22532            None,
22533            Some(incarnation),
22534            Arc::new(AtomicUsize::new(0)),
22535            &mut attempts,
22536        ))
22537        .await
22538        .expect("retry migration");
22539        assert!(matches!(result, DecommissionEntryOutcome::Complete));
22540        assert_eq!(faults.load(Ordering::Relaxed), 1, "inject one actual read failure");
22541        assert_eq!(attempts, 4, "carry two attempts, then count the entry pass and copy retry");
22542    }
22543
22544    #[test]
22545    fn decommission_object_attempt_metric_keeps_the_peak() {
22546        let mut attempts = 0;
22547        record_decommission_object_attempt(&mut attempts);
22548        record_decommission_object_attempt(&mut attempts);
22549        let mut deferred = DeferredDecommissionEntry {
22550            name: "object".into(),
22551            attempts,
22552        };
22553        record_decommission_object_attempt(&mut deferred.attempts);
22554        assert_eq!(deferred.attempts, 3);
22555        assert!(*DECOMMISSION_OBJECT_ATTEMPTS_MAX.lock().expect("attempt peak lock") >= 3);
22556    }
22557
22558    /// Gate-wait durations are recorded on the same jittered path the retry
22559    /// uses, so the histogram cannot drift from the actual wait.
22560    #[test]
22561    fn decommission_capacity_gate_retry_delay_is_bounded_by_its_ceiling() {
22562        for attempt in 1..=6 {
22563            let delay = decommission_capacity_gate_retry_delay(attempt);
22564            assert!(
22565                delay <= decommission_retry_backoff_ceiling(DECOMMISSION_CAPACITY_GATE_RETRY_BASE, attempt),
22566                "gate wait must stay within its jittered ceiling"
22567            );
22568        }
22569    }
22570
22571    /// A set that still holds a deferred entry must not report success.
22572    #[test]
22573    fn decommission_deferred_drain_invariant_rejects_unresolved_entries() {
22574        assert!(ensure_decommission_deferred_drained("bucket-a", 2, 0).is_ok());
22575
22576        let err = ensure_decommission_deferred_drained("bucket-a", 2, 3)
22577            .expect_err("a set with deferred entries remaining must not complete");
22578        let message = err.to_string();
22579        assert!(message.contains("bucket-a"), "{message}");
22580        assert!(message.contains('2'), "{message}");
22581        assert!(message.contains('3'), "{message}");
22582    }
22583
22584    /// Contention defers, benign contention completes without counting a
22585    /// failure, and fatal errors stay terminal. Folding either of the first two
22586    /// into the fatal arm would pause a healthy bucket, and folding a fatal
22587    /// error into a deferral would hide a real migration failure.
22588    #[test]
22589    fn decommission_entry_failure_resolution_covers_each_class() {
22590        let gate_busy = decommission_capacity_blocked_error(format!(
22591            "{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}1{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}"
22592        ));
22593        let intent_conflict =
22594            decommission_capacity_blocked_error("decommission target mutation has an unresolved target capacity intent");
22595        for err in [gate_busy, intent_conflict] {
22596            let outcome = resolve_decommission_entry_failure(err.clone())
22597                .expect("transient capacity contention must defer, not fail the entry");
22598            match outcome {
22599                DecommissionEntryOutcome::Deferred(retained) => assert_eq!(retained, err),
22600                DecommissionEntryOutcome::Complete => panic!("contention must not complete the entry"),
22601            }
22602        }
22603
22604        for benign in [
22605            Error::ObjectNotFound("bucket".to_string(), "object".to_string()),
22606            Error::VersionNotFound("bucket".to_string(), "object".to_string(), "version".to_string()),
22607            Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "v1".to_string()),
22608            Error::InvalidUploadID("bucket".to_string(), "object".to_string(), "upload".to_string()),
22609        ] {
22610            assert!(
22611                matches!(resolve_decommission_entry_failure(benign.clone()), Ok(DecommissionEntryOutcome::Complete)),
22612                "benign contention must complete without failing the entry: {benign}"
22613            );
22614        }
22615
22616        assert!(resolve_decommission_entry_failure(Error::SlowDown).is_err());
22617        assert!(resolve_decommission_entry_failure(Error::OperationCanceled).is_err());
22618    }
22619
22620    #[test]
22621    fn decommission_copy_cleanup_safe_error_accepts_missing_source_errors() {
22622        assert!(is_decommission_copy_cleanup_safe_error(&Error::ObjectNotFound(
22623            "bucket".to_string(),
22624            "object".to_string()
22625        )));
22626        assert!(is_decommission_copy_cleanup_safe_error(&Error::VersionNotFound(
22627            "bucket".to_string(),
22628            "object".to_string(),
22629            "version".to_string()
22630        )));
22631    }
22632
22633    #[test]
22634    fn decommission_free_version_attempt_treats_missing_source_as_consumed() {
22635        let attempt =
22636            classify_decommission_free_version_attempt(Err(Error::ObjectNotFound("bucket".to_string(), "object".to_string())));
22637
22638        assert!(matches!(attempt, DecommissionFreeVersionAttempt::Consumed));
22639    }
22640
22641    #[test]
22642    fn decommission_free_version_attempt_preserves_capacity_failure() {
22643        let attempt = classify_decommission_free_version_attempt(Err(Error::DiskFull));
22644
22645        assert!(matches!(attempt, DecommissionFreeVersionAttempt::CapacityFailure(Error::DiskFull)));
22646    }
22647
22648    #[test]
22649    fn decommission_delete_marker_copy_error_rejects_data_movement_overwrite() {
22650        let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
22651
22652        assert!(!is_decommission_copy_cleanup_safe_error(&err));
22653    }
22654
22655    #[test]
22656    fn decommission_remote_tiered_copy_error_rejects_data_movement_overwrite() {
22657        let err = Error::DataMovementOverwriteErr("bucket".to_string(), "object".to_string(), "version".to_string());
22658
22659        assert!(!is_decommission_copy_cleanup_safe_error(&err));
22660    }
22661
22662    #[test]
22663    fn decommission_target_capacity_error_accepts_direct_capacity_errors() {
22664        assert!(is_decommission_target_capacity_error(&Error::DiskFull));
22665        assert!(is_decommission_target_capacity_error(&Error::StorageFull));
22666    }
22667
22668    /// The decommission loop classifies errors that came back through a
22669    /// data-movement stage wrapper. Before backlog#1827 T2 the wrapper flattened
22670    /// everything into `Error::other(String)`, so these two classifiers had to
22671    /// match on rendered text; now the wrapped error is recoverable by type.
22672    #[test]
22673    fn decommission_classifiers_see_through_a_stage_wrapper() {
22674        let wrap = |inner: Error| {
22675            data_movement::data_movement_stage_error_for_test("decommission_object", "put_object", "bucket-a", "object-a", inner)
22676        };
22677
22678        // Capacity: the target pool filling up must still stop the loop.
22679        assert!(is_decommission_target_capacity_error(&wrap(Error::DiskFull)));
22680        assert!(is_decommission_target_capacity_error(&wrap(Error::StorageFull)));
22681        assert!(!is_decommission_target_capacity_error(&wrap(Error::SlowDown)));
22682
22683        let gate_busy = decommission_capacity_blocked_error(format!(
22684            "{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}7{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}"
22685        ));
22686        assert_eq!(decommission_capacity_target_gate_busy_index(&wrap(gate_busy)), Some(7));
22687
22688        // Cleanup safety: a not-found surfacing from inside a stage is the same
22689        // condition as one surfacing directly, so the source entry stays
22690        // eligible for cleanup.
22691        let not_found = Error::ObjectNotFound("bucket-a".to_string(), "object-a".to_string());
22692        assert!(is_decommission_copy_cleanup_safe_error(&not_found));
22693        assert!(is_decommission_copy_cleanup_safe_error(&wrap(not_found)));
22694        assert!(!is_decommission_copy_cleanup_safe_error(&wrap(Error::SlowDown)));
22695    }
22696
22697    #[test]
22698    fn decommission_target_gate_retry_recognizes_multipart_part_errors() {
22699        let wrap = |inner: Error| {
22700            data_movement::data_movement_part_stage_error_for_test(
22701                "decommission_object",
22702                "put_object_part",
22703                "bucket-a",
22704                "object-a",
22705                1,
22706                inner,
22707            )
22708        };
22709        let gate_busy_message =
22710            format!("{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_PREFIX}7{DECOMMISSION_CAPACITY_TARGET_GATE_BUSY_SUFFIX}");
22711        let wrapped = wrap(decommission_capacity_blocked_error(&gate_busy_message));
22712        assert!(is_decommission_capacity_target_gate_busy(&wrapped));
22713        assert_eq!(decommission_capacity_target_gate_busy_index(&wrapped), Some(7));
22714        assert_eq!(
22715            wrapped.to_string(),
22716            format!(
22717                "Io error: decommission_object: put_object_part failed for bucket-a/object-a part 1: {}",
22718                decommission_capacity_blocked_error(&gate_busy_message)
22719            )
22720        );
22721
22722        for unrelated in [
22723            Error::SlowDown,
22724            Error::DiskFull,
22725            decommission_capacity_blocked_error("target capacity is exhausted"),
22726            Error::other(gate_busy_message),
22727        ] {
22728            let wrapped = wrap(unrelated);
22729            assert!(!is_decommission_capacity_target_gate_busy(&wrapped));
22730            assert_eq!(decommission_capacity_target_gate_busy_index(&wrapped), None);
22731        }
22732        for missing_target in [
22733            Error::FileNotFound,
22734            Error::ObjectNotFound("bucket-a".to_string(), "object-a".to_string()),
22735            Error::VersionNotFound("bucket-a".to_string(), "object-a".to_string(), "version-a".to_string()),
22736        ] {
22737            assert!(is_decommission_copy_cleanup_safe_error(&missing_target));
22738            assert!(
22739                !is_decommission_copy_cleanup_safe_error(&wrap(missing_target)),
22740                "a missing target part must never authorize source cleanup"
22741            );
22742        }
22743        assert!(is_decommission_target_capacity_error(&wrap(Error::DiskFull)));
22744    }
22745
22746    #[test]
22747    fn decommission_target_capacity_error_accepts_wrapped_capacity_errors() {
22748        let disk_full = Error::other(format!("decommission_object: put_object failed for bucket/object: {}", Error::DiskFull));
22749        let storage_full = Error::other(format!(
22750            "decommission_object: put_object failed for bucket/object: {}",
22751            Error::StorageFull
22752        ));
22753
22754        assert!(is_decommission_target_capacity_error(&disk_full));
22755        assert!(is_decommission_target_capacity_error(&storage_full));
22756    }
22757
22758    #[test]
22759    fn decommission_capacity_intent_conflict_accepts_context_wrapped_error() {
22760        let err = with_decommission_entry_context(
22761            "migrate_object",
22762            "bucket",
22763            "object",
22764            decommission_capacity_blocked_error("target has an unresolved target capacity intent"),
22765        );
22766
22767        assert!(is_decommission_capacity_intent_conflict(&err));
22768        assert_eq!(
22769            decommission_capacity_retry_kind(&err, DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS - 1),
22770            Some(DecommissionCapacityRetryKind::IntentConflict)
22771        );
22772        assert_eq!(
22773            decommission_capacity_retry_kind(&err, DECOMMISSION_CAPACITY_INTENT_CONFLICT_MAX_ATTEMPTS),
22774            None,
22775            "a durable intent conflict must retain its bounded recovery policy"
22776        );
22777    }
22778
22779    #[test]
22780    fn decommission_target_capacity_error_rejects_unrelated_errors() {
22781        assert!(!is_decommission_target_capacity_error(&Error::SlowDown));
22782    }
22783
22784    #[test]
22785    fn decommission_scanner_backlog_handoff_retry_is_typed_and_bounded() {
22786        use data_movement::scanner_backlog::ScannerPauseBacklogRetirementError;
22787
22788        let handoff = data_movement::data_movement_context_error(
22789            "scanner pause backlog retirement planner failed: competing proofs".to_string(),
22790            ScannerPauseBacklogRetirementError::HandoffConflict {
22791                reason: "competing maximum-membership commit proofs".to_string(),
22792            },
22793        );
22794        assert!(is_decommission_scanner_backlog_handoff_conflict(&handoff));
22795        assert_eq!(
22796            decommission_scanner_backlog_retry_kind(&handoff, DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS - 1),
22797            Some(DecommissionScannerBacklogRetryKind::HandoffConflict)
22798        );
22799        assert_eq!(
22800            decommission_scanner_backlog_retry_kind(&handoff, DECOMMISSION_SCANNER_BACKLOG_HANDOFF_MAX_ATTEMPTS),
22801            None,
22802            "a divergent handoff must not retry forever"
22803        );
22804
22805        for non_retryable in [
22806            ScannerPauseBacklogRetirementError::AuthorityConflict {
22807                reason: "larger source authority".to_string(),
22808            },
22809            ScannerPauseBacklogRetirementError::InvalidRecord {
22810                reason: "unread member".to_string(),
22811            },
22812        ] {
22813            let err = data_movement::data_movement_context_error("planner failed".to_string(), non_retryable);
22814            assert!(!is_decommission_scanner_backlog_handoff_conflict(&err));
22815            assert_eq!(
22816                decommission_scanner_backlog_retry_kind(&err, 0),
22817                None,
22818                "only a handoff conflict may be retried"
22819            );
22820        }
22821    }
22822
22823    #[test]
22824    fn should_skip_decommission_delete_marker_characterizes_empty_marker_without_replication() {
22825        let version = rustfs_filemeta::FileInfo {
22826            deleted: true,
22827            ..Default::default()
22828        };
22829
22830        assert!(should_skip_decommission_delete_marker(&version, 1, false));
22831    }
22832
22833    #[test]
22834    fn should_skip_decommission_delete_marker_characterizes_replication_configured() {
22835        let version = rustfs_filemeta::FileInfo {
22836            deleted: true,
22837            ..Default::default()
22838        };
22839
22840        assert!(!should_skip_decommission_delete_marker(&version, 1, true));
22841    }
22842
22843    #[test]
22844    fn should_skip_decommission_delete_marker_rejects_non_deleted_versions() {
22845        let version = rustfs_filemeta::FileInfo::default();
22846
22847        assert!(!should_skip_decommission_delete_marker(&version, 1, false));
22848    }
22849
22850    #[test]
22851    fn should_skip_decommission_delete_marker_rejects_multiple_remaining_versions() {
22852        let version = rustfs_filemeta::FileInfo {
22853            deleted: true,
22854            ..Default::default()
22855        };
22856
22857        assert!(!should_skip_decommission_delete_marker(&version, 2, false));
22858    }
22859
22860    #[test]
22861    fn decommission_delete_marker_opts_preserves_replication_state() {
22862        let mod_time = OffsetDateTime::now_utc();
22863        let version = rustfs_filemeta::FileInfo {
22864            mod_time: Some(mod_time),
22865            replication_state_internal: Some(crate::bucket::replication::replication_state_to_filemeta(&ReplicationState {
22866                replica_status: ReplicationStatusType::Replica,
22867                delete_marker: true,
22868                replicate_decision_str: "existing".to_string(),
22869                ..Default::default()
22870            })),
22871            ..Default::default()
22872        };
22873
22874        let incarnation = uuid::Uuid::new_v4();
22875        let opts = decommission_delete_marker_opts(&version, Some("version-id".to_string()), 7, Some(incarnation));
22876        let replication = opts.delete_replication.expect("replication state should be preserved");
22877
22878        assert!(opts.versioned);
22879        assert!(opts.data_movement);
22880        assert!(opts.delete_marker);
22881        assert!(opts.skip_decommissioned);
22882        assert_eq!(opts.src_pool_idx, 7);
22883        assert_eq!(opts.version_id.as_deref(), Some("version-id"));
22884        assert_eq!(opts.mod_time, Some(mod_time));
22885        assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
22886        assert_eq!(replication.replica_status, ReplicationStatusType::Replica);
22887        assert!(replication.delete_marker);
22888        assert_eq!(replication.replicate_decision_str, "existing");
22889    }
22890
22891    #[test]
22892    fn decommission_delete_marker_opts_preserves_suspended_null_version() {
22893        let version = rustfs_filemeta::FileInfo {
22894            name: "object".to_string(),
22895            deleted: true,
22896            ..Default::default()
22897        };
22898        let opts = decommission_delete_marker_opts(&version, None, 7, None);
22899
22900        assert!(!opts.versioned);
22901        assert!(opts.version_suspended);
22902        assert_eq!(opts.version_id.as_deref(), Some(uuid::Uuid::nil().to_string().as_str()));
22903
22904        let owner = DecommissionCapacityOwner {
22905            source_pool_index: 7,
22906            operation_id: uuid::Uuid::new_v4(),
22907            generation: 1,
22908            owner_nonce: uuid::Uuid::new_v4(),
22909            mutation_id: None,
22910        };
22911        assert_eq!(
22912            decommission_capacity_version_mutation_id(owner, "bucket", &version),
22913            decommission_capacity_mutation_id(
22914                owner,
22915                "bucket",
22916                &version.name,
22917                opts.version_id.as_deref(),
22918                opts.delete_marker,
22919                opts.mod_time,
22920            )
22921        );
22922    }
22923
22924    #[test]
22925    fn test_decommission_object_migration_read_opts_are_raw_data_movement() {
22926        let opts = decommission_object_migration_read_opts(Some("vid-1".to_string()));
22927
22928        assert_eq!(opts.version_id.as_deref(), Some("vid-1"));
22929        assert!(opts.no_lock);
22930        assert!(opts.data_movement);
22931        assert!(opts.raw_data_movement_read);
22932        assert!(opts.skip_rebalancing);
22933        assert!(opts.skip_decommissioned);
22934    }
22935
22936    #[test]
22937    fn decommission_remote_tiered_opts_preserves_versioning_context() {
22938        let mod_time = OffsetDateTime::now_utc();
22939        let version = rustfs_filemeta::FileInfo {
22940            mod_time: Some(mod_time),
22941            metadata: HashMap::from([("x-amz-meta-key".to_string(), "value".to_string())]),
22942            ..Default::default()
22943        };
22944
22945        let incarnation = uuid::Uuid::new_v4();
22946        let opts = decommission_remote_tiered_opts(&version, Some("version-id".to_string()), 9, Some(incarnation));
22947
22948        assert!(opts.versioned);
22949        assert!(opts.data_movement);
22950        assert_eq!(opts.src_pool_idx, 9);
22951        assert_eq!(opts.version_id.as_deref(), Some("version-id"));
22952        assert_eq!(opts.mod_time, Some(mod_time));
22953        assert_eq!(opts.user_defined.get("x-amz-meta-key").map(String::as_str), Some("value"));
22954        assert!(opts.include_part_checksums);
22955        assert!(opts.http_preconditions.is_some());
22956        assert_eq!(opts.expected_bucket_incarnation_id, Some(incarnation));
22957        assert!(!opts.incl_free_versions);
22958
22959        let mut free_version = version;
22960        free_version.set_tier_free_version();
22961        let free_opts = decommission_remote_tiered_opts(&free_version, Some("free-version-id".to_string()), 9, Some(incarnation));
22962        assert!(free_opts.incl_free_versions);
22963    }
22964
22965    #[test]
22966    fn decommission_terminal_state_transitions_update_start_time() {
22967        let start_time = OffsetDateTime::now_utc();
22968        let build_pool_meta = || PoolMeta {
22969            version: POOL_META_VERSION,
22970            pools: vec![PoolStatus {
22971                id: 0,
22972                cmd_line: "/tmp/pool".to_string(),
22973                last_update: start_time,
22974                decommission: Some(PoolDecommissionInfo {
22975                    start_time: Some(start_time),
22976                    ..Default::default()
22977                }),
22978            }],
22979            dont_save: true,
22980        };
22981
22982        let mut pool_meta = build_pool_meta();
22983        assert!(pool_meta.decommission_failed(0));
22984        assert_eq!(pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), None);
22985
22986        let mut pool_meta = build_pool_meta();
22987        assert!(pool_meta.decommission_complete(0));
22988        assert_eq!(
22989            pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time),
22990            Some(start_time)
22991        );
22992
22993        let mut pool_meta = build_pool_meta();
22994        assert!(pool_meta.decommission_cancel(0));
22995        assert_eq!(pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), None);
22996
22997        let mut pool_meta = build_pool_meta();
22998        assert!(pool_meta.decommission_cancel(0));
22999        assert!(!pool_meta.decommission_complete(0));
23000
23001        let mut pool_meta = build_pool_meta();
23002        assert!(pool_meta.decommission_failed(0));
23003        assert!(!pool_meta.decommission_complete(0));
23004    }
23005
23006    #[test]
23007    fn decommission_capacity_reporting_preserves_existing_disk_layout() {
23008        // Frozen V2/V3 reader contract from before capacity contention reporting.
23009        #[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)]
23010        #[serde(deny_unknown_fields)]
23011        struct PreContentionDecommissionInfo {
23012            #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
23013            pub start_time: Option<OffsetDateTime>,
23014            #[serde(rename = "startSize")]
23015            pub start_size: usize,
23016            #[serde(rename = "totalSize")]
23017            pub total_size: usize,
23018            #[serde(rename = "currentSize")]
23019            pub current_size: usize,
23020            #[serde(rename = "complete")]
23021            pub complete: bool,
23022            #[serde(rename = "failed")]
23023            pub failed: bool,
23024            #[serde(rename = "canceled")]
23025            pub canceled: bool,
23026            #[serde(rename = "queued", default)]
23027            pub queued: bool,
23028            #[serde(rename = "queuedBuckets", default)]
23029            pub queued_buckets: Vec<String>,
23030            #[serde(rename = "decommissionedBuckets", default)]
23031            pub decommissioned_buckets: Vec<String>,
23032            #[serde(rename = "bucket", default)]
23033            pub bucket: String,
23034            #[serde(rename = "prefix", default)]
23035            pub prefix: String,
23036            #[serde(rename = "object", default)]
23037            pub object: String,
23038            #[serde(rename = "objectsDecommissioned")]
23039            pub items_decommissioned: usize,
23040            #[serde(rename = "objectsDecommissionedFailed")]
23041            pub items_decommission_failed: usize,
23042            #[serde(rename = "bytesDecommissioned")]
23043            pub bytes_done: usize,
23044            #[serde(rename = "bytesDecommissionedFailed")]
23045            pub bytes_failed: usize,
23046            #[serde(rename = "terminalReloadAttemptAt", with = "time::serde::rfc3339::option", default)]
23047            pub terminal_reload_attempt_at: Option<OffsetDateTime>,
23048            #[serde(rename = "terminalReloadFailures", default)]
23049            pub terminal_reload_failures: Vec<String>,
23050            #[serde(rename = "unresolvedEntries", default)]
23051            pub unresolved_entries: Vec<DecommissionUnresolvedEntry>,
23052            #[serde(rename = "capacityReservation", default)]
23053            pub capacity_reservation: Option<DecommissionCapacityReservation>,
23054            #[serde(rename = "capacityBlockedReason", default)]
23055            pub capacity_blocked_reason: Option<String>,
23056        }
23057        let current = PersistedPoolDecommissionInfo {
23058            capacity_blocked_reason: Some("target gate busy".into()),
23059            ..Default::default()
23060        };
23061        for bytes in [rmp_serde::to_vec(&current), rmp_serde::to_vec_named(&current)] {
23062            let bytes = bytes.expect("serialize current decommission state");
23063            let old: PreContentionDecommissionInfo =
23064                rmp_serde::from_slice(&bytes).expect("existing V2/V3 readers must accept new writes");
23065            assert_eq!(old.capacity_blocked_reason, current.capacity_blocked_reason);
23066        }
23067    }
23068
23069    #[test]
23070    fn pool_meta_persists_decommission_resume_queues() {
23071        let start_time = OffsetDateTime::now_utc();
23072        let unresolved_entry = DecommissionUnresolvedEntry {
23073            bucket: "bucket-b".to_string(),
23074            object: "prefix/unresolved.txt".to_string(),
23075            pool_index: 0,
23076            set_index: 1,
23077            source_generation: start_time,
23078            candidate_count: 2,
23079            disk_error_count: 1,
23080            observed_at: start_time,
23081            reason: "metadata_resolution_failed".to_string(),
23082        };
23083        let pool_meta = PoolMeta {
23084            version: POOL_META_VERSION,
23085            pools: vec![PoolStatus {
23086                id: 1,
23087                cmd_line: "/data/pool1/disk{1...4}".to_string(),
23088                last_update: start_time,
23089                decommission: Some(PoolDecommissionInfo {
23090                    start_time: Some(start_time),
23091                    queued: true,
23092                    queued_buckets: vec!["bucket-a".to_string(), "bucket-b/prefix".to_string()],
23093                    decommissioned_buckets: vec!["bucket-done".to_string()],
23094                    bucket: "bucket-b".to_string(),
23095                    prefix: "prefix".to_string(),
23096                    object: "object.txt".to_string(),
23097                    items_decommissioned: 7,
23098                    items_decommission_failed: 1,
23099                    bytes_done: 1024,
23100                    bytes_failed: 128,
23101                    terminal_reload_attempt_at: Some(start_time),
23102                    terminal_reload_failures: vec!["complete_decommission: peer node-a failed".to_string()],
23103                    unresolved_entries: vec![unresolved_entry.clone()],
23104                    ..Default::default()
23105                }),
23106            }],
23107            dont_save: false,
23108        };
23109
23110        let mut buf = Vec::new();
23111        PersistedPoolMeta::from(&pool_meta)
23112            .serialize(&mut Serializer::new(&mut buf))
23113            .expect("pool meta should serialize");
23114
23115        let mut deserializer = Deserializer::new(Cursor::new(&buf));
23116        let restored: PoolMeta = PersistedPoolMeta::deserialize(&mut deserializer)
23117            .expect("pool meta should deserialize")
23118            .try_into()
23119            .expect("pool meta should validate");
23120
23121        let restored_decommission = restored.pools[0]
23122            .decommission
23123            .as_ref()
23124            .expect("decommission info should survive round-trip");
23125        assert_eq!(
23126            restored_decommission.queued_buckets,
23127            vec!["bucket-a".to_string(), "bucket-b/prefix".to_string()]
23128        );
23129        assert_eq!(restored_decommission.decommissioned_buckets, vec!["bucket-done".to_string()]);
23130        assert_eq!(restored_decommission.bucket, "bucket-b");
23131        assert_eq!(restored_decommission.prefix, "prefix");
23132        assert_eq!(restored_decommission.object, "object.txt");
23133        assert!(restored_decommission.stage.is_empty());
23134        assert_eq!(restored_decommission.items_decommissioned, 7);
23135        assert_eq!(restored_decommission.items_decommission_failed, 1);
23136        assert_eq!(restored_decommission.bytes_done, 1024);
23137        assert_eq!(restored_decommission.bytes_failed, 128);
23138        assert_eq!(restored_decommission.terminal_reload_attempt_at, Some(start_time));
23139        assert_eq!(
23140            restored_decommission.terminal_reload_failures,
23141            vec!["complete_decommission: peer node-a failed".to_string()]
23142        );
23143        assert_eq!(restored_decommission.unresolved_entries, vec![unresolved_entry]);
23144        assert!(restored_decommission.queued);
23145        assert_eq!(restored_decommission.items_since_last_progress_save(), 0);
23146    }
23147
23148    #[test]
23149    fn pool_meta_records_decommission_terminal_reload_failure_once() {
23150        let start_time = OffsetDateTime::now_utc();
23151        let mut pool_meta = PoolMeta {
23152            version: POOL_META_VERSION,
23153            pools: vec![PoolStatus {
23154                id: 1,
23155                cmd_line: "/data/pool1/disk{1...4}".to_string(),
23156                last_update: start_time,
23157                decommission: Some(PoolDecommissionInfo {
23158                    complete: true,
23159                    ..Default::default()
23160                }),
23161            }],
23162            dont_save: false,
23163        };
23164
23165        assert!(
23166            pool_meta
23167                .record_decommission_terminal_reload_failure(0, "complete_decommission", "peer node-a failed".to_string())
23168                .expect("terminal reload failure should be recorded")
23169        );
23170        assert!(
23171            !pool_meta
23172                .record_decommission_terminal_reload_failure(0, "complete_decommission", "peer node-a failed".to_string())
23173                .expect("duplicate terminal reload failure should be ignored")
23174        );
23175
23176        let decommission = pool_meta.pools[0].decommission.as_ref().expect("decommission should exist");
23177        assert!(decommission.terminal_reload_attempt_at.is_some());
23178        assert_eq!(
23179            decommission.terminal_reload_failures,
23180            vec!["complete_decommission: peer node-a failed".to_string()]
23181        );
23182    }
23183
23184    #[test]
23185    fn pool_meta_decode_supports_legacy_payload() {
23186        let start_time = OffsetDateTime::now_utc();
23187        let legacy_meta = LegacyPoolMeta {
23188            version: POOL_META_V1_VERSION,
23189            pools: vec![LegacyPoolStatus {
23190                id: 3,
23191                cmd_line: "/legacy/pool".to_string(),
23192                last_update: start_time,
23193                decommission: Some(LegacyPoolDecommissionInfo {
23194                    start_time: Some(start_time),
23195                    items_decommissioned: 9,
23196                    items_decommission_failed: 2,
23197                    bytes_done: 2048,
23198                    bytes_failed: 256,
23199                    ..Default::default()
23200                }),
23201            }],
23202            dont_save: true,
23203        };
23204
23205        let mut legacy_payload = Vec::new();
23206        legacy_meta
23207            .serialize(&mut Serializer::new(&mut legacy_payload))
23208            .expect("legacy payload should serialize");
23209
23210        // New persisted schema has fewer top-level fields and should not decode this legacy struct payload.
23211        let persisted_decode: std::result::Result<PersistedPoolMeta, _> = rmp_serde::from_slice(&legacy_payload);
23212        assert!(persisted_decode.is_err());
23213
23214        let decoded =
23215            PoolMeta::decode_pool_meta_payload(POOL_META_V1_VERSION, &legacy_payload).expect("legacy payload should decode");
23216        assert_eq!(decoded.version, POOL_META_V1_VERSION);
23217        assert!(!decoded.dont_save, "runtime-only flag should reset on load");
23218        assert_eq!(decoded.pools.len(), 1);
23219        assert_eq!(decoded.pools[0].id, 3);
23220        assert_eq!(decoded.pools[0].cmd_line, "/legacy/pool");
23221        assert_eq!(decoded.pools[0].last_update, start_time);
23222
23223        let decommission = decoded.pools[0].decommission.as_ref().expect("decommission should decode");
23224        assert_eq!(decommission.start_time, Some(start_time));
23225        assert_eq!(decommission.items_decommissioned, 9);
23226        assert_eq!(decommission.items_decommission_failed, 2);
23227        assert_eq!(decommission.bytes_done, 2048);
23228        assert_eq!(decommission.bytes_failed, 256);
23229        assert_eq!(decommission.items_since_last_progress_save(), 0);
23230        // These fields were skipped in legacy payload and should be defaulted.
23231        assert!(decommission.queued_buckets.is_empty());
23232        assert!(decommission.decommissioned_buckets.is_empty());
23233        assert!(decommission.bucket.is_empty());
23234        assert!(decommission.prefix.is_empty());
23235        assert!(decommission.object.is_empty());
23236        assert!(decommission.unresolved_entries.is_empty());
23237    }
23238
23239    #[test]
23240    fn pool_meta_decode_rejects_unknown_legacy_fields() {
23241        #[derive(Serialize)]
23242        struct LegacyPoolMetaWithUnknownField {
23243            version: u16,
23244            pools: Vec<LegacyPoolStatus>,
23245            dont_save: bool,
23246            unexpected: bool,
23247        }
23248
23249        let payload = rmp_serde::to_vec_named(&LegacyPoolMetaWithUnknownField {
23250            version: POOL_META_V1_VERSION,
23251            pools: Vec::new(),
23252            dont_save: true,
23253            unexpected: true,
23254        })
23255        .expect("legacy pool metadata with unknown field should serialize");
23256
23257        let err = PoolMeta::decode_pool_meta_payload(POOL_META_V1_VERSION, payload.as_slice())
23258            .expect_err("unknown legacy pool metadata field should fail decode");
23259        let rendered = err.to_string();
23260        assert!(rendered.contains("PoolMeta v1 decode failed for both persisted and legacy formats"));
23261        assert!(rendered.contains("unknown field") || rendered.contains("missing field"));
23262    }
23263
23264    #[test]
23265    fn pool_meta_decode_rejects_unknown_persisted_fields() {
23266        #[derive(Serialize)]
23267        struct PersistedPoolMetaWithUnknownField {
23268            version: u16,
23269            pools: Vec<PersistedPoolStatus>,
23270            unexpected: bool,
23271        }
23272
23273        let payload = rmp_serde::to_vec_named(&PersistedPoolMetaWithUnknownField {
23274            version: POOL_META_VERSION,
23275            pools: Vec::new(),
23276            unexpected: true,
23277        })
23278        .expect("pool metadata with unknown field should serialize");
23279
23280        let err = PoolMeta::decode_pool_meta_payload(POOL_META_VERSION, payload.as_slice())
23281            .expect_err("unknown persisted pool metadata field should fail decode");
23282        let rendered = err.to_string();
23283        assert!(rendered.contains("PoolMeta v2 decode failed"));
23284        assert!(rendered.contains("unknown field") || rendered.contains("missing field"));
23285    }
23286
23287    #[test]
23288    fn pool_meta_decode_rejects_missing_critical_persisted_fields() {
23289        #[derive(Serialize)]
23290        struct PersistedPoolMetaWithoutPools {
23291            version: u16,
23292        }
23293
23294        let payload = rmp_serde::to_vec_named(&PersistedPoolMetaWithoutPools {
23295            version: POOL_META_VERSION,
23296        })
23297        .expect("pool metadata without pools should serialize");
23298
23299        let err = PoolMeta::decode_pool_meta_payload(POOL_META_VERSION, payload.as_slice())
23300            .expect_err("missing persisted pool metadata pools should fail decode");
23301        assert!(err.to_string().contains("PoolMeta v2 decode failed"));
23302    }
23303
23304    #[test]
23305    fn pool_meta_decode_rejects_unknown_decommission_fields() {
23306        #[derive(Serialize)]
23307        struct PersistedPoolStatusWithUnknownDecommission {
23308            #[serde(rename = "id")]
23309            id: usize,
23310            #[serde(rename = "cmdline")]
23311            cmd_line: String,
23312            #[serde(rename = "lastUpdate", with = "time::serde::rfc3339")]
23313            last_update: OffsetDateTime,
23314            #[serde(rename = "decommissionInfo")]
23315            decommission: Option<PersistedPoolDecommissionInfoWithUnknownField>,
23316        }
23317
23318        #[derive(Serialize)]
23319        struct PersistedPoolDecommissionInfoWithUnknownField {
23320            #[serde(rename = "startTime", with = "time::serde::rfc3339::option")]
23321            start_time: Option<OffsetDateTime>,
23322            #[serde(rename = "startSize")]
23323            start_size: usize,
23324            #[serde(rename = "totalSize")]
23325            total_size: usize,
23326            #[serde(rename = "currentSize")]
23327            current_size: usize,
23328            #[serde(rename = "complete")]
23329            complete: bool,
23330            #[serde(rename = "failed")]
23331            failed: bool,
23332            #[serde(rename = "canceled")]
23333            canceled: bool,
23334            #[serde(rename = "queuedBuckets")]
23335            queued_buckets: Vec<String>,
23336            #[serde(rename = "decommissionedBuckets")]
23337            decommissioned_buckets: Vec<String>,
23338            #[serde(rename = "bucket")]
23339            bucket: String,
23340            #[serde(rename = "prefix")]
23341            prefix: String,
23342            #[serde(rename = "object")]
23343            object: String,
23344            #[serde(rename = "objectsDecommissioned")]
23345            items_decommissioned: usize,
23346            #[serde(rename = "objectsDecommissionedFailed")]
23347            items_decommission_failed: usize,
23348            #[serde(rename = "bytesDecommissioned")]
23349            bytes_done: usize,
23350            #[serde(rename = "bytesDecommissionedFailed")]
23351            bytes_failed: usize,
23352            #[serde(rename = "unexpected")]
23353            unexpected: bool,
23354        }
23355
23356        #[derive(Serialize)]
23357        struct PersistedPoolMetaWithUnknownDecommission {
23358            version: u16,
23359            pools: Vec<PersistedPoolStatusWithUnknownDecommission>,
23360        }
23361
23362        let start_time = OffsetDateTime::now_utc();
23363        let payload = rmp_serde::to_vec_named(&PersistedPoolMetaWithUnknownDecommission {
23364            version: POOL_META_VERSION,
23365            pools: vec![PersistedPoolStatusWithUnknownDecommission {
23366                id: 0,
23367                cmd_line: "/data/pool".to_string(),
23368                last_update: start_time,
23369                decommission: Some(PersistedPoolDecommissionInfoWithUnknownField {
23370                    start_time: Some(start_time),
23371                    start_size: 0,
23372                    total_size: 0,
23373                    current_size: 0,
23374                    complete: false,
23375                    failed: false,
23376                    canceled: false,
23377                    queued_buckets: Vec::new(),
23378                    decommissioned_buckets: Vec::new(),
23379                    bucket: String::new(),
23380                    prefix: String::new(),
23381                    object: String::new(),
23382                    items_decommissioned: 0,
23383                    items_decommission_failed: 0,
23384                    bytes_done: 0,
23385                    bytes_failed: 0,
23386                    unexpected: true,
23387                }),
23388            }],
23389        })
23390        .expect("pool metadata with unknown decommission field should serialize");
23391
23392        let err = PoolMeta::decode_pool_meta_payload(POOL_META_VERSION, payload.as_slice())
23393            .expect_err("unknown persisted decommission metadata field should fail decode");
23394        assert!(err.to_string().contains("PoolMeta v2 decode failed"));
23395    }
23396
23397    #[test]
23398    fn pool_meta_decode_rejects_invalid_decommission_terminal_state() {
23399        let start_time = OffsetDateTime::now_utc();
23400        let persisted_meta = PersistedPoolMeta {
23401            version: POOL_META_VERSION,
23402            pools: vec![PersistedPoolStatus {
23403                id: 1,
23404                cmd_line: "/data/pool1/disk{1...4}".to_string(),
23405                last_update: start_time,
23406                decommission: Some(PersistedPoolDecommissionInfo {
23407                    start_time: Some(start_time),
23408                    complete: true,
23409                    failed: true,
23410                    canceled: false,
23411                    ..Default::default()
23412                }),
23413            }],
23414        };
23415
23416        let mut payload = Vec::new();
23417        persisted_meta
23418            .serialize(&mut Serializer::new(&mut payload))
23419            .expect("persisted payload should serialize");
23420
23421        let err = PoolMeta::decode_pool_meta_payload(POOL_META_VERSION, &payload)
23422            .expect_err("invalid terminal state should fail decode");
23423        assert!(err.to_string().contains("invalid decommission terminal state"));
23424    }
23425
23426    #[test]
23427    fn pool_meta_decode_rejects_invalid_legacy_decommission_terminal_state() {
23428        let start_time = OffsetDateTime::now_utc();
23429        let legacy_meta = LegacyPoolMeta {
23430            version: POOL_META_V1_VERSION,
23431            pools: vec![LegacyPoolStatus {
23432                id: 1,
23433                cmd_line: "/legacy/pool".to_string(),
23434                last_update: start_time,
23435                decommission: Some(LegacyPoolDecommissionInfo {
23436                    start_time: Some(start_time),
23437                    complete: true,
23438                    failed: false,
23439                    canceled: true,
23440                    ..Default::default()
23441                }),
23442            }],
23443            dont_save: false,
23444        };
23445
23446        let mut payload = Vec::new();
23447        legacy_meta
23448            .serialize(&mut Serializer::new(&mut payload))
23449            .expect("legacy payload should serialize");
23450
23451        let err = PoolMeta::decode_pool_meta_payload(POOL_META_V1_VERSION, &payload)
23452            .expect_err("invalid legacy terminal state should fail decode");
23453        assert!(err.to_string().contains("invalid decommission terminal state"));
23454    }
23455}
23456
23457// impl Fn(MetaCacheEntry) -> impl Future<Output = Result<(), Error>>
23458
23459pub type ListCallback = Arc<dyn Fn(MetaCacheEntry) -> BoxFuture<'static, ()> + Send + Sync + 'static>;
23460
23461const DECOMMISSION_ENTRY_QUEUE_HARD_CAP: usize = 256;
23462
23463struct QueuedDecommissionEntry {
23464    entry: MetaCacheEntry,
23465    queue_permit: OwnedSemaphorePermit,
23466}
23467
23468enum DecommissionEntryEnqueueResult {
23469    Enqueued,
23470    Canceled,
23471    Closed,
23472}
23473
23474fn decommission_entry_queue_capacity(worker_limit: usize) -> usize {
23475    worker_limit.saturating_mul(2).clamp(1, DECOMMISSION_ENTRY_QUEUE_HARD_CAP)
23476}
23477
23478async fn enqueue_decommission_entry(
23479    rx: &CancellationToken,
23480    outstanding: &Arc<Semaphore>,
23481    tx: &mpsc::Sender<QueuedDecommissionEntry>,
23482    entry: MetaCacheEntry,
23483) -> DecommissionEntryEnqueueResult {
23484    let queue_permit = match tokio::select! {
23485        biased;
23486        _ = rx.cancelled() => return DecommissionEntryEnqueueResult::Canceled,
23487        permit = outstanding.clone().acquire_owned() => permit,
23488    } {
23489        Ok(permit) => permit,
23490        Err(_) => return DecommissionEntryEnqueueResult::Closed,
23491    };
23492
23493    let queued = QueuedDecommissionEntry { entry, queue_permit };
23494    tokio::select! {
23495        biased;
23496        _ = rx.cancelled() => DecommissionEntryEnqueueResult::Canceled,
23497        result = tx.send(queued) => {
23498            if result.is_ok() {
23499                DecommissionEntryEnqueueResult::Enqueued
23500            } else {
23501                DecommissionEntryEnqueueResult::Closed
23502            }
23503        }
23504    }
23505}
23506
23507async fn drain_decommission_entry_queue(rx: &CancellationToken, outstanding: &Arc<Semaphore>, capacity: usize) -> bool {
23508    let Ok(permits) = u32::try_from(capacity) else {
23509        return true;
23510    };
23511
23512    tokio::select! {
23513        _ = rx.cancelled() => true,
23514        result = outstanding.acquire_many(permits) => result.is_err(),
23515    }
23516}
23517
23518async fn record_decommission_entry_error(
23519    entry_error: &Arc<tokio::sync::Mutex<Option<Error>>>,
23520    rx: &CancellationToken,
23521    err: Error,
23522) -> bool {
23523    if rx.is_cancelled() {
23524        return false;
23525    }
23526
23527    let mut first_err = entry_error.lock().await;
23528    if first_err.is_none() && !rx.is_cancelled() {
23529        *first_err = Some(err);
23530        rx.cancel();
23531        return true;
23532    }
23533    false
23534}
23535
23536fn ensure_decommission_unresolved_verification_disk_count(
23537    expected: usize,
23538    actual: usize,
23539    pool_index: usize,
23540    set_index: usize,
23541) -> Result<()> {
23542    if actual == expected {
23543        return Ok(());
23544    }
23545    Err(Error::other(format!(
23546        "decommission unresolved-entry verification for pool {pool_index} set {set_index} requires all {expected} source disks, but only {actual} are online"
23547    )))
23548}
23549
23550impl SetDisks {
23551    async fn decommission_unresolved_entry_absent_on_all_disks(
23552        &self,
23553        pool_index: usize,
23554        entry: &DecommissionUnresolvedEntry,
23555    ) -> Result<bool> {
23556        let (disks, _) = self.get_online_disks_with_healing(false).await;
23557        ensure_decommission_unresolved_verification_disk_count(self.set_drive_count, disks.len(), pool_index, entry.set_index)?;
23558
23559        let object = encode_dir_object(&entry.object);
23560        let reads = join_all(disks.iter().map(|disk| disk.read_xl(&entry.bucket, &object, false))).await;
23561        let mut found = false;
23562        for read in reads {
23563            match read {
23564                Ok(_) => found = true,
23565                Err(DiskError::FileNotFound | DiskError::FileVersionNotFound | DiskError::VolumeNotFound) => {}
23566                Err(err) => {
23567                    return Err(Error::other(format!(
23568                        "decommission unresolved-entry verification failed for pool {pool_index} set {} path {}/{}: {err}",
23569                        entry.set_index, entry.bucket, entry.object
23570                    )));
23571                }
23572            }
23573        }
23574        Ok(!found)
23575    }
23576
23577    #[tracing::instrument(skip(self, store, rx, cb_func, entry_error))]
23578    #[allow(clippy::too_many_arguments)]
23579    async fn list_objects_to_decommission(
23580        self: &Arc<Self>,
23581        store: Arc<ECStore>,
23582        rx: CancellationToken,
23583        bucket_info: DecomBucketInfo,
23584        cb_func: ListCallback,
23585        entry_error: Arc<tokio::sync::Mutex<Option<Error>>>,
23586        pool_index: usize,
23587        set_index: usize,
23588        source_generation: OffsetDateTime,
23589        require_all_disks: bool,
23590    ) -> Result<()> {
23591        let (disks, _) = self.get_online_disks_with_healing(false).await;
23592        ensure_decommission_listing_disks_available(!disks.is_empty(), &bucket_info.name)?;
23593        if require_all_disks {
23594            ensure_decommission_unresolved_verification_disk_count(self.set_drive_count, disks.len(), pool_index, set_index)?;
23595        }
23596
23597        let listing_quorum = self.set_drive_count.div_ceil(2);
23598
23599        let resolver = MetadataResolutionParams {
23600            dir_quorum: listing_quorum,
23601            obj_quorum: listing_quorum,
23602            bucket: bucket_info.name.clone(),
23603            ..Default::default()
23604        };
23605
23606        let cb1 = cb_func.clone();
23607        let unresolved_error = entry_error.clone();
23608        let unresolved_rx = rx.clone();
23609        let unresolved_bucket = bucket_info.name.clone();
23610        let unresolved_prefix = bucket_info.prefix.clone();
23611        let unresolved_pool_index = pool_index;
23612        let unresolved_set_index = set_index;
23613        let unresolved_generation = source_generation;
23614        let unresolved_store = store;
23615
23616        list_path_raw(
23617            rx,
23618            ListPathRawOptions {
23619                disks: disks.iter().cloned().map(Some).collect(),
23620                bucket: bucket_info.name.clone(),
23621                path: bucket_info.prefix.clone(),
23622                recursive: true,
23623                min_disks: listing_quorum,
23624                skip_walkdir_total_timeout: true,
23625                walkdir_stall_timeout: Some(DECOMMISSION_BACKGROUND_WALKDIR_STALL_TIMEOUT),
23626                agreed: Some(Box::new(move |entry: MetaCacheEntry| Box::pin(cb1(entry)))),
23627                partial: Some(Box::new(move |entries: MetaCacheEntries, errs: &[Option<DiskError>]| {
23628                    let resolver = resolver.clone();
23629                    let cb_func = cb_func.clone();
23630                    let bucket = unresolved_bucket.clone();
23631                    let prefix = unresolved_prefix.clone();
23632                    let unresolved_error = unresolved_error.clone();
23633                    let unresolved_rx = unresolved_rx.clone();
23634                    let unresolved_store = unresolved_store.clone();
23635                    let pool_index = unresolved_pool_index;
23636                    let set_index = unresolved_set_index;
23637                    let source_generation = unresolved_generation;
23638                    let disk_error_count = errs.iter().flatten().count();
23639                    if unresolved_rx.is_cancelled() {
23640                        return Box::pin(async {});
23641                    }
23642
23643                    match resolve_decommission_partial_listing_entry(
23644                        entries,
23645                        resolver,
23646                        &bucket,
23647                        &prefix,
23648                        disk_error_count,
23649                        pool_index,
23650                        set_index,
23651                        source_generation,
23652                    ) {
23653                        Ok(entry) => {
23654                            warn!("decommission_pool: list_objects_to_decommission get {}", &entry.name);
23655                            Box::pin(async move {
23656                                cb_func(entry).await;
23657                            })
23658                        }
23659                        Err(unresolved_entry) => Box::pin(async move {
23660                            if unresolved_rx.is_cancelled() {
23661                                return;
23662                            }
23663                            let err = decommission_unresolved_listing_error(&unresolved_entry);
23664                            warn!(
23665                                event = EVENT_DECOMMISSION_BUCKET,
23666                                component = LOG_COMPONENT_ECSTORE,
23667                                subsystem = LOG_SUBSYSTEM_POOLS,
23668                                bucket = %bucket,
23669                                prefix = %prefix,
23670                                state = "unresolved_entry",
23671                                error = %err,
23672                                "Decommission listing failed closed on unresolved metadata"
23673                            );
23674                            let err = match unresolved_store
23675                                .persist_decommission_unresolved_entry(pool_index, source_generation, unresolved_entry)
23676                                .await
23677                            {
23678                                Ok(()) => err,
23679                                Err(ledger_err) => Error::other(format!("{err}; {ledger_err}")),
23680                            };
23681                            record_decommission_entry_error(&unresolved_error, &unresolved_rx, err).await;
23682                        }),
23683                    }
23684                })),
23685                ..Default::default()
23686            },
23687        )
23688        .await?;
23689
23690        if let Some(err) = entry_error.lock().await.clone() {
23691            return Err(err);
23692        }
23693
23694        Ok(())
23695    }
23696}
23697
23698fn is_disk_online_state(state: &str) -> bool {
23699    // The disk state strings are produced from rustfs_utils::os::get_drive_stats or DiskError::to_string().
23700    // Conventionally, online is "ok"/"online" (may evolve). Be conservative:
23701    // - Treat empty as unknown -> include it (to avoid dropping capacity).
23702    // - Exclude explicit offline-ish states.
23703    let s = state.trim().to_lowercase();
23704    if s.is_empty() {
23705        return true;
23706    }
23707    if s.contains("offline") {
23708        return false;
23709    }
23710    if s.contains("not found") || s.contains("disk not found") {
23711        return false;
23712    }
23713    true
23714}
23715
23716fn decommission_physical_pool_capacity(
23717    disks: &[rustfs_madmin::Disk],
23718    pool_index: usize,
23719    layout: DecommissionErasureLayout,
23720    logical: PoolSpaceInfo,
23721    owned_physical_used: Option<&HashMap<i32, usize>>,
23722) -> (usize, usize, usize) {
23723    let width = layout.width();
23724    let mut sets: HashMap<i32, Vec<&rustfs_madmin::Disk>> = HashMap::new();
23725    let mut seen = HashSet::new();
23726    for disk in disks {
23727        let state = disk.state.trim().to_ascii_lowercase();
23728        if disk.pool_index != pool_index as i32
23729            || disk.set_index < 0
23730            || disk.disk_index < 0
23731            || disk.disk_index as usize >= width
23732            || !matches!(state.as_str(), "ok" | "online")
23733        {
23734            continue;
23735        }
23736        let identity = (disk.set_index, disk.disk_index);
23737        if seen.insert(identity) {
23738            sets.entry(disk.set_index).or_default().push(disk);
23739        }
23740    }
23741
23742    let mut physical_total = 0usize;
23743    let mut physical_free = 0usize;
23744    let mut physical_used = 0usize;
23745    let mut observed_set = false;
23746    for set_disks in sets.values() {
23747        if set_disks.is_empty() {
23748            continue;
23749        }
23750        observed_set = true;
23751        let min_total = set_disks
23752            .iter()
23753            .map(|disk| disk.total_space as usize)
23754            .min()
23755            .unwrap_or_default();
23756        let min_free = set_disks
23757            .iter()
23758            .map(|disk| disk.available_space as usize)
23759            .min()
23760            .unwrap_or_default();
23761        let physical_used_for_set = owned_physical_used
23762            .and_then(|owned| owned.get(&set_disks[0].set_index).copied())
23763            .unwrap_or_else(|| {
23764                set_disks
23765                    .iter()
23766                    .map(|disk| {
23767                        (disk.used_space as usize).max((disk.total_space as usize).saturating_sub(disk.available_space as usize))
23768                    })
23769                    .max()
23770                    .unwrap_or_default()
23771                    .saturating_mul(width)
23772            });
23773        physical_total = physical_total.saturating_add(min_total.saturating_mul(width));
23774        physical_used = physical_used.saturating_add(physical_used_for_set);
23775        if set_disks.len() >= width {
23776            physical_free = physical_free.saturating_add(min_free.saturating_mul(width));
23777        }
23778    }
23779
23780    if observed_set {
23781        return (physical_total, physical_free, physical_used);
23782    }
23783
23784    let fallback_total = capacity_mul_div_ceil(logical.total, width, layout.data);
23785    let fallback_used = capacity_mul_div_ceil(logical.used, width, layout.data)
23786        .max(fallback_total.saturating_sub(capacity_mul_div_ceil(logical.free, width, layout.data)));
23787    (fallback_total, 0, fallback_used)
23788}
23789
23790#[deprecated(since = "0.1.0", note = "Use fallback_total_capacity_dedup instead")]
23791#[allow(
23792    dead_code,
23793    reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
23794)]
23795fn fallback_total_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
23796    fallback_total_capacity_dedup(disks)
23797}
23798
23799#[deprecated(since = "0.1.0", note = "Use fallback_free_capacity_dedup instead")]
23800#[allow(
23801    dead_code,
23802    reason = "superseded by the replacement named in the comment at pools.rs:5071 (backlog#1823)"
23803)]
23804fn fallback_free_capacity(disks: &[rustfs_madmin::Disk]) -> usize {
23805    fallback_free_capacity_dedup(disks)
23806}
23807
23808pub fn get_total_usable_capacity(disks: &[rustfs_madmin::Disk], info: &rustfs_madmin::StorageInfo) -> usize {
23809    // If backend info is missing or inconsistent, do a safe fallback to avoid reporting nonsense.
23810    if info.backend.standard_sc_data.is_empty() {
23811        return fallback_total_capacity_dedup(disks);
23812    }
23813    let mut capacity = 0usize;
23814    let mut matched_any = false;
23815    let mut counted_disks: HashSet<String> = HashSet::new();
23816
23817    for disk in disks.iter() {
23818        if disk.pool_index < 0 {
23819            continue;
23820        }
23821        let pool_idx = disk.pool_index as usize;
23822        if info.backend.standard_sc_data.len() <= pool_idx {
23823            continue;
23824        }
23825
23826        let usable_disks_per_set = info.backend.standard_sc_data[pool_idx];
23827        if usable_disks_per_set == 0 {
23828            continue;
23829        }
23830
23831        if (disk.disk_index as usize) < usable_disks_per_set {
23832            // 🔧 Generate a unique identity using a combination of fields
23833            let disk_key = format!(
23834                "{}|{}|p{}s{}d{}",
23835                disk.endpoint,   // Node address
23836                disk.drive_path, // mount path
23837                disk.pool_index, // Pool index
23838                disk.set_index,  // Collection index
23839                disk.disk_index  // Disk index
23840            );
23841            debug!("get_total_usable_capacity disk_key: {}", disk_key);
23842            // 🔧 Only disks that have not been counted are counted towards capacity
23843            if counted_disks.insert(disk_key) {
23844                matched_any = true;
23845                capacity += disk.total_space as usize;
23846            } else {
23847                // Log duplicate disks: this likely indicates a configuration issue and should always be visible.
23848                warn!(
23849                    "Duplicate disk detected in capacity calculation: {} at {}",
23850                    disk.endpoint, disk.drive_path
23851                );
23852            }
23853        }
23854    }
23855
23856    if matched_any {
23857        capacity
23858    } else {
23859        // Even if standard_sc_data exists, it might not match disk indexes due to upstream bugs.
23860        // Fallback to summing all online disks to prevent under-reporting.
23861        fallback_total_capacity_dedup(disks)
23862    }
23863}
23864
23865pub fn get_total_usable_capacity_free(disks: &[rustfs_madmin::Disk], info: &rustfs_madmin::StorageInfo) -> usize {
23866    if info.backend.standard_sc_data.is_empty() {
23867        return fallback_free_capacity_dedup(disks);
23868    }
23869
23870    let mut capacity = 0usize;
23871    let mut matched_any = false;
23872    let mut counted_disks: HashSet<String> = HashSet::new();
23873
23874    for disk in disks.iter() {
23875        if disk.pool_index < 0 {
23876            continue;
23877        }
23878        let pool_idx = disk.pool_index as usize;
23879        if info.backend.standard_sc_data.len() <= pool_idx {
23880            continue;
23881        }
23882
23883        let usable_disks_per_set = info.backend.standard_sc_data[pool_idx];
23884        if usable_disks_per_set == 0 {
23885            continue;
23886        }
23887
23888        if (disk.disk_index as usize) < usable_disks_per_set {
23889            let disk_key = format!(
23890                "{}|{}|p{}s{}d{}",
23891                disk.endpoint, disk.drive_path, disk.pool_index, disk.set_index, disk.disk_index
23892            );
23893
23894            if counted_disks.insert(disk_key) {
23895                matched_any = true;
23896                capacity += disk.available_space as usize;
23897            }
23898        }
23899    }
23900
23901    if matched_any {
23902        capacity
23903    } else {
23904        fallback_free_capacity_dedup(disks)
23905    }
23906}
23907
23908/// Total fallback capacity calculation with deweight
23909///
23910/// Replace original function: fallback_total_capacity()
23911pub(crate) fn fallback_total_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usize {
23912    let mut counted_disks: HashSet<String> = HashSet::new();
23913    let mut total = 0usize;
23914
23915    for disk in disks.iter() {
23916        // Only online disks are counted
23917        if !is_disk_online_state(&disk.state) {
23918            continue;
23919        }
23920
23921        // Use endpoint + drive_path as a unique identifier
23922        let disk_key = format!("{}|{}", disk.endpoint, disk.drive_path);
23923
23924        // Capacity is counted only when the disk is encountered for the first time
23925        if counted_disks.insert(disk_key) {
23926            total += disk.total_space as usize;
23927        }
23928    }
23929
23930    total
23931}
23932
23933/// Remove the heavy fallback idle capacity calculation
23934///
23935/// Replace original function: fallback_free_capacity()
23936pub(crate) fn fallback_free_capacity_dedup(disks: &[rustfs_madmin::Disk]) -> usize {
23937    let mut counted_disks: HashSet<String> = HashSet::new();
23938    let mut total = 0usize;
23939
23940    for disk in disks.iter() {
23941        if !is_disk_online_state(&disk.state) {
23942            continue;
23943        }
23944
23945        let disk_key = format!("{}|{}", disk.endpoint, disk.drive_path);
23946
23947        if counted_disks.insert(disk_key) {
23948            total += disk.available_space as usize;
23949        }
23950    }
23951
23952    total
23953}
23954
23955#[cfg(test)]
23956mod pools_tests {
23957    use std::collections::HashMap;
23958
23959    use super::DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
23960    use super::persist_v3_pool_meta_for_test;
23961    use super::record_decommission_entry_error;
23962    use super::resolve_decommission_listing_error;
23963    use super::resolve_decommission_partial_listing_entry;
23964    use super::{
23965        DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, DECOMMISSION_CAPACITY_MODEL_VERSION, DECOMMISSION_CAPACITY_RELEASE_CANCELED,
23966        DECOMMISSION_CAPACITY_RELEASE_COMPLETED, DECOMMISSION_CAPACITY_RELEASE_FAILED, DECOMMISSION_CAPACITY_RESERVATION_TTL,
23967        DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
23968        DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP, DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP, DECOMMISSION_ENTRY_QUEUE_HARD_CAP,
23969        DECOMMISSION_META_PREFIXES, DECOMMISSION_PROGRESS_SAVE_INTERVAL, DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
23970        DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT, DecomBucketInfo, DecommissionCanceler, DecommissionCapacityTarget,
23971        DecommissionDurableIlmReceipt, DecommissionEntryEnqueueResult, DecommissionErasureLayout, DecommissionPoolCapacityInfo,
23972        DecommissionStartPoolState, DecommissionTargetConsumption, DecommissionTerminalState, DecommissionUnresolvedEntry,
23973        ListCallback, POOL_META_GENERATION_VERSION, POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_V1_VERSION,
23974        POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolMetaCasToken, PoolMetaPersistenceFence, PoolSpaceInfo, PoolStatus,
23975        QueuedDecommissionEntry, REBAL_META_NAME, acquire_pool_rebalance_activation_locks, active_decommission_source_indices,
23976        apply_decommission_status_space_info, await_decommission_worker, bind_decommission_cancelers,
23977        bind_missing_decommission_cancelers, build_decommission_capacity_reservation,
23978        build_decommission_capacity_reservation_with_model, cancel_decommission_canceler, clamp_decommission_entry_concurrency,
23979        classify_decommission_terminal_state, count_decommission_item, decommission_cancel_signal_result,
23980        decommission_durable_ilm_receipt_path, decommission_durable_ilm_receipt_run_prefix,
23981        decommission_durable_ilm_receipt_run_token, decommission_entry_queue_capacity, decommission_item_size,
23982        decommission_meta_bucket_options, decommission_physical_pool_capacity, decommission_retry_backoff_ceiling,
23983        decommission_retry_backoff_delay, decommission_start_pool_state, decommission_unresolved_listing_error, dedup_indices,
23984        default_decommission_bucket_concurrency, default_decommission_entry_concurrency, drain_decommission_entry_queue,
23985        enqueue_decommission_entry, ensure_decommission_cancel_allowed, ensure_decommission_capacity_reservations_available,
23986        ensure_decommission_clear_allowed, ensure_decommission_generation, ensure_decommission_listing_disks_available,
23987        ensure_decommission_not_rebalancing, ensure_decommission_start_allowed, ensure_decommission_start_keeps_active_pool,
23988        ensure_decommission_start_local_leader, ensure_decommission_start_pool_states,
23989        ensure_decommission_start_rebalance_meta_allowed, ensure_decommission_start_target_capacity,
23990        ensure_decommission_terminal_operation_supported, ensure_decommission_unresolved_verification_disk_count,
23991        ensure_local_decommission_pool_leaders, ensure_pool_meta_write_fence, ensure_valid_decommission_pool_index, get_by_index,
23992        grow_decommission_target_reservation, guard_decommission_cancelers, has_active_decommission_canceler,
23993        is_decommission_active, is_decommission_cancel_requested, load_decommission_entry_versions,
23994        local_decommission_queue_prefix, mark_decommission_bucket_done, merge_decommission_durable_ilm_receipts,
23995        merge_pool_meta_updates_for_save, merge_pool_status_refresh, missing_decommission_worker_prefix,
23996        next_decommission_capacity_generation, observe_decommission_terminal_reload_result,
23997        parse_decommission_durable_ilm_receipt_path, pool_meta_has_active_decommission, publish_pool_meta_updates,
23998        read_pool_meta_replica, reconcile_decommission_meta_buckets, reconcile_decommission_unresolved_entries_for_completion,
23999        record_decommission_unresolved_entry, recover_decommission_capacity_reservations,
24000        renew_decommission_capacity_reservation, require_decommission_store, reserve_decommission_start_cancelers,
24001        reserve_decommission_start_target_capacity, resolve_decommission_bucket_state,
24002        resolve_decommission_check_after_list_result, resolve_decommission_entry_cleanup_delete_result,
24003        resolve_decommission_entry_exact_versions, resolve_decommission_entry_reload_result,
24004        resolve_decommission_listing_worker_result, resolve_decommission_optional_bucket_config_result,
24005        resolve_decommission_pool_meta_reload_result, resolve_decommission_preflight_heal_result,
24006        resolve_decommission_progress_save_result, resolve_decommission_terminal_mark_after_error_result,
24007        resolve_decommission_terminal_mark_result, resolve_decommission_update_after_result,
24008        resolve_start_decommission_pool_meta_reload_result, resumable_decommission_queue_indices,
24009        rollback_start_decommission_pool_meta, run_decommission_buckets_bounded, run_decommission_listing_with_retry,
24010        run_decommission_listing_with_retry_and_drain, run_decommission_phases, run_decommission_side_effect,
24011        save_pool_meta_object_cas, select_decommission_capacity_model, should_cleanup_decommission_source_entry,
24012        should_continue_decommission_queue, should_count_decommission_version_complete,
24013        should_fail_decommission_pool_after_exhausted_source_changed, should_preserve_decommission_canceled_state,
24014        should_reject_decommission_cancel_as_terminal, should_retry_decommission_cancel_reload,
24015        should_retry_decommission_listing, should_skip_canceled_decommission_routine, spawn_decommission_index_cancelers,
24016        split_decommission_buckets, take_and_cancel_decommission_canceler, take_decommission_canceler,
24017        track_decommission_current_object, track_decommission_current_object_stage, update_decommission_for_operation,
24018        validate_start_decommission_request, wait_decommission_retry_backoff, wait_decommission_worker_drain,
24019        with_decommission_entry_context,
24020    };
24021    use super::{
24022        DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS, DECOMMISSION_RETRY_BACKOFF_CAP, DecommissionCapacityAdmission,
24023        DecommissionCapacityOwner, DecommissionCapacityReleaseProof, DecommissionCapacityReservation,
24024        DecommissionCapacityTemporaryMutation, decommission_capacity_mutation_id,
24025        decommission_capacity_target_gate_retry_exhausted, ensure_decommission_target_owner_admission,
24026        ensure_exact_delete_capacity_namespace_fences, ensure_external_decommission_target_admission,
24027        is_decommission_capacity_blocked_error, plan_exact_delete_capacity_reconciliations,
24028        record_decommission_target_consumption, release_decommission_target_inflight, reserve_decommission_target_pending,
24029        resolve_decommission_target_pending, set_decommission_capacity_info_overrides_for_test,
24030    };
24031    use crate::bucket::lifecycle::{
24032        DurableIlmRecordCheckpoint,
24033        bucket_lifecycle_ops::{ManualTransitionQueueSnapshot, ManualTransitionRunOptions},
24034        manual_transition_job::{ManualTransitionJobRecord, manual_transition_job_record_object_name},
24035        validate_durable_ilm_record,
24036    };
24037    use crate::bucket::metadata_sys;
24038    use crate::data_movement;
24039    use crate::disk::{STORAGE_FORMAT_FILE, endpoint::Endpoint};
24040    use crate::error::{Error, StorageError};
24041    use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
24042    use crate::object_api::{ObjectInfo, ObjectOptions};
24043    use crate::runtime::instance::InstanceContext;
24044    use crate::services::rebalance::{RebalStatus, RebalanceInfo, RebalanceMeta, RebalanceStats};
24045    use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
24046    use crate::storage_api_contracts::object::HTTPPreconditions;
24047    use crate::storage_api_contracts::{object::ObjectIO, range::HTTPRangeSpec};
24048    use crate::store::ECStore;
24049    use byteorder::{ByteOrder, LittleEndian};
24050    use rmp_serde::Serializer;
24051    use rustfs_filemeta::{FileInfo, FileInfoVersions, MetaCacheEntry, ObjectPartInfo};
24052    use rustfs_filemeta::{MetaCacheEntries, MetadataResolutionParams};
24053    use rustfs_lock::{GlobalLockManager, LocalClient, LockRequest, LockType, NamespaceLock, ObjectKey};
24054    use rustfs_rio::Index;
24055    use serde::Serialize;
24056    use std::future::Future;
24057    use std::io::Cursor;
24058    use std::sync::{
24059        Arc, Mutex as StdMutex,
24060        atomic::{AtomicBool, AtomicUsize, Ordering},
24061    };
24062    use std::task::{Context, Poll};
24063    use std::time::Duration as StdDuration;
24064    use time::{Duration, OffsetDateTime};
24065    use tokio::io::AsyncReadExt;
24066    use tokio::sync::Semaphore;
24067    use tokio_util::sync::CancellationToken;
24068
24069    #[derive(Debug)]
24070    struct ActivationLockRecorder {
24071        lock_manager: Arc<rustfs_lock::GlobalLockManager>,
24072        owner: &'static str,
24073        resources: StdMutex<Vec<String>>,
24074    }
24075
24076    #[async_trait::async_trait]
24077    impl crate::storage_api_contracts::namespace::NamespaceLocking for ActivationLockRecorder {
24078        type Error = Error;
24079        type NamespaceLock = rustfs_lock::NamespaceLockWrapper;
24080
24081        async fn new_ns_lock(&self, bucket: &str, object: &str) -> crate::error::Result<Self::NamespaceLock> {
24082            self.resources
24083                .lock()
24084                .expect("activation lock recorder should not be poisoned")
24085                .push(object.to_string());
24086            Ok(rustfs_lock::NamespaceLockWrapper::new(
24087                rustfs_lock::NamespaceLock::with_local_manager(
24088                    "activation-lock-test".to_string(),
24089                    Arc::clone(&self.lock_manager),
24090                ),
24091                rustfs_lock::ObjectKey::new(bucket, object),
24092                self.owner.to_string(),
24093            ))
24094        }
24095    }
24096
24097    fn noop_decommission_list_callback() -> ListCallback {
24098        Arc::new(|_| Box::pin(async {}))
24099    }
24100
24101    fn decommission_worker_test_store(pool_meta: PoolMeta, cancelers: Vec<Option<DecommissionCanceler>>) -> Arc<ECStore> {
24102        let ctx = Arc::new(InstanceContext::new());
24103        let endpoint_pools = EndpointServerPools::default();
24104        Arc::new(ECStore {
24105            id: uuid::Uuid::new_v4(),
24106            disk_map: std::collections::HashMap::new(),
24107            pools: Vec::new(),
24108            peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, ctx.clone()),
24109            pool_meta: tokio::sync::RwLock::new(pool_meta),
24110            rebalance_meta: tokio::sync::RwLock::new(None),
24111            decommission_cancelers: tokio::sync::RwLock::new(cancelers),
24112            start_gate: tokio::sync::Mutex::new(()),
24113            pool_meta_save_gate: tokio::sync::Mutex::new(super::PoolMetaWriteState::for_test_bootstrap()),
24114            ctx,
24115            bucket_fence_registry: Arc::default(),
24116        })
24117    }
24118
24119    #[derive(Debug, Default)]
24120    struct PartialPoolMetaWriteStorage {
24121        fail_write: bool,
24122        fail_after_first_write: bool,
24123        pending_write: bool,
24124        write_started: tokio::sync::Notify,
24125        wrote: AtomicBool,
24126        revision: AtomicUsize,
24127        stored: StdMutex<Option<(Vec<u8>, String)>>,
24128        identity: StdMutex<Option<(Vec<u8>, String)>>,
24129    }
24130
24131    #[test]
24132    fn decommission_target_gate_retry_is_bounded() {
24133        assert!(!decommission_capacity_target_gate_retry_exhausted(0));
24134        assert!(!decommission_capacity_target_gate_retry_exhausted(
24135            DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS - 2
24136        ));
24137        assert!(decommission_capacity_target_gate_retry_exhausted(
24138            DECOMMISSION_CAPACITY_TARGET_GATE_MAX_ATTEMPTS - 1
24139        ));
24140    }
24141
24142    #[tokio::test]
24143    async fn pool_meta_recovery_reconciles_prepare_and_partial_commit_without_format_downgrade() {
24144        for previous_version in [POOL_META_V1_VERSION, POOL_META_VERSION, POOL_META_GENERATION_VERSION] {
24145            for commit_first in [false, true] {
24146                let cluster_id = uuid::Uuid::new_v4();
24147                let previous_meta = PoolMeta {
24148                    version: previous_version,
24149                    pools: vec![decommission_test_pool_status(0, None)],
24150                    ..Default::default()
24151                };
24152                let previous_revision = if previous_version == POOL_META_GENERATION_VERSION {
24153                    super::PoolMetaRevision {
24154                        version: previous_version,
24155                        cluster_id: Some(cluster_id),
24156                        epoch: 1,
24157                        generation: 9,
24158                        transaction_id: Some(uuid::Uuid::new_v4()),
24159                    }
24160                } else {
24161                    super::PoolMetaRevision::legacy(previous_version)
24162                };
24163                let previous_data = if previous_revision.is_generation_protocol() {
24164                    super::encode_pool_meta_v3_envelope(&previous_meta, previous_revision, true, None).unwrap()
24165                } else {
24166                    previous_meta.encode_config_data_for_v2_gate(true).unwrap()
24167                };
24168                let previous = super::PoolMetaCommittedCandidate {
24169                    canonical: previous_data,
24170                    meta: previous_meta.clone(),
24171                    revision: previous_revision,
24172                };
24173                let mut next = previous_meta.clone();
24174                next.version = POOL_META_GENERATION_VERSION;
24175                next.pools[0].last_update += Duration::seconds(1);
24176                let revision = super::PoolMetaRevision {
24177                    version: POOL_META_GENERATION_VERSION,
24178                    cluster_id: Some(cluster_id),
24179                    epoch: 1,
24180                    generation: previous_revision.generation + 1,
24181                    transaction_id: Some(uuid::Uuid::new_v4()),
24182                };
24183                let pending = super::encode_pool_meta_v3_envelope(&next, revision, false, Some(&previous)).unwrap();
24184                let durable = super::encode_pool_meta_v3_envelope(&next, revision, true, None).unwrap();
24185                let identity = super::initialized_pool_meta_identity_for_test(cluster_id, 1).unwrap();
24186                let pools = [if commit_first { durable } else { pending.clone() }, pending]
24187                    .into_iter()
24188                    .map(|data| {
24189                        Arc::new(PartialPoolMetaWriteStorage {
24190                            stored: StdMutex::new(Some((data, "initial".to_owned()))),
24191                            identity: StdMutex::new(Some((identity.clone(), "identity".to_owned()))),
24192                            ..Default::default()
24193                        })
24194                    })
24195                    .collect::<Vec<_>>();
24196                let mut state = super::PoolMetaWriteState::for_startup(cluster_id, false);
24197                let selection = super::load_pool_meta_for_transaction_recovery(pools.clone(), &mut state)
24198                    .await
24199                    .unwrap();
24200                assert_eq!(
24201                    selection.meta.pools[0].last_update,
24202                    if commit_first {
24203                        next.pools[0].last_update
24204                    } else {
24205                        previous_meta.pools[0].last_update
24206                    }
24207                );
24208                let outcome = super::repair_pool_meta_transaction(
24209                    pools.clone(),
24210                    &mut state,
24211                    selection,
24212                    &PoolMetaPersistenceFence::Distributed(None),
24213                )
24214                .await
24215                .unwrap();
24216                assert_eq!(outcome.committed.version, POOL_META_GENERATION_VERSION);
24217                assert_eq!(
24218                    outcome.committed.pools[0].last_update,
24219                    if commit_first {
24220                        next.pools[0].last_update
24221                    } else {
24222                        previous_meta.pools[0].last_update
24223                    }
24224                );
24225                outcome.disarm();
24226                let confirmed = super::load_pool_meta_for_transaction_recovery(pools, &mut state)
24227                    .await
24228                    .unwrap();
24229                assert!(!confirmed.replica_state.needs_repair);
24230                state.ensure_write_safe("repaired").unwrap();
24231            }
24232        }
24233    }
24234
24235    #[tokio::test]
24236    async fn pool_meta_recovery_rejects_missing_corrupt_and_incompatible_replicas_without_writes() {
24237        let cluster_id = uuid::Uuid::new_v4();
24238        let identity = super::initialized_pool_meta_identity_for_test(cluster_id, 1).unwrap();
24239        for payload in [None, Some(Vec::new()), Some(vec![1, 0, 255, 255, 0])] {
24240            let pool = Arc::new(PartialPoolMetaWriteStorage {
24241                stored: StdMutex::new(payload.map(|data| (data, "initial".to_owned()))),
24242                identity: StdMutex::new(Some((identity.clone(), "identity".to_owned()))),
24243                ..Default::default()
24244            });
24245            let mut state = super::PoolMetaWriteState::for_startup(cluster_id, false);
24246            super::load_pool_meta_for_transaction_recovery(vec![pool.clone()], &mut state)
24247                .await
24248                .unwrap_err();
24249            assert!(!pool.wrote.load(Ordering::SeqCst));
24250        }
24251    }
24252
24253    #[tokio::test]
24254    async fn pool_meta_first_cas_rejection_is_retryable_but_later_rejection_keeps_prior_write_armed() {
24255        let pool = Arc::new(PartialPoolMetaWriteStorage::default());
24256        let state = super::PoolMetaWriteState::default();
24257        let fence = PoolMetaPersistenceFence::Distributed(None);
24258        let mut arm = state.arm_transaction();
24259        save_pool_meta_object_cas(
24260            pool.clone(),
24261            POOL_META_NAME,
24262            vec![1],
24263            &PoolMetaCasToken::Existing("missing".to_owned()),
24264            &fence,
24265            "prepare_cas",
24266            &mut arm,
24267        )
24268        .await
24269        .unwrap_err();
24270        drop(arm);
24271        state.ensure_write_safe("first rejection").unwrap();
24272        let mut arm = state.arm_transaction();
24273        save_pool_meta_object_cas(
24274            pool.clone(),
24275            POOL_META_NAME,
24276            vec![1],
24277            &PoolMetaCasToken::Missing,
24278            &fence,
24279            "prepare_cas",
24280            &mut arm,
24281        )
24282        .await
24283        .unwrap();
24284        save_pool_meta_object_cas(pool, POOL_META_NAME, vec![2], &PoolMetaCasToken::Missing, &fence, "commit_cas", &mut arm)
24285            .await
24286            .unwrap_err();
24287        drop(arm);
24288        let err = state.ensure_write_safe("later rejection").unwrap_err();
24289        assert_eq!(err.pool_metadata_failure().unwrap().phase, "prepare_cas");
24290        assert!(matches!(
24291            err.pool_metadata_failure().unwrap().source.as_deref(),
24292            Some(Error::PreconditionFailed)
24293        ));
24294    }
24295
24296    #[tokio::test]
24297    async fn pool_meta_recovery_cas_conflict_after_repair_write_keeps_transaction_blocked() {
24298        let base = PoolMeta {
24299            version: POOL_META_VERSION,
24300            pools: vec![decommission_test_pool_status(0, None)],
24301            ..Default::default()
24302        };
24303        let mut current = base.clone();
24304        current.pools[0].last_update += Duration::seconds(1);
24305        let first = Arc::new(PartialPoolMetaWriteStorage {
24306            stored: StdMutex::new(Some((current.encode_config_data_for_v2_gate(true).unwrap(), "first".to_owned()))),
24307            ..Default::default()
24308        });
24309        let second = Arc::new(PartialPoolMetaWriteStorage {
24310            stored: StdMutex::new(Some((base.encode_config_data_for_v2_gate(true).unwrap(), "second".to_owned()))),
24311            ..Default::default()
24312        });
24313        let mut state = super::PoolMetaWriteState::default();
24314        let selection = super::load_pool_meta_for_transaction_recovery(vec![first.clone(), second.clone()], &mut state)
24315            .await
24316            .unwrap();
24317        let raced = base.encode_config_data_for_v2_gate(true).unwrap();
24318        *second.stored.lock().unwrap() = Some((raced.clone(), "raced".to_owned()));
24319        let result = super::repair_pool_meta_transaction(
24320            vec![first.clone(), second.clone()],
24321            &mut state,
24322            selection,
24323            &PoolMetaPersistenceFence::Distributed(None),
24324        )
24325        .await;
24326        assert!(matches!(result, Err(Error::PreconditionFailed)));
24327        assert!(first.wrote.load(Ordering::SeqCst));
24328        assert!(!second.wrote.load(Ordering::SeqCst));
24329        assert_eq!(second.stored.lock().unwrap().as_ref().unwrap().0, raced);
24330        let err = state.ensure_write_safe("recovery retry").unwrap_err();
24331        assert_eq!(err.pool_metadata_failure().unwrap().phase, "recovery_cas");
24332    }
24333
24334    #[tokio::test]
24335    async fn pool_meta_cas_failure_retains_the_original_source_for_caller_and_latch() {
24336        let pool = Arc::new(PartialPoolMetaWriteStorage {
24337            fail_write: true,
24338            ..Default::default()
24339        });
24340        let state = super::PoolMetaWriteState::default();
24341        let mut arm = state.arm_transaction();
24342        let err = save_pool_meta_object_cas(
24343            pool,
24344            POOL_META_NAME,
24345            Vec::new(),
24346            &PoolMetaCasToken::Missing,
24347            &PoolMetaPersistenceFence::Distributed(None),
24348            "commit_cas",
24349            &mut arm,
24350        )
24351        .await
24352        .unwrap_err();
24353        let source = err.pool_metadata_failure().unwrap().source.as_ref().unwrap();
24354        assert!(matches!(source.as_ref(), Error::Timeout));
24355        assert!(Arc::ptr_eq(source, arm.source.as_ref().unwrap()));
24356        drop(arm);
24357        let blocked = state.ensure_write_safe("after CAS failure").unwrap_err();
24358        assert!(Arc::ptr_eq(source, blocked.pool_metadata_failure().unwrap().source.as_ref().unwrap()));
24359    }
24360
24361    #[tokio::test]
24362    async fn pool_meta_cancelled_identity_dispatch_is_armed_before_pool_bin_writes() {
24363        let pool = Arc::new(PartialPoolMetaWriteStorage {
24364            pending_write: true,
24365            ..Default::default()
24366        });
24367        let state = Arc::new(tokio::sync::Mutex::new(super::PoolMetaWriteState::for_startup(
24368            uuid::Uuid::new_v4(),
24369            true,
24370        )));
24371        let task = tokio::spawn({
24372            let pool = pool.clone();
24373            let state = state.clone();
24374            async move {
24375                let mut state = state.lock().await;
24376                super::persist_pool_meta_identity_for_startup(vec![pool], &mut state, false).await
24377            }
24378        });
24379        pool.write_started.notified().await;
24380        task.abort();
24381        assert!(task.await.unwrap_err().is_cancelled());
24382        let state = state.lock().await;
24383        let err = state.ensure_write_safe("identity retry").unwrap_err();
24384        assert_eq!(err.pool_metadata_failure().unwrap().phase, "identity_cas");
24385        assert_eq!(
24386            err.pool_metadata_failure().unwrap().kind,
24387            crate::error::PoolMetadataFailure::TransactionUnknown
24388        );
24389        assert!(pool.stored.lock().unwrap().is_none());
24390    }
24391
24392    #[async_trait::async_trait]
24393    impl ObjectIO for PartialPoolMetaWriteStorage {
24394        type Error = Error;
24395        type RangeSpec = HTTPRangeSpec;
24396        type HeaderMap = http::HeaderMap;
24397        type ObjectOptions = crate::object_api::ObjectOptions;
24398        type ObjectInfo = crate::object_api::ObjectInfo;
24399        type GetObjectReader = crate::object_api::GetObjectReader;
24400        type PutObjectReader = crate::object_api::PutObjReader;
24401
24402        async fn get_object_reader(
24403            &self,
24404            bucket: &str,
24405            object: &str,
24406            _range: Option<Self::RangeSpec>,
24407            _h: Self::HeaderMap,
24408            _opts: &Self::ObjectOptions,
24409        ) -> std::result::Result<Self::GetObjectReader, Error> {
24410            let stored = if object == POOL_META_IDENTITY_NAME {
24411                &self.identity
24412            } else {
24413                &self.stored
24414            };
24415            let Some((data, etag)) = stored
24416                .lock()
24417                .expect("pool metadata test storage should not be poisoned")
24418                .clone()
24419            else {
24420                return Err(Error::FileNotFound);
24421            };
24422            Ok(crate::object_api::GetObjectReader {
24423                stream: Box::new(Cursor::new(data.clone())),
24424                object_info: crate::object_api::ObjectInfo {
24425                    bucket: bucket.to_string(),
24426                    name: object.to_string(),
24427                    size: data.len() as i64,
24428                    etag: Some(etag),
24429                    ..Default::default()
24430                },
24431                buffered_body: None,
24432                body_source: Default::default(),
24433            })
24434        }
24435
24436        async fn put_object(
24437            &self,
24438            _bucket: &str,
24439            object: &str,
24440            data: &mut Self::PutObjectReader,
24441            opts: &Self::ObjectOptions,
24442        ) -> std::result::Result<Self::ObjectInfo, Error> {
24443            self.write_started.notify_one();
24444            if self.pending_write {
24445                std::future::pending().await
24446            }
24447            if object == POOL_META_NAME
24448                && (self.fail_write || (self.fail_after_first_write && self.revision.load(Ordering::SeqCst) > 0))
24449            {
24450                return Err(Error::Timeout);
24451            }
24452            let stored = if object == POOL_META_IDENTITY_NAME {
24453                &self.identity
24454            } else {
24455                &self.stored
24456            };
24457            let current_etag = stored
24458                .lock()
24459                .expect("pool metadata test storage should not be poisoned")
24460                .as_ref()
24461                .map(|(_, etag)| etag.clone());
24462            if opts
24463                .http_preconditions
24464                .as_ref()
24465                .and_then(HTTPPreconditions::if_none_match_value)
24466                == Some("*")
24467                && current_etag.is_some()
24468            {
24469                return Err(Error::PreconditionFailed);
24470            }
24471            if let Some(expected) = opts.http_preconditions.as_ref().and_then(HTTPPreconditions::if_match_value)
24472                && current_etag.as_deref() != Some(expected)
24473            {
24474                return Err(Error::PreconditionFailed);
24475            }
24476            let mut payload = Vec::new();
24477            data.stream.read_to_end(&mut payload).await?;
24478            let etag = format!("pool-meta-test-{}", self.revision.fetch_add(1, Ordering::SeqCst) + 1);
24479            *stored.lock().expect("pool metadata test storage should not be poisoned") = Some((payload, etag.clone()));
24480            self.wrote.store(true, Ordering::SeqCst);
24481            Ok(crate::object_api::ObjectInfo {
24482                etag: Some(etag),
24483                ..Default::default()
24484            })
24485        }
24486    }
24487
24488    mod phase_tests {
24489        use super::super as implementation;
24490        use super::*;
24491
24492        #[derive(Clone, Copy, Debug, Default)]
24493        enum WriteFault {
24494            #[default]
24495            None,
24496            Reject,
24497            Timeout,
24498            TimeoutAfterWrite,
24499            EmptyEtag,
24500            OmitWrite,
24501        }
24502
24503        #[derive(Clone, Debug, Default)]
24504        struct WriteStep {
24505            gate: Option<Arc<tokio::sync::Semaphore>>,
24506            fault: WriteFault,
24507        }
24508
24509        #[derive(Clone, Debug)]
24510        struct WriteCall {
24511            pool: usize,
24512            phase: usize,
24513            if_match: Option<String>,
24514        }
24515
24516        #[derive(Debug, Default)]
24517        struct WriteTrace {
24518            calls: StdMutex<Vec<WriteCall>>,
24519            active: AtomicUsize,
24520            maximum: AtomicUsize,
24521            finished: AtomicUsize,
24522            changed: tokio::sync::Notify,
24523        }
24524
24525        impl WriteTrace {
24526            async fn wait_for(&self, calls: usize, finished: usize) {
24527                tokio::time::timeout(StdDuration::from_secs(3), async {
24528                    loop {
24529                        let changed = self.changed.notified();
24530                        if self.calls.lock().expect("trace lock").len() >= calls
24531                            && self.finished.load(Ordering::SeqCst) >= finished
24532                        {
24533                            return;
24534                        }
24535                        changed.await;
24536                    }
24537                })
24538                .await
24539                .expect("phase must reach the requested barrier");
24540            }
24541
24542            fn commit_count(&self) -> usize {
24543                self.calls
24544                    .lock()
24545                    .expect("trace lock")
24546                    .iter()
24547                    .filter(|call| call.phase == 1)
24548                    .count()
24549            }
24550        }
24551
24552        struct ActiveWrite<'a>(&'a WriteTrace);
24553        impl Drop for ActiveWrite<'_> {
24554            fn drop(&mut self) {
24555                self.0.active.fetch_sub(1, Ordering::SeqCst);
24556                self.0.finished.fetch_add(1, Ordering::SeqCst);
24557                self.0.changed.notify_one();
24558            }
24559        }
24560
24561        #[derive(Debug)]
24562        struct PhaseStorage {
24563            inner: PartialPoolMetaWriteStorage,
24564            pool: usize,
24565            steps: [WriteStep; 2],
24566            trace: Arc<WriteTrace>,
24567        }
24568
24569        #[async_trait::async_trait]
24570        impl ObjectIO for PhaseStorage {
24571            type Error = Error;
24572            type RangeSpec = HTTPRangeSpec;
24573            type HeaderMap = http::HeaderMap;
24574            type ObjectOptions = crate::object_api::ObjectOptions;
24575            type ObjectInfo = crate::object_api::ObjectInfo;
24576            type GetObjectReader = crate::object_api::GetObjectReader;
24577            type PutObjectReader = crate::object_api::PutObjReader;
24578
24579            async fn get_object_reader(
24580                &self,
24581                bucket: &str,
24582                object: &str,
24583                range: Option<Self::RangeSpec>,
24584                headers: Self::HeaderMap,
24585                opts: &Self::ObjectOptions,
24586            ) -> Result<Self::GetObjectReader, Error> {
24587                self.inner.get_object_reader(bucket, object, range, headers, opts).await
24588            }
24589
24590            async fn put_object(
24591                &self,
24592                bucket: &str,
24593                object: &str,
24594                data: &mut Self::PutObjectReader,
24595                opts: &Self::ObjectOptions,
24596            ) -> Result<Self::ObjectInfo, Error> {
24597                if object != POOL_META_NAME {
24598                    return self.inner.put_object(bucket, object, data, opts).await;
24599                }
24600                let mut payload = Vec::new();
24601                data.stream.read_to_end(&mut payload).await?;
24602                let phase = match implementation::decode_pool_meta_replica(payload.clone()) {
24603                    implementation::PoolMetaReplica::Valid { committed, .. } => usize::from(committed),
24604                    _ => panic!("phase fixture must receive valid pool metadata"),
24605                };
24606                assert!(opts.max_parity && opts.no_lock);
24607                assert_eq!(opts.write_completion, crate::object_api::WriteCompletion::TailDrained);
24608                self.trace.calls.lock().expect("trace lock").push(WriteCall {
24609                    pool: self.pool,
24610                    phase,
24611                    if_match: opts
24612                        .http_preconditions
24613                        .as_ref()
24614                        .and_then(HTTPPreconditions::if_match_value)
24615                        .map(str::to_owned),
24616                });
24617                let active = self.trace.active.fetch_add(1, Ordering::SeqCst) + 1;
24618                self.trace.maximum.fetch_max(active, Ordering::SeqCst);
24619                let _active = ActiveWrite(&self.trace);
24620                self.trace.changed.notify_one();
24621                let step = &self.steps[phase];
24622                if let Some(gate) = &step.gate {
24623                    gate.acquire().await.expect("phase gate stays open").forget();
24624                }
24625                match step.fault {
24626                    WriteFault::Reject => return Err(Error::PreconditionFailed),
24627                    WriteFault::Timeout => return Err(Error::Timeout),
24628                    WriteFault::OmitWrite => {
24629                        return Ok(crate::object_api::ObjectInfo {
24630                            etag: Some("uncommitted".to_owned()),
24631                            ..Default::default()
24632                        });
24633                    }
24634                    _ => {}
24635                }
24636                let mut data = crate::object_api::PutObjReader::from_vec(payload);
24637                let mut result = self.inner.put_object(bucket, object, &mut data, opts).await?;
24638                match step.fault {
24639                    WriteFault::TimeoutAfterWrite => return Err(Error::Timeout),
24640                    WriteFault::EmptyEtag => result.etag = Some("  ".to_owned()),
24641                    _ => {}
24642                }
24643                Ok(result)
24644            }
24645        }
24646
24647        struct Fixture {
24648            pools: Vec<Arc<PhaseStorage>>,
24649            trace: Arc<WriteTrace>,
24650            previous: implementation::PoolMetaCommittedCandidate,
24651            desired: PoolMeta,
24652            pending: Vec<u8>,
24653            durable: Vec<u8>,
24654            revision: implementation::PoolMetaRevision,
24655            cluster_id: uuid::Uuid,
24656        }
24657
24658        impl Fixture {
24659            fn new(steps: Vec<[WriteStep; 2]>) -> Self {
24660                let cluster_id = uuid::Uuid::new_v4();
24661                let meta = PoolMeta {
24662                    version: POOL_META_GENERATION_VERSION,
24663                    pools: (0..steps.len())
24664                        .map(|index| {
24665                            decommission_test_pool_status(
24666                                index,
24667                                (index == 0).then(|| PoolDecommissionInfo {
24668                                    start_time: Some(OffsetDateTime::UNIX_EPOCH),
24669                                    start_size: 4096,
24670                                    total_size: 16384,
24671                                    current_size: 8192,
24672                                    items_decommissioned: 7,
24673                                    items_decommission_failed: 1,
24674                                    bytes_done: 4096,
24675                                    bytes_failed: 512,
24676                                    queued_buckets: vec!["remaining-a".to_owned(), "remaining-b".to_owned()],
24677                                    decommissioned_buckets: vec!["completed".to_owned()],
24678                                    bucket: "remaining-a".to_owned(),
24679                                    prefix: "objects/".to_owned(),
24680                                    object: "objects/007".to_owned(),
24681                                    ..Default::default()
24682                                }),
24683                            )
24684                        })
24685                        .collect(),
24686                    ..Default::default()
24687                };
24688                let previous_revision = implementation::PoolMetaRevision {
24689                    version: POOL_META_GENERATION_VERSION,
24690                    cluster_id: Some(cluster_id),
24691                    epoch: 1,
24692                    generation: 7,
24693                    transaction_id: Some(uuid::Uuid::new_v4()),
24694                };
24695                let previous = implementation::PoolMetaCommittedCandidate {
24696                    canonical: implementation::encode_pool_meta_v3_envelope(&meta, previous_revision, true, None)
24697                        .expect("committed predecessor"),
24698                    meta: meta.clone(),
24699                    revision: previous_revision,
24700                };
24701                let mut desired = meta;
24702                desired.pools[0].last_update += Duration::seconds(1);
24703                let progress = desired.pools[0].decommission.as_mut().expect("active decommission");
24704                progress.items_decommissioned += 1;
24705                progress.bytes_done += 512;
24706                progress.current_size += 512;
24707                progress.object = "objects/008".to_owned();
24708                let revision = implementation::PoolMetaRevision {
24709                    generation: 8,
24710                    transaction_id: Some(uuid::Uuid::new_v4()),
24711                    ..previous_revision
24712                };
24713                let pending = implementation::encode_pool_meta_v3_envelope(&desired, revision, false, Some(&previous))
24714                    .expect("pending candidate");
24715                let durable =
24716                    implementation::encode_pool_meta_v3_envelope(&desired, revision, true, None).expect("committed candidate");
24717                let identity = implementation::initialized_pool_meta_identity_for_test(cluster_id, 1).expect("identity");
24718                let trace = Arc::new(WriteTrace::default());
24719                let pools = steps
24720                    .into_iter()
24721                    .enumerate()
24722                    .map(|(index, steps)| {
24723                        Arc::new(PhaseStorage {
24724                            inner: PartialPoolMetaWriteStorage {
24725                                stored: StdMutex::new(Some((previous.canonical.clone(), format!("initial-{index}")))),
24726                                identity: StdMutex::new(Some((identity.clone(), "identity".to_owned()))),
24727                                revision: AtomicUsize::new(index * 10),
24728                                ..Default::default()
24729                            },
24730                            pool: index,
24731                            steps,
24732                            trace: trace.clone(),
24733                        })
24734                    })
24735                    .collect();
24736                Self {
24737                    pools,
24738                    trace,
24739                    previous,
24740                    desired,
24741                    pending,
24742                    durable,
24743                    revision,
24744                    cluster_id,
24745                }
24746            }
24747
24748            fn state(&self) -> implementation::PoolMetaWriteState {
24749                implementation::PoolMetaWriteState::for_startup(self.cluster_id, false)
24750            }
24751
24752            fn tokens(&self) -> Vec<PoolMetaCasToken> {
24753                self.pools
24754                    .iter()
24755                    .map(|pool| {
24756                        PoolMetaCasToken::Existing(
24757                            pool.inner
24758                                .stored
24759                                .lock()
24760                                .expect("fixture storage")
24761                                .as_ref()
24762                                .expect("existing replica")
24763                                .1
24764                                .clone(),
24765                        )
24766                    })
24767                    .collect()
24768            }
24769        }
24770
24771        fn steps(count: usize) -> Vec<[WriteStep; 2]> {
24772            (0..count).map(|_| [WriteStep::default(), WriteStep::default()]).collect()
24773        }
24774
24775        fn signal() -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
24776            Some(Arc::default())
24777        }
24778
24779        async fn prepare(
24780            fixture: &Fixture,
24781            fence: &PoolMetaPersistenceFence<'_>,
24782            arm: &mut implementation::PoolMetaTransactionArm,
24783        ) -> Vec<Result<crate::object_api::ObjectInfo, Error>> {
24784            implementation::save_pool_meta_phase(
24785                &fixture.pools,
24786                &fixture.pending,
24787                &fixture.tokens(),
24788                fence,
24789                "prepare_cas",
24790                arm,
24791                #[cfg(feature = "e2e-test-hooks")]
24792                None,
24793            )
24794            .await
24795        }
24796
24797        #[tokio::test]
24798        async fn pool_meta_v3_phase_concurrency_is_bounded() {
24799            let gate = Arc::new(tokio::sync::Semaphore::new(0));
24800            let mut config = steps(6);
24801            for step in &mut config {
24802                step[0].gate = Some(gate.clone());
24803            }
24804            let fixture = Fixture::new(config);
24805            let state = fixture.state();
24806            let mut arm = state.arm_transaction();
24807            let fence = PoolMetaPersistenceFence::Distributed(signal());
24808            let mut pending = Box::pin(prepare(&fixture, &fence, &mut arm));
24809            tokio::select! { _ = fixture.trace.wait_for(4, 0) => {}, _ = &mut pending => panic!("blocked writes finished") }
24810            assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 4);
24811            gate.add_permits(1);
24812            tokio::select! { _ = fixture.trace.wait_for(5, 1) => {}, _ = &mut pending => panic!("unreleased writes finished") }
24813            assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 5);
24814            gate.add_permits(5);
24815            assert!(pending.await.into_iter().all(|result| result.is_ok()));
24816            assert_eq!(fixture.trace.maximum.load(Ordering::SeqCst), 4);
24817            assert_eq!(fixture.trace.active.load(Ordering::SeqCst), 0);
24818            arm.disarm();
24819        }
24820
24821        #[tokio::test]
24822        async fn pool_meta_v3_commit_waits_for_all_prepare_etags() {
24823            let gate = Arc::new(tokio::sync::Semaphore::new(0));
24824            let mut config = steps(4);
24825            config[3][0].gate = Some(gate.clone());
24826            let fixture = Fixture::new(config);
24827            let mut state = fixture.state();
24828            let mut save = Box::pin(
24829                fixture
24830                    .desired
24831                    .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]),
24832            );
24833            tokio::select! { _ = fixture.trace.wait_for(4, 3) => {}, _ = &mut save => panic!("commit crossed the last prepare") }
24834            assert_eq!(fixture.trace.commit_count(), 0);
24835            gate.add_permits(1);
24836            save.await.expect("all prepares and commits succeed").disarm();
24837            let calls = fixture.trace.calls.lock().expect("trace");
24838            for call in calls.iter().filter(|call| call.phase == 1) {
24839                assert_eq!(call.if_match, Some(format!("pool-meta-test-{}", call.pool * 10 + 1)));
24840            }
24841            assert_eq!(calls.iter().filter(|call| call.phase == 1).count(), 4);
24842        }
24843
24844        #[tokio::test]
24845        async fn pool_meta_v3_empty_prepare_etag_never_starts_commit() {
24846            let mut config = steps(4);
24847            config[3][0].fault = WriteFault::EmptyEtag;
24848            let fixture = Fixture::new(config);
24849            let mut state = fixture.state();
24850            let err = fixture
24851                .desired
24852                .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0])
24853                .await
24854                .expect_err("a missing conditional-write revision must fail");
24855            assert!(err.to_string().contains("without a conditional-write revision"));
24856            assert_eq!(fixture.trace.finished.load(Ordering::SeqCst), 4);
24857            assert_eq!(fixture.trace.commit_count(), 0);
24858            assert!(state.ensure_write_safe("empty etag wrote pending data").is_err());
24859        }
24860
24861        #[tokio::test]
24862        async fn pool_meta_v3_prepare_failure_drains_late_writes() {
24863            let gate = Arc::new(tokio::sync::Semaphore::new(0));
24864            let mut config = steps(4);
24865            config[1][0].fault = WriteFault::Timeout;
24866            config[3][0].gate = Some(gate.clone());
24867            let fixture = Fixture::new(config);
24868            let mut state = fixture.state();
24869            let mut save = Box::pin(
24870                fixture
24871                    .desired
24872                    .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]),
24873            );
24874            tokio::select! { _ = fixture.trace.wait_for(4, 3) => {}, _ = &mut save => panic!("phase returned with a live replica") }
24875            assert_eq!(fixture.trace.commit_count(), 0);
24876            gate.add_permits(1);
24877            assert!(save.await.is_err());
24878            assert_eq!(fixture.trace.finished.load(Ordering::SeqCst), 4);
24879            assert!(fixture.pools[3].inner.wrote.load(Ordering::SeqCst));
24880            assert!(state.ensure_write_safe("ambiguous prepare").is_err());
24881        }
24882
24883        #[tokio::test]
24884        async fn pool_meta_v3_rejections_restore_only_an_unmutated_phase() {
24885            for previous in [None, Some("identity_cas")] {
24886                let mut config = steps(4);
24887                for step in &mut config {
24888                    step[0].fault = WriteFault::Reject;
24889                }
24890                let fixture = Fixture::new(config);
24891                let state = fixture.state();
24892                let mut arm = state.arm_transaction();
24893                arm.phase = previous;
24894                let results = prepare(&fixture, &PoolMetaPersistenceFence::Distributed(signal()), &mut arm).await;
24895                assert!(
24896                    results
24897                        .into_iter()
24898                        .all(|result| matches!(result, Err(Error::PreconditionFailed)))
24899                );
24900                assert_eq!(arm.phase, previous);
24901                drop(arm);
24902                assert_eq!(state.ensure_write_safe("all rejected").is_ok(), previous.is_none());
24903            }
24904            let mut config = steps(4);
24905            for step in &mut config[1..] {
24906                step[0].fault = WriteFault::Reject;
24907            }
24908            let fixture = Fixture::new(config);
24909            let state = fixture.state();
24910            let mut arm = state.arm_transaction();
24911            let results = prepare(&fixture, &PoolMetaPersistenceFence::Distributed(signal()), &mut arm).await;
24912            assert!(results[0].is_ok());
24913            assert_eq!(arm.phase, Some("prepare_cas"));
24914            drop(arm);
24915            assert!(state.ensure_write_safe("one sibling wrote").is_err());
24916        }
24917
24918        #[tokio::test]
24919        async fn pool_meta_v3_parallel_errors_keep_original_source_pointers() {
24920            let mut config = steps(4);
24921            config[0][0].fault = WriteFault::Timeout;
24922            config[1][0].fault = WriteFault::Timeout;
24923            config[3][0].fault = WriteFault::Reject;
24924            let fixture = Fixture::new(config);
24925            let state = fixture.state();
24926            let mut arm = state.arm_transaction();
24927            let results = prepare(&fixture, &PoolMetaPersistenceFence::Distributed(signal()), &mut arm).await;
24928            let err = results[0].as_ref().expect_err("first pool error stays first");
24929            let source = err
24930                .pool_metadata_failure()
24931                .expect("typed error")
24932                .source
24933                .as_ref()
24934                .expect("original source");
24935            assert!(matches!(source.as_ref(), Error::Timeout));
24936            assert!(Arc::ptr_eq(source, arm.source.as_ref().expect("arm source")));
24937            drop(arm);
24938            let blocked = state
24939                .ensure_write_safe("after parallel errors")
24940                .expect_err("recovery required");
24941            assert!(Arc::ptr_eq(
24942                source,
24943                blocked
24944                    .pool_metadata_failure()
24945                    .expect("typed latch")
24946                    .source
24947                    .as_ref()
24948                    .expect("latch source")
24949            ));
24950        }
24951
24952        #[tokio::test]
24953        async fn pool_meta_v3_retry_rejections_keep_the_previous_io_source() {
24954            let mut config = steps(4);
24955            config[0][0].fault = WriteFault::Reject;
24956            config[1][0].fault = WriteFault::Timeout;
24957            let first = Fixture::new(config);
24958            let state = first.state();
24959            let mut arm = state.arm_transaction();
24960            let fence = PoolMetaPersistenceFence::Distributed(signal());
24961            let results = prepare(&first, &fence, &mut arm).await;
24962            assert!(matches!(&results[0], Err(Error::PreconditionFailed)));
24963            let original = arm.source.as_ref().expect("first attempt I/O source").clone();
24964            assert!(matches!(original.as_ref(), Error::Timeout));
24965            let mut config = steps(4);
24966            for step in &mut config {
24967                step[0].fault = WriteFault::Reject;
24968            }
24969            let retry = Fixture::new(config);
24970            assert!(
24971                prepare(&retry, &fence, &mut arm)
24972                    .await
24973                    .into_iter()
24974                    .all(|result| matches!(result, Err(Error::PreconditionFailed)))
24975            );
24976            assert!(Arc::ptr_eq(&original, arm.source.as_ref().expect("source survives phase retry")));
24977            implementation::save_pool_meta_object_cas(
24978                retry.pools[0].clone(),
24979                POOL_META_NAME,
24980                retry.pending.clone(),
24981                &retry.tokens()[0],
24982                &fence,
24983                "prepare_cas",
24984                &mut arm,
24985            )
24986            .await
24987            .expect_err("serial retry rejected");
24988            assert!(Arc::ptr_eq(&original, arm.source.as_ref().expect("source survives serial retry")));
24989            drop(arm);
24990            let blocked = state
24991                .ensure_write_safe("cancelled after retry rejection")
24992                .expect_err("recovery required");
24993            assert!(Arc::ptr_eq(
24994                &original,
24995                blocked
24996                    .pool_metadata_failure()
24997                    .expect("typed latch")
24998                    .source
24999                    .as_ref()
25000                    .expect("source")
25001            ));
25002        }
25003
25004        #[tokio::test]
25005        async fn pool_meta_v3_cancelled_drain_does_not_replace_io_error_with_cas_rejection() {
25006            let gate = Arc::new(tokio::sync::Semaphore::new(0));
25007            let mut config = steps(4);
25008            config[0][0].gate = Some(gate.clone());
25009            config[2][0].gate = Some(gate);
25010            config[1][0].fault = WriteFault::Timeout;
25011            config[3][0].fault = WriteFault::Reject;
25012            let fixture = Fixture::new(config);
25013            let state = fixture.state();
25014            let mut arm = state.arm_transaction();
25015            let fence = PoolMetaPersistenceFence::Distributed(signal());
25016            let mut phase = Box::pin(prepare(&fixture, &fence, &mut arm));
25017            tokio::select! { _ = fixture.trace.wait_for(4, 2) => {}, _ = &mut phase => panic!("blocked phase finished") }
25018            assert!(futures::poll!(phase.as_mut()).is_pending(), "drain remains parked on the two slow pools");
25019            drop(phase);
25020            let source = arm.source.as_ref().expect("observed I/O source").clone();
25021            assert!(matches!(source.as_ref(), Error::Timeout));
25022            drop(arm);
25023            let blocked = state
25024                .ensure_write_safe("cancelled after mixed results")
25025                .expect_err("recovery required");
25026            assert!(Arc::ptr_eq(
25027                &source,
25028                blocked
25029                    .pool_metadata_failure()
25030                    .expect("typed latch")
25031                    .source
25032                    .as_ref()
25033                    .expect("source")
25034            ));
25035            assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
25036        }
25037
25038        #[tokio::test]
25039        async fn pool_meta_v3_cancelled_prepare_commit_and_publication_keep_gate_blocked() {
25040            for phase in [0, 1] {
25041                let gate = Arc::new(tokio::sync::Semaphore::new(0));
25042                let mut config = steps(4);
25043                for step in &mut config {
25044                    step[phase].gate = Some(gate.clone());
25045                }
25046                let fixture = Fixture::new(config);
25047                let mut state = fixture.state();
25048                let mut save = Box::pin(
25049                    fixture
25050                        .desired
25051                        .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0]),
25052                );
25053                tokio::select! { _ = fixture.trace.wait_for(4 * (phase + 1), 4 * phase) => {}, _ = &mut save => panic!("blocked phase finished") }
25054                drop(save);
25055                assert!(state.ensure_write_safe("cancelled phase").is_err());
25056                assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
25057                assert_eq!(fixture.trace.active.load(Ordering::SeqCst), 0);
25058            }
25059            let fixture = Fixture::new(steps(4));
25060            let mut state = fixture.state();
25061            let outcome = fixture
25062                .desired
25063                .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0])
25064                .await
25065                .expect("durable save before publication");
25066            drop(outcome);
25067            assert!(state.ensure_write_safe("publication was cancelled").is_err());
25068            assert_eq!(state.active_transactions.load(Ordering::SeqCst), 0);
25069        }
25070
25071        #[tokio::test]
25072        async fn pool_meta_v3_fence_loss_blocks_queued_dispatch_and_marks_late_writes() {
25073            let lock = NamespaceLock::with_clients_and_quorum(
25074                "parallel-fence".to_owned(),
25075                vec![Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())))],
25076                1,
25077            );
25078            let request = LockRequest::new(
25079                ObjectKey::new(super::super::RUSTFS_META_BUCKET, POOL_META_NAME),
25080                LockType::Exclusive,
25081                "phase-test",
25082            )
25083            .with_acquire_timeout(StdDuration::from_secs(1))
25084            .with_ttl(StdDuration::from_secs(1))
25085            .with_refresh_interval(StdDuration::from_secs(1));
25086            let guard = lock
25087                .acquire_guard(&request)
25088                .await
25089                .expect("lease request")
25090                .expect("lease acquired");
25091            let gate = Arc::new(tokio::sync::Semaphore::new(0));
25092            let mut config = steps(6);
25093            for step in &mut config {
25094                step[0].gate = Some(gate.clone());
25095            }
25096            let fixture = Fixture::new(config);
25097            let state = fixture.state();
25098            let mut arm = state.arm_transaction();
25099            let fence = PoolMetaPersistenceFence::Distributed(guard.lock_lost_signal());
25100            let mut pending = Box::pin(prepare(&fixture, &fence, &mut arm));
25101            tokio::select! { _ = fixture.trace.wait_for(4, 0) => {}, _ = &mut pending => panic!("gate ignored") }
25102            tokio::time::timeout(StdDuration::from_secs(3), guard.lock_lost_notified())
25103                .await
25104                .expect("lease expires");
25105            gate.add_permits(6);
25106            assert!(pending.await.into_iter().all(|result| result.is_err()));
25107            assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 4, "queued pools must not reach storage");
25108            assert!(fixture.pools[..4].iter().all(|pool| pool.inner.wrote.load(Ordering::SeqCst)));
25109            assert_eq!(arm.phase, Some("prepare_cas"));
25110            drop(arm);
25111            assert!(state.ensure_write_safe("writes landed after lease loss").is_err());
25112        }
25113
25114        #[tokio::test]
25115        async fn pool_meta_v3_all_commit_errors_still_require_canonical_reread() {
25116            for fault in [WriteFault::TimeoutAfterWrite, WriteFault::OmitWrite] {
25117                let mut config = steps(4);
25118                for step in &mut config {
25119                    step[1].fault = fault;
25120                }
25121                let fixture = Fixture::new(config);
25122                let mut state = fixture.state();
25123                let result = fixture
25124                    .desired
25125                    .save_no_lock_armed(fixture.pools.clone(), &mut state, signal(), &[0])
25126                    .await;
25127                assert_eq!(fixture.trace.commit_count(), 4);
25128                if matches!(fault, WriteFault::TimeoutAfterWrite) {
25129                    let outcome = result.expect("reread proves commits whose replies timed out");
25130                    assert_eq!(outcome.committed.pools[0].last_update, fixture.desired.pools[0].last_update);
25131                    outcome.disarm();
25132                    state
25133                        .ensure_write_safe("confirmed ambiguous commits")
25134                        .expect("confirmed transaction is safe");
25135                } else {
25136                    assert!(result.is_err(), "successful replies cannot replace canonical proof");
25137                    assert!(state.ensure_write_safe("no committed canonical generation").is_err());
25138                }
25139            }
25140        }
25141
25142        #[tokio::test]
25143        async fn pool_meta_v3_none_and_legacy_paths_remain_serial() {
25144            for (legacy, needs_repair) in [(false, false), (true, false), (false, true)] {
25145                let gate = Arc::new(tokio::sync::Semaphore::new(0));
25146                let mut config = steps(4);
25147                for step in &mut config {
25148                    step[0].gate = Some(gate.clone());
25149                    step[1].gate = Some(gate.clone());
25150                }
25151                let mut fixture = Fixture::new(config);
25152                if legacy {
25153                    let mut meta = fixture.previous.meta.clone();
25154                    meta.version = POOL_META_VERSION;
25155                    let data = meta.encode_config_data_for_v2_gate(true).expect("legacy metadata");
25156                    for pool in &fixture.pools {
25157                        pool.inner.stored.lock().expect("fixture").as_mut().expect("replica").0 = data.clone();
25158                    }
25159                    fixture.desired.version = POOL_META_VERSION;
25160                }
25161                if needs_repair {
25162                    fixture.pools[3]
25163                        .inner
25164                        .stored
25165                        .lock()
25166                        .expect("fixture")
25167                        .as_mut()
25168                        .expect("replica")
25169                        .0 = fixture.pending.clone();
25170                }
25171                let mut state = fixture.state();
25172                let mut save = Box::pin(fixture.desired.save_no_lock_armed(
25173                    fixture.pools.clone(),
25174                    &mut state,
25175                    if legacy || needs_repair { signal() } else { None },
25176                    &[0],
25177                ));
25178                tokio::select! { _ = fixture.trace.wait_for(1, 0) => {}, _ = &mut save => panic!("serial write gate ignored") }
25179                assert!(futures::poll!(save.as_mut()).is_pending());
25180                assert_eq!(fixture.trace.calls.lock().expect("trace").len(), 1);
25181                gate.add_permits(16);
25182                save.await.expect("serial save").disarm();
25183                assert_eq!(fixture.trace.maximum.load(Ordering::SeqCst), 1);
25184            }
25185        }
25186
25187        #[cfg(feature = "e2e-test-hooks")]
25188        #[tokio::test]
25189        async fn pool_meta_phase_barrier_leaves_noop_arm_for_the_next_changed_transaction() {
25190            for case in [
25191                implementation::PoolMetaPhaseBarrierCase::PrepareSubset,
25192                implementation::PoolMetaPhaseBarrierCase::PreparedAll,
25193                implementation::PoolMetaPhaseBarrierCase::CommitOne,
25194                implementation::PoolMetaPhaseBarrierCase::BeforePublish,
25195            ] {
25196                let fixture = Fixture::new(steps(4));
25197                let directory = tempfile::TempDir::new().expect("barrier directory");
25198                tokio::fs::write(
25199                    directory.path().join("arm.json"),
25200                    serde_json::to_vec(&serde_json::json!({"nonce": uuid::Uuid::new_v4(), "case": case})).expect("arm"),
25201                )
25202                .await
25203                .expect("write arm");
25204                let noop = implementation::encode_pool_meta_v3_envelope(&fixture.previous.meta, fixture.revision, true, None)
25205                    .expect("new revision with unchanged pools");
25206                assert_ne!(noop, fixture.previous.canonical);
25207                let barrier = implementation::PoolMetaPhaseBarrier::bind_in_directory(
25208                    directory.path().to_path_buf(),
25209                    &fixture.previous,
25210                    &fixture.previous.meta,
25211                    fixture.revision,
25212                    &noop,
25213                )
25214                .await
25215                .expect("unchanged pools must not consume the arm");
25216                assert!(barrier.is_none());
25217                assert!(!directory.path().join("claimed").exists());
25218                assert!(!directory.path().join("events.jsonl").exists());
25219
25220                let previous = implementation::PoolMetaCommittedCandidate {
25221                    meta: fixture.previous.meta.clone(),
25222                    revision: fixture.revision,
25223                    canonical: noop,
25224                };
25225                let mut candidate = previous.meta.clone();
25226                let progress = candidate.pools[0].decommission.as_mut().expect("progress");
25227                progress.items_decommissioned += 1;
25228                progress.bytes_done += 512;
25229                let revision = implementation::PoolMetaRevision {
25230                    generation: previous.revision.generation + 1,
25231                    transaction_id: Some(uuid::Uuid::new_v4()),
25232                    ..previous.revision
25233                };
25234                let durable = implementation::encode_pool_meta_v3_envelope(&candidate, revision, true, None)
25235                    .expect("changed progress without a timestamp change");
25236                let barrier = implementation::PoolMetaPhaseBarrier::bind_in_directory(
25237                    directory.path().to_path_buf(),
25238                    &previous,
25239                    &candidate,
25240                    revision,
25241                    &durable,
25242                )
25243                .await
25244                .expect("next changed transaction must bind")
25245                .expect("bound barrier");
25246                assert_eq!(barrier.transaction_id, revision.transaction_id.expect("transaction"));
25247                assert!(directory.path().join("claimed").is_file());
25248                let events = tokio::fs::read_to_string(directory.path().join("events.jsonl"))
25249                    .await
25250                    .expect("armed event");
25251                assert_eq!(events.lines().count(), 1);
25252                let event: serde_json::Value = serde_json::from_str(events.trim()).expect("armed event JSON");
25253                assert_eq!(event["kind"], "armed");
25254                assert_eq!(event["previous"]["generation"], previous.revision.generation);
25255                assert_eq!(event["candidate"]["generation"], revision.generation);
25256                assert_ne!(event["previous"]["persisted_pools_sha256"], event["candidate"]["persisted_pools_sha256"]);
25257                assert!(
25258                    implementation::PoolMetaPhaseBarrier::bind_in_directory(
25259                        directory.path().to_path_buf(),
25260                        &previous,
25261                        &candidate,
25262                        revision,
25263                        &durable,
25264                    )
25265                    .await
25266                    .expect("already claimed arm")
25267                    .is_none()
25268                );
25269            }
25270        }
25271
25272        #[cfg(feature = "e2e-test-hooks")]
25273        #[tokio::test]
25274        async fn pool_meta_phase_barrier_uses_persisted_responsibility_not_runtime_progress() {
25275            let fixture = Fixture::new(steps(4));
25276            let directory = tempfile::TempDir::new().expect("barrier directory");
25277            tokio::fs::write(
25278                directory.path().join("arm.json"),
25279                serde_json::to_vec(&serde_json::json!({"nonce": uuid::Uuid::new_v4(), "case": "prepare_subset"})).expect("arm"),
25280            )
25281            .await
25282            .expect("write arm");
25283            let mut candidate = fixture.previous.meta.clone();
25284            candidate.pools[0].decommission.as_mut().expect("progress").stage = "entry_started".to_owned();
25285            for changed in [false, true] {
25286                if changed {
25287                    candidate.pools[0]
25288                        .decommission
25289                        .as_mut()
25290                        .expect("progress")
25291                        .queued_buckets
25292                        .push("new-responsibility".to_owned());
25293                }
25294                let durable =
25295                    implementation::encode_pool_meta_v3_envelope(&candidate, fixture.revision, true, None).expect("candidate");
25296                let barrier = implementation::PoolMetaPhaseBarrier::bind_in_directory(
25297                    directory.path().to_path_buf(),
25298                    &fixture.previous,
25299                    &candidate,
25300                    fixture.revision,
25301                    &durable,
25302                )
25303                .await
25304                .expect("binding by persisted state");
25305                assert_eq!(barrier.is_some(), changed);
25306                assert_eq!(directory.path().join("claimed").exists(), changed);
25307                assert_eq!(directory.path().join("events.jsonl").exists(), changed);
25308            }
25309        }
25310
25311        fn persisted_pools(meta: &PoolMeta) -> serde_json::Value {
25312            serde_json::to_value(
25313                meta.pools
25314                    .iter()
25315                    .map(implementation::PersistedPoolStatus::from)
25316                    .collect::<Vec<_>>(),
25317            )
25318            .expect("persisted pool state")
25319        }
25320
25321        #[tokio::test]
25322        async fn pool_meta_v3_all_prepare_subsets_preserve_previous_snapshot() {
25323            for previous_version in [POOL_META_VERSION, POOL_META_GENERATION_VERSION] {
25324                for mask in 0usize..16 {
25325                    let mut fixture = Fixture::new(steps(4));
25326                    if previous_version != POOL_META_GENERATION_VERSION {
25327                        fixture.previous.meta.version = previous_version;
25328                        fixture.previous.revision = implementation::PoolMetaRevision::legacy(previous_version);
25329                        fixture.revision.generation = 1;
25330                        fixture.previous.canonical = fixture
25331                            .previous
25332                            .meta
25333                            .encode_config_data_for_v2_gate(true)
25334                            .expect("legacy predecessor");
25335                        fixture.pending = implementation::encode_pool_meta_v3_envelope(
25336                            &fixture.desired,
25337                            fixture.revision,
25338                            false,
25339                            Some(&fixture.previous),
25340                        )
25341                        .expect("legacy pending envelope");
25342                    }
25343                    let payloads = (0..4)
25344                        .map(|pool| {
25345                            if mask & (1 << pool) != 0 {
25346                                fixture.pending.clone()
25347                            } else {
25348                                fixture.previous.canonical.clone()
25349                            }
25350                        })
25351                        .collect::<Vec<_>>();
25352                    let selected = implementation::select_pool_meta_replica(
25353                        payloads
25354                            .iter()
25355                            .cloned()
25356                            .map(implementation::decode_pool_meta_replica)
25357                            .collect(),
25358                    )
25359                    .expect("every pending subset selects its predecessor");
25360                    assert_eq!(
25361                        persisted_pools(&selected.meta),
25362                        persisted_pools(&fixture.previous.meta),
25363                        "mask={mask:04b}"
25364                    );
25365                    assert_eq!(selected.revision, fixture.previous.revision);
25366                    if previous_version == POOL_META_GENERATION_VERSION {
25367                        for (pool, data) in fixture.pools.iter().zip(payloads) {
25368                            pool.inner.stored.lock().expect("fixture").as_mut().expect("replica").0 = data;
25369                        }
25370                        let mut state = fixture.state();
25371                        let selected = implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state)
25372                            .await
25373                            .expect("production recovery selection");
25374                        let outcome = implementation::repair_pool_meta_transaction(
25375                            fixture.pools.clone(),
25376                            &mut state,
25377                            selected,
25378                            &PoolMetaPersistenceFence::Distributed(None),
25379                        )
25380                        .await
25381                        .expect("production recovery repair");
25382                        assert_eq!(persisted_pools(&outcome.committed), persisted_pools(&fixture.previous.meta));
25383                        outcome.disarm();
25384                        let confirmed =
25385                            implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state)
25386                                .await
25387                                .expect("repaired reread");
25388                        assert_eq!(persisted_pools(&confirmed.meta), persisted_pools(&fixture.previous.meta));
25389                        assert!(!confirmed.replica_state.needs_repair);
25390                    }
25391                }
25392            }
25393        }
25394
25395        #[tokio::test]
25396        async fn pool_meta_v3_every_sole_commit_position_recovers_new_snapshot() {
25397            for committed_pool in 0..4 {
25398                let fixture = Fixture::new(steps(4));
25399                for (index, pool) in fixture.pools.iter().enumerate() {
25400                    pool.inner.stored.lock().expect("fixture").as_mut().expect("replica").0 = if index == committed_pool {
25401                        fixture.durable.clone()
25402                    } else {
25403                        fixture.pending.clone()
25404                    };
25405                }
25406                let mut state = fixture.state();
25407                let selected = implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state)
25408                    .await
25409                    .expect("sole commit must select new generation");
25410                assert_eq!(selected.revision, fixture.revision);
25411                assert_eq!(persisted_pools(&selected.meta), persisted_pools(&fixture.desired));
25412                let outcome = implementation::repair_pool_meta_transaction(
25413                    fixture.pools.clone(),
25414                    &mut state,
25415                    selected,
25416                    &PoolMetaPersistenceFence::Distributed(None),
25417                )
25418                .await
25419                .expect("repair partial commit");
25420                assert_eq!(persisted_pools(&outcome.committed), persisted_pools(&fixture.desired));
25421                outcome.disarm();
25422                let confirmed = implementation::load_pool_meta_for_transaction_recovery(fixture.pools.clone(), &mut state)
25423                    .await
25424                    .expect("repaired sole commit reread");
25425                assert_eq!(persisted_pools(&confirmed.meta), persisted_pools(&fixture.desired));
25426                assert!(!confirmed.replica_state.needs_repair);
25427                let fork = implementation::encode_pool_meta_v3_envelope(
25428                    &fixture.desired,
25429                    implementation::PoolMetaRevision {
25430                        transaction_id: Some(uuid::Uuid::new_v4()),
25431                        ..fixture.revision
25432                    },
25433                    true,
25434                    None,
25435                )
25436                .expect("same generation fork");
25437                assert!(
25438                    implementation::select_pool_meta_replica(vec![
25439                        implementation::decode_pool_meta_replica(fixture.durable.clone()),
25440                        implementation::decode_pool_meta_replica(fork)
25441                    ])
25442                    .is_err()
25443                );
25444            }
25445        }
25446    }
25447
25448    #[tokio::test]
25449    async fn test_pool_meta_cas_deterministically_rejects_stale_writer() {
25450        let storage = Arc::new(PartialPoolMetaWriteStorage {
25451            fail_write: false,
25452            fail_after_first_write: false,
25453            pending_write: false,
25454            write_started: tokio::sync::Notify::new(),
25455            wrote: AtomicBool::new(false),
25456            revision: AtomicUsize::new(0),
25457            stored: StdMutex::new(None),
25458            identity: StdMutex::new(None),
25459        });
25460        let stale_token = read_pool_meta_replica(storage.clone(), true).await.cas;
25461        assert!(matches!(&stale_token, PoolMetaCasToken::Missing));
25462        let winner = PoolMeta {
25463            version: POOL_META_VERSION,
25464            pools: vec![decommission_test_pool_status(0, None)],
25465            ..Default::default()
25466        }
25467        .encode_config_data_for_test()
25468        .expect("winning pool metadata should encode");
25469        let stale = PoolMeta {
25470            version: POOL_META_VERSION,
25471            pools: vec![decommission_test_pool_status(
25472                0,
25473                Some(PoolDecommissionInfo {
25474                    complete: true,
25475                    ..Default::default()
25476                }),
25477            )],
25478            ..Default::default()
25479        }
25480        .encode_config_data_for_test()
25481        .expect("stale pool metadata should encode");
25482        let fence = PoolMetaPersistenceFence::Distributed(None);
25483
25484        let mut arm = super::PoolMetaWriteState::default().arm_transaction();
25485        save_pool_meta_object_cas(
25486            storage.clone(),
25487            POOL_META_NAME,
25488            winner.clone(),
25489            &stale_token,
25490            &fence,
25491            "prepare_cas",
25492            &mut arm,
25493        )
25494        .await
25495        .expect("the first writer should create pool metadata");
25496        let err =
25497            save_pool_meta_object_cas(storage.clone(), POOL_META_NAME, stale, &stale_token, &fence, "prepare_cas", &mut arm)
25498                .await
25499                .expect_err("the second writer must not reuse the stale missing-object revision");
25500
25501        assert_eq!(err, Error::PreconditionFailed);
25502        let stored = storage
25503            .stored
25504            .lock()
25505            .expect("pool metadata test storage should not be poisoned")
25506            .as_ref()
25507            .map(|(data, _)| data.clone())
25508            .expect("the winning pool metadata should remain stored");
25509        assert_eq!(stored, winner);
25510    }
25511
25512    #[tokio::test]
25513    async fn test_pool_meta_v3_single_replica_commit_failure_recovers_on_restart() {
25514        let committed_replica = Arc::new(PartialPoolMetaWriteStorage {
25515            fail_write: false,
25516            fail_after_first_write: false,
25517            pending_write: false,
25518            write_started: tokio::sync::Notify::new(),
25519            wrote: AtomicBool::new(false),
25520            revision: AtomicUsize::new(0),
25521            stored: StdMutex::new(None),
25522            identity: StdMutex::new(None),
25523        });
25524        let pending_replica = Arc::new(PartialPoolMetaWriteStorage {
25525            fail_write: false,
25526            fail_after_first_write: true,
25527            pending_write: false,
25528            write_started: tokio::sync::Notify::new(),
25529            wrote: AtomicBool::new(false),
25530            revision: AtomicUsize::new(0),
25531            stored: StdMutex::new(None),
25532            identity: StdMutex::new(None),
25533        });
25534        let cluster_id = uuid::Uuid::new_v4();
25535        let fresh_bootstrap_nonce = uuid::Uuid::new_v4();
25536        let identity = super::encode_pool_meta_identity(super::PersistedPoolMetaIdentity {
25537            version: super::POOL_META_IDENTITY_VERSION,
25538            cluster_id,
25539            epoch: super::POOL_META_INITIAL_EPOCH,
25540            initialized: false,
25541            fresh_bootstrap_nonce: Some(fresh_bootstrap_nonce),
25542        })
25543        .expect("pending bootstrap identity should encode");
25544        *committed_replica
25545            .identity
25546            .lock()
25547            .expect("identity storage should not be poisoned") = Some((identity.clone(), "identity-0".to_string()));
25548        *pending_replica
25549            .identity
25550            .lock()
25551            .expect("identity storage should not be poisoned") = Some((identity, "identity-0".to_string()));
25552        let mut write_state = super::PoolMetaWriteState::for_startup(cluster_id, true);
25553        let snapshot = PoolMeta {
25554            version: super::POOL_META_GENERATION_VERSION,
25555            pools: vec![decommission_test_pool_status(
25556                0,
25557                Some(PoolDecommissionInfo {
25558                    queued: true,
25559                    ..Default::default()
25560                }),
25561            )],
25562            ..Default::default()
25563        };
25564
25565        snapshot
25566            .save_no_lock_observing(vec![committed_replica.clone(), pending_replica.clone()], &mut write_state)
25567            .await
25568            .expect("one committed replica should durably complete the V3 transaction");
25569
25570        let mut restarted = PoolMeta::default();
25571        let replica_state = restarted
25572            .load_no_lock_from_replicas(vec![committed_replica, pending_replica.clone()])
25573            .await
25574            .expect("restart should select the committed generation over a pending replica");
25575        assert_eq!(restarted.version, super::POOL_META_GENERATION_VERSION);
25576        assert!(restarted.pools[0].decommission.as_ref().is_some_and(|info| info.queued));
25577        assert!(replica_state.needs_repair);
25578
25579        let mut pending_only = PoolMeta::default();
25580        pending_only
25581            .load_no_lock_from_replicas(vec![pending_replica])
25582            .await
25583            .expect("a prepare-only replica should expose the embedded predecessor");
25584        assert!(pending_only.pools.is_empty());
25585    }
25586
25587    #[tokio::test]
25588    async fn test_partial_pool_meta_save_failure_blocks_following_side_effect() {
25589        let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
25590        let committed = Arc::new(PartialPoolMetaWriteStorage {
25591            fail_write: false,
25592            fail_after_first_write: false,
25593            pending_write: false,
25594            write_started: tokio::sync::Notify::new(),
25595            wrote: AtomicBool::new(false),
25596            revision: AtomicUsize::new(0),
25597            stored: StdMutex::new(None),
25598            identity: StdMutex::new(None),
25599        });
25600        let failed = Arc::new(PartialPoolMetaWriteStorage {
25601            fail_write: true,
25602            fail_after_first_write: false,
25603            pending_write: false,
25604            write_started: tokio::sync::Notify::new(),
25605            wrote: AtomicBool::new(false),
25606            revision: AtomicUsize::new(0),
25607            stored: StdMutex::new(None),
25608            identity: StdMutex::new(None),
25609        });
25610        let snapshot = PoolMeta {
25611            version: super::POOL_META_VERSION,
25612            pools: vec![decommission_test_pool_status(0, None)],
25613            ..Default::default()
25614        };
25615
25616        {
25617            let mut save_guard = store.pool_meta_save_gate.lock().await;
25618            snapshot
25619                .save_no_lock_observing(vec![committed.clone(), failed], &mut save_guard)
25620                .await
25621                .expect_err("the second replica write should fail after the first commits");
25622        }
25623        assert!(committed.wrote.load(Ordering::SeqCst));
25624
25625        let ran = Arc::new(AtomicBool::new(false));
25626        let ran_by_operation = ran.clone();
25627        let movement_gate = store.ctx.data_movement_operation_gate();
25628        let result: std::result::Result<(), Error> = store
25629            .run_guarded_decommission_side_effect(&CancellationToken::new(), &movement_gate, move || async move {
25630                ran_by_operation.store(true, Ordering::SeqCst);
25631                Ok(())
25632            })
25633            .await;
25634
25635        assert!(result.is_err(), "a partial pool metadata save must latch the sticky safety gate");
25636        assert!(!ran.load(Ordering::SeqCst), "the side effect must not run after a partial save");
25637    }
25638
25639    #[tokio::test]
25640    async fn test_cancelled_pool_meta_save_blocks_following_side_effect() {
25641        let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
25642        let committed = Arc::new(PartialPoolMetaWriteStorage {
25643            fail_write: false,
25644            fail_after_first_write: false,
25645            pending_write: false,
25646            write_started: tokio::sync::Notify::new(),
25647            wrote: AtomicBool::new(false),
25648            revision: AtomicUsize::new(0),
25649            stored: StdMutex::new(None),
25650            identity: StdMutex::new(None),
25651        });
25652        let pending = Arc::new(PartialPoolMetaWriteStorage {
25653            fail_write: false,
25654            fail_after_first_write: false,
25655            pending_write: true,
25656            write_started: tokio::sync::Notify::new(),
25657            wrote: AtomicBool::new(false),
25658            revision: AtomicUsize::new(0),
25659            stored: StdMutex::new(None),
25660            identity: StdMutex::new(None),
25661        });
25662        let snapshot = PoolMeta {
25663            version: super::POOL_META_VERSION,
25664            pools: vec![decommission_test_pool_status(0, None)],
25665            ..Default::default()
25666        };
25667        let save_store = store.clone();
25668        let save_committed = committed.clone();
25669        let save_pending = pending.clone();
25670        let save_task = tokio::spawn(async move {
25671            let mut save_guard = save_store.pool_meta_save_gate.lock().await;
25672            snapshot
25673                .save_no_lock_observing(vec![save_committed, save_pending], &mut save_guard)
25674                .await
25675        });
25676
25677        tokio::time::timeout(StdDuration::from_secs(1), pending.write_started.notified())
25678            .await
25679            .expect("the second replica write should start");
25680        assert!(committed.wrote.load(Ordering::SeqCst));
25681        save_task.abort();
25682        assert!(
25683            save_task.await.expect_err("the save task should be cancelled").is_cancelled(),
25684            "the pending replica write should be aborted"
25685        );
25686
25687        {
25688            let save_guard = store.pool_meta_save_gate.lock().await;
25689            save_guard
25690                .ensure_write_safe("cancelled pool metadata save")
25691                .expect_err("a cancelled replica update must latch the sticky safety gate");
25692        }
25693
25694        let ran = Arc::new(AtomicBool::new(false));
25695        let ran_by_operation = ran.clone();
25696        let movement_gate = store.ctx.data_movement_operation_gate();
25697        let result: std::result::Result<(), Error> = store
25698            .run_guarded_decommission_side_effect(&CancellationToken::new(), &movement_gate, move || async move {
25699                ran_by_operation.store(true, Ordering::SeqCst);
25700                Ok(())
25701            })
25702            .await;
25703
25704        assert!(result.is_err(), "a cancelled pool metadata save must block following side effects");
25705        assert!(!ran.load(Ordering::SeqCst), "the side effect must not run after a cancelled save");
25706    }
25707
25708    #[tokio::test]
25709    async fn test_cancelled_pool_meta_publish_keeps_write_gate_blocked() {
25710        let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
25711        let committed = Arc::new(PartialPoolMetaWriteStorage {
25712            fail_write: false,
25713            fail_after_first_write: false,
25714            pending_write: false,
25715            write_started: tokio::sync::Notify::new(),
25716            wrote: AtomicBool::new(false),
25717            revision: AtomicUsize::new(0),
25718            stored: StdMutex::new(None),
25719            identity: StdMutex::new(None),
25720        });
25721        let snapshot = PoolMeta {
25722            version: super::POOL_META_VERSION,
25723            pools: vec![decommission_test_pool_status(0, None)],
25724            ..Default::default()
25725        };
25726        let publish_started = Arc::new(tokio::sync::Notify::new());
25727        let publish_release = Arc::new(tokio::sync::Notify::new());
25728        let save_store = store.clone();
25729        let save_committed = committed.clone();
25730        let task_publish_started = publish_started.clone();
25731        let task_publish_release = publish_release.clone();
25732        let save_task = tokio::spawn(async move {
25733            let mut save_guard = save_store.pool_meta_save_gate.lock().await;
25734            let outcome = snapshot
25735                .save_no_lock_armed(vec![save_committed], &mut save_guard, None, &[0])
25736                .await?;
25737            task_publish_started.notify_one();
25738            task_publish_release.notified().await;
25739            outcome.disarm();
25740            Ok::<(), Error>(())
25741        });
25742
25743        tokio::time::timeout(StdDuration::from_secs(1), publish_started.notified())
25744            .await
25745            .expect("the replica save should complete before publication");
25746        assert!(committed.wrote.load(Ordering::SeqCst));
25747        save_task.abort();
25748        assert!(
25749            save_task
25750                .await
25751                .expect_err("the publication task should be cancelled")
25752                .is_cancelled(),
25753            "the task should be aborted while publication is pending"
25754        );
25755
25756        let save_guard = store.pool_meta_save_gate.lock().await;
25757        save_guard
25758            .ensure_write_safe("cancelled pool metadata publication")
25759            .expect_err("cancellation after replica save but before publication must keep writes blocked");
25760    }
25761
25762    #[tokio::test]
25763    async fn test_lost_pool_meta_fence_rejects_replica_write() {
25764        let client = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
25765        let lock = NamespaceLock::with_clients_and_quorum("pool-meta-fence-loss".to_string(), vec![client], 1);
25766        let request = LockRequest::new(
25767            ObjectKey::new(super::RUSTFS_META_BUCKET, super::POOL_META_NAME),
25768            LockType::Exclusive,
25769            "stale-writer",
25770        )
25771        .with_acquire_timeout(StdDuration::from_secs(1))
25772        .with_ttl(StdDuration::from_millis(50))
25773        .with_refresh_interval(StdDuration::from_millis(50));
25774        let guard = lock
25775            .acquire_guard(&request)
25776            .await
25777            .expect("pool metadata fence acquisition should not error")
25778            .expect("the stale writer should acquire the pool metadata fence");
25779        tokio::time::timeout(StdDuration::from_secs(2), guard.lock_lost_notified())
25780            .await
25781            .expect("the stale writer lease should expire");
25782
25783        let storage = Arc::new(PartialPoolMetaWriteStorage {
25784            fail_write: false,
25785            fail_after_first_write: false,
25786            pending_write: false,
25787            write_started: tokio::sync::Notify::new(),
25788            wrote: AtomicBool::new(false),
25789            revision: AtomicUsize::new(0),
25790            stored: StdMutex::new(None),
25791            identity: StdMutex::new(None),
25792        });
25793        let snapshot = PoolMeta {
25794            version: super::POOL_META_VERSION,
25795            pools: vec![decommission_test_pool_status(0, None)],
25796            ..Default::default()
25797        };
25798        let mut write_state = super::PoolMetaWriteState::default();
25799        let err = snapshot
25800            .save_no_lock_armed(vec![storage.clone()], &mut write_state, guard.lock_lost_signal(), &[0])
25801            .await
25802            .expect_err("a writer must not persist after losing the distributed pool metadata fence");
25803
25804        assert!(err.to_string().contains("distributed fence was lost"));
25805        assert!(!storage.wrote.load(Ordering::SeqCst), "the stale writer must not reach replica storage");
25806        write_state
25807            .ensure_write_safe("lost pool metadata fence")
25808            .expect("fence loss before dispatch has no unknown durable write and must remain retryable");
25809    }
25810
25811    #[tokio::test]
25812    async fn test_pool_meta_fence_loss_after_publish_keeps_write_gate_blocked() {
25813        let client = Arc::new(LocalClient::with_manager(Arc::new(GlobalLockManager::new())));
25814        let lock = NamespaceLock::with_clients_and_quorum("pool-meta-publish-fence-loss".to_string(), vec![client], 1);
25815        let request = LockRequest::new(
25816            ObjectKey::new(super::RUSTFS_META_BUCKET, super::POOL_META_NAME),
25817            LockType::Exclusive,
25818            "publishing-writer",
25819        )
25820        .with_acquire_timeout(StdDuration::from_secs(1))
25821        .with_ttl(StdDuration::from_secs(1))
25822        .with_refresh_interval(StdDuration::from_secs(1));
25823        let guard = lock
25824            .acquire_guard(&request)
25825            .await
25826            .expect("pool metadata fence acquisition should not error")
25827            .expect("the publishing writer should acquire the pool metadata fence");
25828        let storage = Arc::new(PartialPoolMetaWriteStorage {
25829            fail_write: false,
25830            fail_after_first_write: false,
25831            pending_write: false,
25832            write_started: tokio::sync::Notify::new(),
25833            wrote: AtomicBool::new(false),
25834            revision: AtomicUsize::new(0),
25835            stored: StdMutex::new(None),
25836            identity: StdMutex::new(None),
25837        });
25838        let mut saved = PoolMeta {
25839            version: super::POOL_META_VERSION,
25840            pools: vec![decommission_test_pool_status(0, None)],
25841            ..Default::default()
25842        };
25843        saved.pools[0].last_update = OffsetDateTime::UNIX_EPOCH + Duration::seconds(1);
25844        let mut current = saved.clone();
25845        current.pools[0].last_update = OffsetDateTime::UNIX_EPOCH;
25846        let mut write_state = super::PoolMetaWriteState::for_test_bootstrap();
25847        let outcome = saved
25848            .save_no_lock_armed(vec![storage], &mut write_state, guard.lock_lost_signal(), &[0])
25849            .await
25850            .expect("the replica save should finish while the fence is valid");
25851
25852        ensure_pool_meta_write_fence(&guard, "test pool metadata publish")
25853            .expect("the fence should remain valid before publication");
25854        publish_pool_meta_updates(&mut current, &saved, &[0]);
25855        tokio::time::timeout(StdDuration::from_secs(2), guard.lock_lost_notified())
25856            .await
25857            .expect("the fence should expire after publication");
25858        ensure_pool_meta_write_fence(&guard, "test pool metadata publish")
25859            .expect_err("the post-publication fence check must observe the loss");
25860
25861        assert_eq!(current.pools[0].last_update, saved.pools[0].last_update);
25862        drop(outcome);
25863        write_state
25864            .ensure_write_safe("lost pool metadata publish fence")
25865            .expect_err("the sticky write gate must remain armed after post-publication fence loss");
25866    }
25867
25868    #[tokio::test]
25869    async fn test_clear_decommission_transaction_survives_caller_abort() {
25870        let pool_meta = PoolMeta {
25871            version: super::POOL_META_VERSION,
25872            pools: vec![decommission_test_pool_status(
25873                0,
25874                Some(PoolDecommissionInfo {
25875                    failed: true,
25876                    ..Default::default()
25877                }),
25878            )],
25879            ..Default::default()
25880        };
25881        let store = decommission_worker_test_store(pool_meta, vec![None]);
25882        let save_started = Arc::new(tokio::sync::Notify::new());
25883        let save_release = Arc::new(tokio::sync::Notify::new());
25884        let save_completed = Arc::new(AtomicBool::new(false));
25885        let caller_store = store.clone();
25886        let caller_save_started = save_started.clone();
25887        let caller_save_release = save_release.clone();
25888        let caller_save_completed = save_completed.clone();
25889        let caller_task = tokio::spawn(async move {
25890            caller_store
25891                .clear_decommission_with_save(0, move || async move {
25892                    caller_save_started.notify_one();
25893                    caller_save_release.notified().await;
25894                    caller_save_completed.store(true, Ordering::SeqCst);
25895                    Ok(())
25896                })
25897                .await
25898        });
25899
25900        tokio::time::timeout(StdDuration::from_secs(1), save_started.notified())
25901            .await
25902            .expect("the clear transaction should reach persistence");
25903        caller_task.abort();
25904        assert!(
25905            caller_task
25906                .await
25907                .expect_err("the RPC waiter should be cancelled")
25908                .is_cancelled(),
25909            "the clear caller should be aborted while persistence is pending"
25910        );
25911        save_release.notify_one();
25912
25913        let _start_guard = tokio::time::timeout(StdDuration::from_secs(1), store.start_gate.lock())
25914            .await
25915            .expect("the detached clear transaction should finish");
25916        assert!(
25917            save_completed.load(Ordering::SeqCst),
25918            "the detached transaction must finish persistence after the caller is aborted"
25919        );
25920        let pool_meta = store.pool_meta.read().await;
25921        assert!(
25922            pool_meta.pools[0]
25923                .decommission
25924                .as_ref()
25925                .is_some_and(|info| !info.has_decommission_state())
25926        );
25927    }
25928
25929    fn decommission_test_pool_endpoint(idx: usize, is_local: bool) -> PoolEndpoints {
25930        let port = 9000usize + idx;
25931        let mut endpoint =
25932            Endpoint::try_from(format!("http://127.0.0.1:{port}/disk").as_str()).expect("test endpoint should parse");
25933        endpoint.is_local = is_local;
25934        endpoint.pool_idx = i32::try_from(idx).expect("test pool index should fit i32");
25935
25936        PoolEndpoints {
25937            legacy: false,
25938            set_count: 1,
25939            drives_per_set: 1,
25940            endpoints: Endpoints::from(vec![endpoint]),
25941            cmd_line: format!("pool-{idx}"),
25942            platform: String::new(),
25943        }
25944    }
25945
25946    fn decommission_test_pool_status(idx: usize, decommission: Option<PoolDecommissionInfo>) -> PoolStatus {
25947        PoolStatus {
25948            id: idx,
25949            cmd_line: format!("pool-{idx}"),
25950            last_update: OffsetDateTime::now_utc(),
25951            decommission,
25952        }
25953    }
25954
25955    fn decommission_test_active_model_meta(model_versions: &[u16]) -> PoolMeta {
25956        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
25957        PoolMeta {
25958            version: POOL_META_VERSION,
25959            pools: model_versions
25960                .iter()
25961                .copied()
25962                .enumerate()
25963                .map(|(idx, model_version)| {
25964                    let reservation = build_decommission_capacity_reservation_with_model(
25965                        DecommissionPoolCapacityInfo::for_test(idx, layout, 0, 10, 10),
25966                        layout,
25967                        uuid::Uuid::new_v4(),
25968                        u64::try_from(idx).unwrap_or_default() + 1,
25969                        OffsetDateTime::UNIX_EPOCH,
25970                        model_version,
25971                    )
25972                    .expect("test capacity model should be supported");
25973                    decommission_test_pool_status(
25974                        idx,
25975                        Some(PoolDecommissionInfo {
25976                            start_time: Some(OffsetDateTime::UNIX_EPOCH),
25977                            capacity_reservation: Some(reservation),
25978                            ..Default::default()
25979                        }),
25980                    )
25981                })
25982                .collect(),
25983            ..Default::default()
25984        }
25985    }
25986
25987    fn decommission_test_cleanup_meta(
25988        model_version: u16,
25989        temporary_mutations: Vec<DecommissionCapacityTemporaryMutation>,
25990        inflight_physical_bytes: usize,
25991        pending: Option<(uuid::Uuid, usize)>,
25992    ) -> PoolMeta {
25993        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
25994        let mut reservation = build_decommission_capacity_reservation_with_model(
25995            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
25996            layout,
25997            uuid::Uuid::new_v4(),
25998            1,
25999            OffsetDateTime::UNIX_EPOCH,
26000            model_version,
26001        )
26002        .expect("test cleanup reservation should be valid");
26003        let (pending_mutation_id, pending_physical_bytes) = pending.unzip();
26004        let pending_physical_bytes = pending_physical_bytes.unwrap_or_default();
26005        reservation.observed_target_physical_bytes = inflight_physical_bytes;
26006        reservation.inflight_target_physical_bytes = inflight_physical_bytes;
26007        reservation.pending_target_physical_bytes = pending_physical_bytes;
26008        reservation.targets.push(DecommissionCapacityTarget {
26009            pool_index: 1,
26010            layout,
26011            physical_total_at_reservation: 200,
26012            physical_free_at_reservation: 200,
26013            reserved_physical_bytes: reservation.peak_physical_bytes,
26014            consumed_physical_bytes: 0,
26015            observed_physical_bytes: inflight_physical_bytes,
26016            inflight_physical_bytes,
26017            pending_physical_bytes,
26018            pending_mutation_id,
26019            temporary_mutations,
26020        });
26021        PoolMeta {
26022            version: POOL_META_VERSION,
26023            pools: vec![
26024                decommission_test_pool_status(
26025                    0,
26026                    Some(PoolDecommissionInfo {
26027                        start_time: Some(OffsetDateTime::UNIX_EPOCH),
26028                        capacity_reservation: Some(reservation),
26029                        ..Default::default()
26030                    }),
26031                ),
26032                decommission_test_pool_status(1, None),
26033            ],
26034            ..Default::default()
26035        }
26036    }
26037
26038    #[tokio::test]
26039    async fn test_activation_fence_uses_one_lock_order_and_serializes_callers() {
26040        let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
26041        let first = Arc::new(ActivationLockRecorder {
26042            lock_manager: Arc::clone(&manager),
26043            owner: "first",
26044            resources: StdMutex::new(Vec::new()),
26045        });
26046        let second = Arc::new(ActivationLockRecorder {
26047            lock_manager: manager,
26048            owner: "second",
26049            resources: StdMutex::new(Vec::new()),
26050        });
26051
26052        let first_guards = acquire_pool_rebalance_activation_locks(first.clone(), None)
26053            .await
26054            .expect("first activation should acquire both locks");
26055        assert_eq!(
26056            *first
26057                .resources
26058                .lock()
26059                .expect("activation lock recorder should not be poisoned"),
26060            vec![REBAL_META_NAME.to_string(), POOL_META_NAME.to_string()]
26061        );
26062
26063        let mut second_acquire = Box::pin(acquire_pool_rebalance_activation_locks(second.clone(), None));
26064        let mut context = Context::from_waker(futures::task::noop_waker_ref());
26065        assert!(matches!(second_acquire.as_mut().poll(&mut context), Poll::Pending));
26066
26067        drop(first_guards);
26068        second_acquire
26069            .await
26070            .expect("second activation should acquire both locks after the first releases them");
26071        assert_eq!(
26072            *second
26073                .resources
26074                .lock()
26075                .expect("activation lock recorder should not be poisoned"),
26076            vec![REBAL_META_NAME.to_string(), POOL_META_NAME.to_string()]
26077        );
26078    }
26079
26080    #[tokio::test]
26081    async fn test_activation_cancellation_releases_rebalance_fence_while_pool_fence_is_contended() {
26082        use crate::storage_api_contracts::namespace::NamespaceLocking as _;
26083        let pool = Arc::new(ActivationLockRecorder {
26084            lock_manager: Arc::new(rustfs_lock::GlobalLockManager::new()),
26085            owner: "activation-cancellation",
26086            resources: StdMutex::new(Vec::new()),
26087        });
26088        let pool_lock = pool
26089            .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, POOL_META_NAME)
26090            .await
26091            .expect("pool lock should be created");
26092        let pool_reader = pool_lock
26093            .get_read_lock(std::time::Duration::from_secs(5))
26094            .await
26095            .expect("ordinary mutation should hold the pool read fence");
26096        pool.resources.lock().expect("recorder should not be poisoned").clear();
26097        let mut activation = Box::pin(acquire_pool_rebalance_activation_locks(Arc::clone(&pool), None));
26098        assert!(matches!(futures::poll!(&mut activation), Poll::Pending));
26099        assert_eq!(
26100            *pool.resources.lock().expect("recorder should not be poisoned"),
26101            vec![REBAL_META_NAME.to_string(), POOL_META_NAME.to_string()],
26102            "activation must hold the run fence before waiting for the pool fence",
26103        );
26104        drop(activation);
26105        let rebalance_lock = pool
26106            .new_ns_lock(crate::disk::RUSTFS_META_BUCKET, REBAL_META_NAME)
26107            .await
26108            .expect("run lock should be created");
26109        let run_writer = rebalance_lock
26110            .get_write_lock(std::time::Duration::from_secs(5))
26111            .await
26112            .expect("cancelling activation must release its already-acquired run fence");
26113        assert!(
26114            !pool_reader.is_released(),
26115            "cancelling activation must not release another caller's pool fence"
26116        );
26117        assert!(!run_writer.is_lock_lost());
26118    }
26119
26120    #[test]
26121    fn decommission_receipt_run_token_changes_with_persisted_start_time() {
26122        let first = OffsetDateTime::from_unix_timestamp(1_000).expect("first run timestamp should be valid");
26123        let second = OffsetDateTime::from_unix_timestamp(2_000).expect("second run timestamp should be valid");
26124        let first_token = decommission_durable_ilm_receipt_run_token("pool-0", first);
26125        let second_token = decommission_durable_ilm_receipt_run_token("pool-0", second);
26126
26127        assert_ne!(first_token, second_token);
26128        assert_eq!(first_token, decommission_durable_ilm_receipt_run_token("pool-0", first));
26129        let operation_id = "a".repeat(64);
26130        let old_receipt = decommission_durable_ilm_receipt_path(
26131            &first_token,
26132            &format!("ilm/tier-delete-journal/{operation_id}.json"),
26133            "operation_id",
26134            &operation_id,
26135        );
26136        assert!(!old_receipt.starts_with(&decommission_durable_ilm_receipt_run_prefix(&second_token)));
26137    }
26138
26139    #[test]
26140    fn decommission_recovery_control_receipt_path_round_trips() {
26141        let run_token = "b".repeat(64);
26142        let control_id = "a".repeat(64);
26143        let source_path = format!(
26144            "ilm/recovery-controls/transition_transaction/{}/{}/{}.json",
26145            &control_id[..2],
26146            &control_id[2..4],
26147            control_id
26148        );
26149        let path = decommission_durable_ilm_receipt_path(&run_token, &source_path, "control_id", &control_id);
26150
26151        let locator = parse_decommission_durable_ilm_receipt_path(&path).expect("recovery control receipt path should parse");
26152
26153        assert_eq!(locator.run_token, run_token);
26154        assert_eq!(locator.source_path, source_path);
26155        assert_eq!(locator.id_kind, "control_id");
26156        assert_eq!(locator.id, control_id);
26157    }
26158
26159    #[test]
26160    fn decommission_receipt_merge_preserves_terminal_proof() {
26161        let operation_id = "a".repeat(64);
26162        let source_path = format!("ilm/tier-delete-journal/{operation_id}.json");
26163        let checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal {
26164            content_sha256: "b".repeat(64),
26165            identity_sha256: "c".repeat(64),
26166            committed: false,
26167            dispatch_identity_sha256: None,
26168            state: None,
26169        };
26170        let terminal_checkpoint = DurableIlmRecordCheckpoint::TierDeleteJournal {
26171            content_sha256: "d".repeat(64),
26172            identity_sha256: "c".repeat(64),
26173            committed: true,
26174            dispatch_identity_sha256: None,
26175            state: None,
26176        };
26177        let incoming = DecommissionDurableIlmReceipt {
26178            source_path,
26179            namespace: "tier-delete-journal".to_string(),
26180            id_kind: "operation_id".to_string(),
26181            id: operation_id,
26182            checkpoint: checkpoint.clone(),
26183            terminal_checkpoint: None,
26184            fleet_topology_generation: None,
26185        };
26186        let existing = DecommissionDurableIlmReceipt {
26187            terminal_checkpoint: Some(terminal_checkpoint.clone()),
26188            ..incoming.clone()
26189        };
26190
26191        let merged = merge_decommission_durable_ilm_receipts(&existing, &incoming)
26192            .expect("retry receipt must merge with a terminal receipt");
26193
26194        assert_eq!(merged.checkpoint, checkpoint);
26195        assert_eq!(merged.terminal_checkpoint, Some(terminal_checkpoint.clone()));
26196
26197        let topology_bound = DecommissionDurableIlmReceipt {
26198            fleet_topology_generation: Some("e".repeat(64)),
26199            ..incoming
26200        };
26201        let mixed_error = merge_decommission_durable_ilm_receipts(&existing, &topology_bound)
26202            .expect_err("a topology-bound v6 receipt must not mask an unbound receipt")
26203            .to_string();
26204        assert!(mixed_error.contains("fleet topology conflict"));
26205
26206        let topology_existing = DecommissionDurableIlmReceipt {
26207            terminal_checkpoint: Some(terminal_checkpoint),
26208            ..topology_bound.clone()
26209        };
26210        let topology_merged = merge_decommission_durable_ilm_receipts(&topology_existing, &topology_bound)
26211            .expect("receipts bound to the same fleet topology should merge");
26212        assert_eq!(topology_merged.fleet_topology_generation, Some("e".repeat(64)));
26213    }
26214
26215    #[test]
26216    fn decommission_manual_job_receipt_compacts_large_progress() {
26217        let prefix = "p".repeat(12 * 1024);
26218        let options = ManualTransitionRunOptions {
26219            prefix,
26220            ..Default::default()
26221        };
26222        let mut job = ManualTransitionJobRecord::new(uuid::Uuid::new_v4(), "bounded-receipt-bucket", &options, "owner");
26223        let token_bytes = serde_json::to_vec(&serde_json::json!({
26224            "marker": "m".repeat(12 * 1024),
26225            "version_marker": "opaque-version"
26226        }))
26227        .expect("large continuation token should encode");
26228        let mut report = job.report.clone();
26229        report.scanned = 1;
26230        report.continuation_token = Some(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&token_bytes));
26231        job.update_running_progress(report, ManualTransitionQueueSnapshot::default());
26232        let path = manual_transition_job_record_object_name(job.job_id).expect("manual job path should build");
26233        let job_bytes = job.encode().expect("large manual job should remain within its record limit");
26234        assert!(job_bytes.len() > DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE);
26235        let record = validate_durable_ilm_record(&path, &job_bytes).expect("large manual job should validate");
26236        let expected_checkpoint = record.checkpoint.clone();
26237        let mut receipt = DecommissionDurableIlmReceipt::new(&path, &record, None);
26238        receipt.terminal_checkpoint = Some(record.checkpoint);
26239
26240        let encoded = receipt.encode().expect("bounded progress proof should fit the receipt limit");
26241        let decoded = DecommissionDurableIlmReceipt::decode(&encoded).expect("bounded receipt should round trip");
26242
26243        assert!(encoded.len() <= DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE);
26244        assert_eq!(decoded.source_path, path);
26245        assert_eq!(decoded.checkpoint, expected_checkpoint);
26246        assert_eq!(decoded.terminal_checkpoint, Some(expected_checkpoint));
26247    }
26248
26249    #[test]
26250    fn test_apply_decommission_status_space_info_adds_idle_pool_usage() {
26251        let status = apply_decommission_status_space_info(
26252            decommission_test_pool_status(0, None),
26253            PoolSpaceInfo {
26254                free: 25,
26255                total: 100,
26256                used: 75,
26257            },
26258        );
26259
26260        let decommission = status.decommission.expect("idle pool status should include usage info");
26261        assert_eq!(decommission.total_size, 100);
26262        assert_eq!(decommission.current_size, 25);
26263        assert!(decommission.start_time.is_none());
26264        assert!(!decommission.complete);
26265        assert!(!decommission.failed);
26266        assert!(!decommission.canceled);
26267    }
26268
26269    #[test]
26270    fn test_apply_decommission_status_space_info_refreshes_active_decommission_sizes() {
26271        let status = apply_decommission_status_space_info(
26272            decommission_test_pool_status(
26273                0,
26274                Some(PoolDecommissionInfo {
26275                    total_size: 1,
26276                    current_size: 1,
26277                    ..Default::default()
26278                }),
26279            ),
26280            PoolSpaceInfo {
26281                free: 25,
26282                total: 100,
26283                used: 75,
26284            },
26285        );
26286
26287        let decommission = status.decommission.expect("active decommission info should remain present");
26288        assert_eq!(decommission.total_size, 100);
26289        assert_eq!(decommission.current_size, 25);
26290    }
26291
26292    #[test]
26293    fn test_merge_pool_status_refresh_uses_persisted_terminal_decommission() {
26294        let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
26295        let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
26296        let mut current = PoolMeta {
26297            pools: vec![PoolStatus {
26298                id: 0,
26299                cmd_line: "pool-0".to_string(),
26300                last_update: older,
26301                decommission: Some(PoolDecommissionInfo {
26302                    start_time: Some(older),
26303                    ..Default::default()
26304                }),
26305            }],
26306            ..Default::default()
26307        };
26308        let persisted = PoolMeta {
26309            pools: vec![PoolStatus {
26310                id: 0,
26311                cmd_line: "pool-0".to_string(),
26312                last_update: newer,
26313                decommission: Some(PoolDecommissionInfo {
26314                    complete: true,
26315                    ..Default::default()
26316                }),
26317            }],
26318            ..Default::default()
26319        };
26320
26321        assert!(merge_pool_status_refresh(&mut current, persisted, &[false]));
26322
26323        let info = current.pools[0]
26324            .decommission
26325            .as_ref()
26326            .expect("decommission info should be present");
26327        assert!(info.complete);
26328        assert!(!info.failed);
26329        assert!(!info.canceled);
26330    }
26331
26332    #[test]
26333    fn test_merge_pool_status_refresh_keeps_newer_local_active_progress() {
26334        let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
26335        let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
26336        let mut current = PoolMeta {
26337            pools: vec![PoolStatus {
26338                id: 0,
26339                cmd_line: "pool-0".to_string(),
26340                last_update: newer,
26341                decommission: Some(PoolDecommissionInfo {
26342                    start_time: Some(older),
26343                    items_decommissioned: 10,
26344                    bytes_done: 1_024,
26345                    ..Default::default()
26346                }),
26347            }],
26348            ..Default::default()
26349        };
26350        let persisted = PoolMeta {
26351            pools: vec![PoolStatus {
26352                id: 0,
26353                cmd_line: "pool-0".to_string(),
26354                last_update: older,
26355                decommission: Some(PoolDecommissionInfo {
26356                    start_time: Some(older),
26357                    items_decommissioned: 1,
26358                    bytes_done: 128,
26359                    ..Default::default()
26360                }),
26361            }],
26362            ..Default::default()
26363        };
26364
26365        assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
26366
26367        let info = current.pools[0]
26368            .decommission
26369            .as_ref()
26370            .expect("local decommission info should remain present");
26371        assert_eq!(info.items_decommissioned, 10);
26372        assert_eq!(info.bytes_done, 1_024);
26373    }
26374
26375    #[test]
26376    fn test_merge_pool_status_refresh_preserves_observed_v2_floor() {
26377        let timestamp = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
26378        let mut current = PoolMeta {
26379            version: POOL_META_V1_VERSION,
26380            pools: vec![PoolStatus {
26381                id: 0,
26382                cmd_line: "pool-0".to_string(),
26383                last_update: timestamp,
26384                decommission: Some(PoolDecommissionInfo::default()),
26385            }],
26386            dont_save: false,
26387        };
26388        let persisted = PoolMeta {
26389            version: POOL_META_VERSION,
26390            pools: current.pools.clone(),
26391            dont_save: false,
26392        };
26393
26394        assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
26395        assert_eq!(current.version, POOL_META_VERSION);
26396        let encoded = current
26397            .encode_config_data_for_v2_gate(false)
26398            .expect("a peer-observed v2 floor must remain sticky");
26399        assert_eq!(LittleEndian::read_u16(&encoded[2..4]), POOL_META_VERSION);
26400    }
26401
26402    #[test]
26403    fn test_merge_pool_status_refresh_keeps_newer_local_active_over_older_terminal() {
26404        let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
26405        let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
26406        let mut current = PoolMeta {
26407            pools: vec![PoolStatus {
26408                id: 0,
26409                cmd_line: "pool-0".to_string(),
26410                last_update: newer,
26411                decommission: Some(PoolDecommissionInfo {
26412                    start_time: Some(older),
26413                    items_decommissioned: 10,
26414                    bytes_done: 1_024,
26415                    ..Default::default()
26416                }),
26417            }],
26418            ..Default::default()
26419        };
26420        let persisted = PoolMeta {
26421            pools: vec![PoolStatus {
26422                id: 0,
26423                cmd_line: "pool-0".to_string(),
26424                last_update: older,
26425                decommission: Some(PoolDecommissionInfo {
26426                    failed: true,
26427                    ..Default::default()
26428                }),
26429            }],
26430            ..Default::default()
26431        };
26432
26433        assert!(!merge_pool_status_refresh(&mut current, persisted, &[true]));
26434
26435        let info = current.pools[0]
26436            .decommission
26437            .as_ref()
26438            .expect("local active decommission info should remain present");
26439        assert!(!info.failed);
26440        assert_eq!(info.items_decommissioned, 10);
26441        assert_eq!(info.bytes_done, 1_024);
26442    }
26443
26444    #[test]
26445    fn test_merge_pool_status_refresh_fails_closed_on_missing_persisted_pools() {
26446        let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
26447        let mut current = PoolMeta {
26448            pools: vec![decommission_test_pool_status(
26449                0,
26450                Some(PoolDecommissionInfo {
26451                    complete: true,
26452                    ..Default::default()
26453                }),
26454            )],
26455            ..Default::default()
26456        };
26457        current.pools[0].last_update = newer;
26458
26459        assert!(
26460            !merge_pool_status_refresh(&mut current, PoolMeta::default(), &[false]),
26461            "an empty persisted snapshot must fail closed instead of replacing local state"
26462        );
26463
26464        let info = current.pools[0]
26465            .decommission
26466            .as_ref()
26467            .expect("local decommission info should survive a missing snapshot");
26468        assert!(info.complete);
26469        assert_eq!(current.pools[0].last_update, newer);
26470    }
26471
26472    #[test]
26473    fn test_merge_pool_status_refresh_ignores_mislabeled_pool_entries() {
26474        let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
26475        let mut current = PoolMeta {
26476            pools: vec![decommission_test_pool_status(0, None)],
26477            ..Default::default()
26478        };
26479        let mut persisted = PoolMeta {
26480            pools: vec![decommission_test_pool_status(0, Some(PoolDecommissionInfo::default()))],
26481            ..Default::default()
26482        };
26483        persisted.pools[0].id = 7;
26484        persisted.pools[0].last_update = older;
26485
26486        assert!(
26487            !merge_pool_status_refresh(&mut current, persisted, &[false]),
26488            "a pool entry whose id does not match its index must be ignored"
26489        );
26490        assert!(current.pools[0].decommission.is_none());
26491    }
26492
26493    #[test]
26494    fn test_pool_meta_save_merge_preserves_newer_untouched_pool() {
26495        let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
26496        let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
26497        let mut current = PoolMeta {
26498            pools: vec![
26499                PoolStatus {
26500                    id: 0,
26501                    cmd_line: "pool-0".to_string(),
26502                    last_update: older,
26503                    decommission: Some(PoolDecommissionInfo {
26504                        failed: true,
26505                        ..Default::default()
26506                    }),
26507                },
26508                PoolStatus {
26509                    id: 1,
26510                    cmd_line: "pool-1".to_string(),
26511                    last_update: older,
26512                    decommission: None,
26513                },
26514            ],
26515            ..Default::default()
26516        };
26517        let mut persisted = current.clone();
26518        persisted.pools[1].last_update = newer;
26519        persisted.pools[1].decommission = Some(PoolDecommissionInfo {
26520            failed: true,
26521            ..Default::default()
26522        });
26523
26524        assert!(current.clear_decommission(0).expect("terminal decommission should clear"));
26525        merge_pool_meta_updates_for_save(&mut persisted, &current, &[0], "clear decommission")
26526            .expect("the target pool update should merge into the latest snapshot");
26527
26528        assert!(
26529            persisted.pools[0]
26530                .decommission
26531                .as_ref()
26532                .is_some_and(|info| !info.has_decommission_state())
26533        );
26534        assert_eq!(persisted.pools[1].last_update, newer);
26535        assert!(persisted.pools[1].decommission.as_ref().is_some_and(|info| info.failed));
26536    }
26537
26538    #[test]
26539    fn test_pool_meta_save_merge_rejects_stale_terminal_resurrection() {
26540        let older = OffsetDateTime::from_unix_timestamp(1_000).expect("test timestamp should be valid");
26541        let newer = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
26542        let mut persisted = PoolMeta {
26543            pools: vec![PoolStatus {
26544                id: 0,
26545                cmd_line: "pool-0".to_string(),
26546                last_update: newer,
26547                decommission: Some(PoolDecommissionInfo {
26548                    canceled: true,
26549                    ..Default::default()
26550                }),
26551            }],
26552            ..Default::default()
26553        };
26554        let stale = PoolMeta {
26555            pools: vec![PoolStatus {
26556                id: 0,
26557                cmd_line: "pool-0".to_string(),
26558                last_update: older,
26559                decommission: Some(PoolDecommissionInfo {
26560                    start_time: Some(older),
26561                    ..Default::default()
26562                }),
26563            }],
26564            ..Default::default()
26565        };
26566
26567        let err = merge_pool_meta_updates_for_save(&mut persisted, &stale, &[0], "stale writer")
26568            .expect_err("a stale active snapshot must not resurrect a canceled pool");
26569
26570        assert!(err.to_string().contains("stale pool metadata update rejected"));
26571        assert!(persisted.pools[0].decommission.as_ref().is_some_and(|info| info.canceled));
26572    }
26573
26574    #[test]
26575    fn test_pool_meta_save_merge_rejects_clear_over_completed_or_unresolved_state() {
26576        let timestamp = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid");
26577        for persisted_info in [
26578            PoolDecommissionInfo {
26579                complete: true,
26580                ..Default::default()
26581            },
26582            PoolDecommissionInfo {
26583                failed: true,
26584                unresolved_entries: vec![DecommissionUnresolvedEntry {
26585                    bucket: "bucket-a".to_string(),
26586                    object: "object-a".to_string(),
26587                    pool_index: 0,
26588                    set_index: 0,
26589                    source_generation: timestamp,
26590                    candidate_count: 1,
26591                    disk_error_count: 1,
26592                    observed_at: timestamp,
26593                    reason: "test unresolved entry".to_string(),
26594                }],
26595                ..Default::default()
26596            },
26597        ] {
26598            let mut persisted = PoolMeta {
26599                pools: vec![PoolStatus {
26600                    id: 0,
26601                    cmd_line: "pool-0".to_string(),
26602                    last_update: timestamp,
26603                    decommission: Some(persisted_info),
26604                }],
26605                ..Default::default()
26606            };
26607            let cleared = PoolMeta {
26608                pools: vec![PoolStatus {
26609                    id: 0,
26610                    cmd_line: "pool-0".to_string(),
26611                    last_update: OffsetDateTime::now_utc(),
26612                    decommission: Some(PoolDecommissionInfo::default()),
26613                }],
26614                ..Default::default()
26615            };
26616
26617            let err = merge_pool_meta_updates_for_save(&mut persisted, &cleared, &[0], "clear decommission")
26618                .expect_err("a stale clear must not erase completed or unresolved state");
26619
26620            assert!(
26621                err.to_string()
26622                    .contains("completed or unresolved decommission state cannot be cleared")
26623            );
26624        }
26625    }
26626
26627    #[test]
26628    fn test_pool_meta_publish_preserves_untouched_runtime_progress() {
26629        let mut current = PoolMeta {
26630            pools: vec![
26631                decommission_test_pool_status(0, None),
26632                decommission_test_pool_status(
26633                    1,
26634                    Some(PoolDecommissionInfo {
26635                        items_decommissioned: 10,
26636                        bytes_done: 1_024,
26637                        ..Default::default()
26638                    }),
26639                ),
26640            ],
26641            ..Default::default()
26642        };
26643        let mut saved = current.clone();
26644        saved.pools[0].decommission = Some(PoolDecommissionInfo {
26645            complete: true,
26646            ..Default::default()
26647        });
26648        let saved_pool_1 = saved.pools[1].decommission.as_mut().expect("pool 1 progress should exist");
26649        saved_pool_1.items_decommissioned = 1;
26650        saved_pool_1.bytes_done = 128;
26651
26652        publish_pool_meta_updates(&mut current, &saved, &[0]);
26653
26654        assert!(current.pools[0].decommission.as_ref().is_some_and(|info| info.complete));
26655        let pool_1 = current.pools[1]
26656            .decommission
26657            .as_ref()
26658            .expect("pool 1 progress should remain present");
26659        assert_eq!(pool_1.items_decommissioned, 10);
26660        assert_eq!(pool_1.bytes_done, 1_024);
26661    }
26662
26663    #[test]
26664    fn test_dedup_indices_removes_duplicates_preserving_order() {
26665        assert_eq!(dedup_indices(&[0, 2, 1, 2, 3, 0]), vec![0, 2, 1, 3]);
26666    }
26667
26668    #[test]
26669    fn test_dedup_indices_handles_empty_input() {
26670        let empty: Vec<usize> = Vec::new();
26671        assert!(dedup_indices(&empty).is_empty());
26672    }
26673
26674    #[test]
26675    fn test_default_decommission_bucket_concurrency_is_conservative() {
26676        assert_eq!(default_decommission_bucket_concurrency(0), 1);
26677        assert_eq!(default_decommission_bucket_concurrency(1), 1);
26678        assert_eq!(default_decommission_bucket_concurrency(2), 2);
26679        assert_eq!(default_decommission_bucket_concurrency(8), 4);
26680    }
26681
26682    #[test]
26683    fn test_default_decommission_entry_concurrency_is_conservative() {
26684        assert_eq!(default_decommission_entry_concurrency(0), 1);
26685        assert_eq!(default_decommission_entry_concurrency(1), 1);
26686        assert_eq!(default_decommission_entry_concurrency(4), 4);
26687        assert_eq!(default_decommission_entry_concurrency(16), DECOMMISSION_ENTRY_CONCURRENCY_DEFAULT_CAP);
26688    }
26689
26690    #[test]
26691    fn test_decommission_entry_concurrency_clamps_operator_configuration() {
26692        assert_eq!(clamp_decommission_entry_concurrency(0), 1);
26693        assert_eq!(clamp_decommission_entry_concurrency(1), 1);
26694        assert_eq!(
26695            clamp_decommission_entry_concurrency(DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP),
26696            DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP
26697        );
26698        assert_eq!(clamp_decommission_entry_concurrency(usize::MAX), DECOMMISSION_ENTRY_CONCURRENCY_HARD_CAP);
26699    }
26700
26701    #[test]
26702    fn test_split_decommission_buckets_keeps_meta_buckets_last() {
26703        let (regular, meta) = split_decommission_buckets(vec![
26704            DecomBucketInfo {
26705                name: "bucket-a".to_string(),
26706                ..Default::default()
26707            },
26708            DecomBucketInfo {
26709                name: crate::disk::RUSTFS_META_BUCKET.to_string(),
26710                prefix: crate::config::com::CONFIG_PREFIX.to_string(),
26711            },
26712            DecomBucketInfo {
26713                name: "bucket-b".to_string(),
26714                ..Default::default()
26715            },
26716            DecomBucketInfo {
26717                name: crate::disk::RUSTFS_META_BUCKET.to_string(),
26718                prefix: crate::disk::BUCKET_META_PREFIX.to_string(),
26719            },
26720            DecomBucketInfo {
26721                name: crate::disk::RUSTFS_META_BUCKET.to_string(),
26722                prefix: crate::bucket::lifecycle::ILM_META_PREFIX.to_string(),
26723            },
26724        ]);
26725
26726        assert_eq!(
26727            regular.iter().map(|bucket| bucket.name.as_str()).collect::<Vec<_>>(),
26728            vec!["bucket-a", "bucket-b",]
26729        );
26730        assert_eq!(
26731            meta.iter().map(|bucket| bucket.prefix.as_str()).collect::<Vec<_>>(),
26732            vec![
26733                crate::config::com::CONFIG_PREFIX,
26734                crate::disk::BUCKET_META_PREFIX,
26735                crate::bucket::lifecycle::ILM_META_PREFIX,
26736            ]
26737        );
26738    }
26739
26740    #[test]
26741    fn test_resume_reconciles_missing_decommission_meta_prefixes() {
26742        let mut meta = PoolMeta {
26743            pools: vec![decommission_test_pool_status(
26744                0,
26745                Some(PoolDecommissionInfo {
26746                    queued_buckets: vec![
26747                        format!("{}/{}", crate::disk::RUSTFS_META_BUCKET, crate::config::com::CONFIG_PREFIX),
26748                        format!("{}/{}", crate::disk::RUSTFS_META_BUCKET, crate::disk::BUCKET_META_PREFIX),
26749                    ],
26750                    ..Default::default()
26751                }),
26752            )],
26753            ..Default::default()
26754        };
26755
26756        assert!(reconcile_decommission_meta_buckets(&mut meta, 0));
26757        assert_eq!(
26758            meta.pending_buckets(0)
26759                .iter()
26760                .filter(|bucket| bucket.name == crate::disk::RUSTFS_META_BUCKET)
26761                .map(|bucket| bucket.prefix.as_str())
26762                .collect::<Vec<_>>(),
26763            DECOMMISSION_META_PREFIXES
26764        );
26765        assert!(!reconcile_decommission_meta_buckets(&mut meta, 0));
26766    }
26767
26768    #[tokio::test]
26769    async fn test_decommission_metadata_phase_precedes_regular_failure() {
26770        let events = Arc::new(StdMutex::new(Vec::new()));
26771        let err = run_decommission_phases(
26772            CancellationToken::new(),
26773            vec![
26774                DecomBucketInfo {
26775                    name: "regular-fails".to_string(),
26776                    ..Default::default()
26777                },
26778                DecomBucketInfo {
26779                    name: "regular-not-started".to_string(),
26780                    ..Default::default()
26781                },
26782            ],
26783            vec![
26784                DecomBucketInfo {
26785                    name: crate::disk::RUSTFS_META_BUCKET.to_string(),
26786                    prefix: crate::config::com::CONFIG_PREFIX.to_string(),
26787                },
26788                DecomBucketInfo {
26789                    name: crate::disk::RUSTFS_META_BUCKET.to_string(),
26790                    prefix: crate::disk::BUCKET_META_PREFIX.to_string(),
26791                },
26792            ],
26793            1,
26794            {
26795                let events = Arc::clone(&events);
26796                move |bucket, _rx| {
26797                    let events = Arc::clone(&events);
26798                    Box::pin(async move {
26799                        let event = if bucket.name == crate::disk::RUSTFS_META_BUCKET {
26800                            format!("meta:{}", bucket.prefix)
26801                        } else {
26802                            format!("regular:{}", bucket.name)
26803                        };
26804                        events.lock().expect("phase event lock should not be poisoned").push(event);
26805                        if bucket.name == "regular-fails" {
26806                            Err(Error::SlowDown)
26807                        } else {
26808                            Ok(())
26809                        }
26810                    })
26811                }
26812            },
26813        )
26814        .await
26815        .expect_err("regular failure should remain fatal after metadata completes");
26816
26817        assert!(matches!(err, Error::SlowDown));
26818        assert_eq!(
26819            *events.lock().expect("phase event lock should not be poisoned"),
26820            vec![
26821                format!("meta:{}", crate::config::com::CONFIG_PREFIX),
26822                format!("meta:{}", crate::disk::BUCKET_META_PREFIX),
26823                "regular:regular-fails".to_string(),
26824            ]
26825        );
26826    }
26827
26828    #[tokio::test]
26829    async fn test_run_decommission_buckets_bounded_respects_limit() {
26830        let rx = CancellationToken::new();
26831        let running = Arc::new(AtomicUsize::new(0));
26832        let max_running = Arc::new(AtomicUsize::new(0));
26833        let started = Arc::new(AtomicUsize::new(0));
26834        let buckets = (0..8)
26835            .map(|idx| DecomBucketInfo {
26836                name: format!("bucket-{idx}"),
26837                ..Default::default()
26838            })
26839            .collect::<Vec<_>>();
26840
26841        run_decommission_buckets_bounded(rx, buckets, 2, {
26842            let running = Arc::clone(&running);
26843            let max_running = Arc::clone(&max_running);
26844            let started = Arc::clone(&started);
26845            move |_bucket, _rx| {
26846                let running = Arc::clone(&running);
26847                let max_running = Arc::clone(&max_running);
26848                let started = Arc::clone(&started);
26849                Box::pin(async move {
26850                    started.fetch_add(1, Ordering::SeqCst);
26851                    let current = running.fetch_add(1, Ordering::SeqCst) + 1;
26852                    max_running.fetch_max(current, Ordering::SeqCst);
26853                    tokio::time::sleep(StdDuration::from_millis(10)).await;
26854                    running.fetch_sub(1, Ordering::SeqCst);
26855                    Ok(())
26856                })
26857            }
26858        })
26859        .await
26860        .expect("bounded bucket scheduler should complete");
26861
26862        assert_eq!(started.load(Ordering::SeqCst), 8);
26863        assert_eq!(max_running.load(Ordering::SeqCst), 2);
26864        assert_eq!(running.load(Ordering::SeqCst), 0);
26865    }
26866
26867    #[tokio::test]
26868    async fn test_run_decommission_buckets_bounded_cancels_and_stops_launching_after_failure() {
26869        let rx = CancellationToken::new();
26870        let started = Arc::new(AtomicUsize::new(0));
26871        let observed_cancel = Arc::new(AtomicBool::new(false));
26872        let buckets = (0..5)
26873            .map(|idx| DecomBucketInfo {
26874                name: format!("bucket-{idx}"),
26875                ..Default::default()
26876            })
26877            .collect::<Vec<_>>();
26878
26879        let err = tokio::time::timeout(
26880            StdDuration::from_secs(2),
26881            run_decommission_buckets_bounded(rx.clone(), buckets, 2, {
26882                let started = Arc::clone(&started);
26883                let observed_cancel = Arc::clone(&observed_cancel);
26884                move |bucket, rx| {
26885                    let started = Arc::clone(&started);
26886                    let observed_cancel = Arc::clone(&observed_cancel);
26887                    Box::pin(async move {
26888                        started.fetch_add(1, Ordering::SeqCst);
26889                        if bucket.name == "bucket-0" {
26890                            while started.load(Ordering::SeqCst) < 2 {
26891                                tokio::task::yield_now().await;
26892                            }
26893                            return Err(Error::SlowDown);
26894                        }
26895
26896                        rx.cancelled().await;
26897                        observed_cancel.store(true, Ordering::SeqCst);
26898                        Ok(())
26899                    })
26900                }
26901            }),
26902        )
26903        .await
26904        .expect("bucket scheduler should not hang after a bucket failure")
26905        .expect_err("first bucket failure should be returned");
26906
26907        assert!(matches!(err, Error::SlowDown));
26908        assert!(rx.is_cancelled());
26909        assert!(observed_cancel.load(Ordering::SeqCst));
26910        assert_eq!(started.load(Ordering::SeqCst), 2);
26911    }
26912
26913    #[tokio::test]
26914    async fn test_run_decommission_buckets_bounded_external_cancel_stops_pending_buckets() {
26915        let rx = CancellationToken::new();
26916        let started = Arc::new(AtomicUsize::new(0));
26917        let buckets = (0..4)
26918            .map(|idx| DecomBucketInfo {
26919                name: format!("bucket-{idx}"),
26920                ..Default::default()
26921            })
26922            .collect::<Vec<_>>();
26923
26924        let err = run_decommission_buckets_bounded(rx.clone(), buckets, 1, {
26925            let started = Arc::clone(&started);
26926            move |_bucket, rx| {
26927                let started = Arc::clone(&started);
26928                Box::pin(async move {
26929                    started.fetch_add(1, Ordering::SeqCst);
26930                    rx.cancel();
26931                    Ok(())
26932                })
26933            }
26934        })
26935        .await
26936        .expect_err("external cancellation with pending buckets should stop the scheduler");
26937
26938        assert!(matches!(err, Error::OperationCanceled));
26939        assert!(rx.is_cancelled());
26940        assert_eq!(started.load(Ordering::SeqCst), 1);
26941    }
26942
26943    #[tokio::test]
26944    async fn test_wait_decommission_worker_drain_waits_for_entry_permit() {
26945        let workers = Arc::new(Semaphore::new(1));
26946        let permit = workers
26947            .clone()
26948            .acquire_owned()
26949            .await
26950            .expect("test worker permit should acquire");
26951
26952        let drain = tokio::spawn({
26953            let workers = workers.clone();
26954            async move { wait_decommission_worker_drain(&workers, 1).await }
26955        });
26956
26957        tokio::task::yield_now().await;
26958        assert!(!drain.is_finished(), "drain should wait while a worker permit is held");
26959
26960        drop(permit);
26961        let result = tokio::time::timeout(StdDuration::from_secs(1), drain)
26962            .await
26963            .expect("drain should finish after permit release")
26964            .expect("drain task should not panic");
26965        assert!(result.is_ok());
26966    }
26967
26968    #[test]
26969    fn test_decommission_entry_queue_capacity_is_bounded() {
26970        assert_eq!(decommission_entry_queue_capacity(0), 1);
26971        assert_eq!(decommission_entry_queue_capacity(1), 2);
26972        assert_eq!(
26973            decommission_entry_queue_capacity(DECOMMISSION_ENTRY_QUEUE_HARD_CAP),
26974            DECOMMISSION_ENTRY_QUEUE_HARD_CAP
26975        );
26976        assert_eq!(decommission_entry_queue_capacity(usize::MAX), DECOMMISSION_ENTRY_QUEUE_HARD_CAP);
26977    }
26978
26979    #[tokio::test]
26980    async fn test_drain_decommission_entry_queue_waits_for_all_outstanding_entries() {
26981        let outstanding = Arc::new(Semaphore::new(1));
26982        let held = outstanding
26983            .clone()
26984            .acquire_owned()
26985            .await
26986            .expect("test outstanding permit should acquire");
26987        let rx = CancellationToken::new();
26988        let drain = tokio::spawn({
26989            let outstanding = outstanding.clone();
26990            let rx = rx.clone();
26991            async move { drain_decommission_entry_queue(&rx, &outstanding, 1).await }
26992        });
26993
26994        tokio::task::yield_now().await;
26995        assert!(!drain.is_finished(), "queue drain must wait for active entry work");
26996        drop(held);
26997
26998        let drained = tokio::time::timeout(StdDuration::from_secs(1), drain)
26999            .await
27000            .expect("queue drain should finish after entry completion")
27001            .expect("queue drain task should not panic");
27002        assert!(!drained);
27003    }
27004
27005    #[tokio::test]
27006    async fn test_enqueue_decommission_entry_observes_cancellation_when_queue_is_full() {
27007        let outstanding = Arc::new(Semaphore::new(2));
27008        let (tx, mut queue) = tokio::sync::mpsc::channel(1);
27009        let held = outstanding
27010            .clone()
27011            .acquire_owned()
27012            .await
27013            .expect("first queue permit should acquire");
27014        tx.send(QueuedDecommissionEntry {
27015            entry: MetaCacheEntry::default(),
27016            queue_permit: held,
27017        })
27018        .await
27019        .expect("first entry should fill the queue");
27020
27021        let rx = CancellationToken::new();
27022        let enqueue = tokio::spawn({
27023            let rx = rx.clone();
27024            let outstanding = outstanding.clone();
27025            let tx = tx.clone();
27026            async move { enqueue_decommission_entry(&rx, &outstanding, &tx, MetaCacheEntry::default()).await }
27027        });
27028
27029        tokio::task::yield_now().await;
27030        rx.cancel();
27031        let result = tokio::time::timeout(StdDuration::from_secs(1), enqueue)
27032            .await
27033            .expect("full queue enqueue should observe cancellation")
27034            .expect("enqueue task should not panic");
27035        assert!(matches!(result, DecommissionEntryEnqueueResult::Canceled));
27036        drop(queue.recv().await);
27037    }
27038
27039    #[tokio::test]
27040    async fn test_decommission_side_effect_gate_quiesces_before_transition() {
27041        let operation_gate = Arc::new(tokio::sync::RwLock::new(()));
27042        let rx = CancellationToken::new();
27043        let started = Arc::new(tokio::sync::Notify::new());
27044        let release = Arc::new(tokio::sync::Notify::new());
27045        let operation = tokio::spawn({
27046            let operation_gate = operation_gate.clone();
27047            let rx = rx.clone();
27048            let started = started.clone();
27049            let release = release.clone();
27050            async move {
27051                run_decommission_side_effect(&rx, &operation_gate, || async {
27052                    started.notify_one();
27053                    release.notified().await;
27054                    Ok::<_, Error>(())
27055                })
27056                .await
27057            }
27058        });
27059
27060        started.notified().await;
27061        rx.cancel();
27062        let transition = tokio::spawn({
27063            let operation_gate = operation_gate.clone();
27064            async move {
27065                let _guard = operation_gate.write().await;
27066            }
27067        });
27068        tokio::task::yield_now().await;
27069        assert!(!transition.is_finished(), "transition must wait for the in-flight side effect");
27070
27071        release.notify_one();
27072        let operation_result = operation.await.expect("operation task should not panic");
27073        assert!(matches!(operation_result, Err(Error::OperationCanceled)));
27074        transition.await.expect("transition task should not panic");
27075
27076        let called = Arc::new(AtomicBool::new(false));
27077        let result = run_decommission_side_effect(&rx, &operation_gate, {
27078            let called = called.clone();
27079            move || async move {
27080                called.store(true, Ordering::SeqCst);
27081                Ok::<_, Error>(())
27082            }
27083        })
27084        .await;
27085        assert!(matches!(result, Err(Error::OperationCanceled)));
27086        assert!(!called.load(Ordering::SeqCst));
27087    }
27088
27089    #[tokio::test]
27090    async fn test_decommission_side_effect_stops_after_pool_meta_write_block() {
27091        let store = decommission_worker_test_store(PoolMeta::default(), Vec::new());
27092        store
27093            .pool_meta_save_gate
27094            .lock()
27095            .await
27096            .observe_replicas(super::PoolMetaReplicaState {
27097                needs_repair: true,
27098                repair_write_safe: false,
27099            });
27100        let called = Arc::new(AtomicBool::new(false));
27101        let operation_gate = store.ctx.data_movement_operation_gate();
27102
27103        let result = store
27104            .run_guarded_decommission_side_effect(&CancellationToken::new(), &operation_gate, {
27105                let called = called.clone();
27106                move || async move {
27107                    called.store(true, Ordering::SeqCst);
27108                    Ok::<_, Error>(())
27109                }
27110            })
27111            .await;
27112
27113        assert!(
27114            result
27115                .expect_err("sticky pool metadata state must block new movement")
27116                .to_string()
27117                .contains("restart after all replicas are readable and consistent")
27118        );
27119        assert!(!called.load(Ordering::SeqCst));
27120    }
27121
27122    #[tokio::test]
27123    async fn test_decommission_reservation_stops_before_canceler_slot_when_pool_meta_is_blocked() {
27124        let generation = OffsetDateTime::UNIX_EPOCH;
27125        let pool_meta = PoolMeta {
27126            version: super::POOL_META_VERSION,
27127            pools: vec![decommission_test_pool_status(
27128                0,
27129                Some(PoolDecommissionInfo {
27130                    start_time: Some(generation),
27131                    ..Default::default()
27132                }),
27133            )],
27134            ..Default::default()
27135        };
27136        let store = decommission_worker_test_store(pool_meta, vec![None]);
27137        store
27138            .pool_meta_save_gate
27139            .lock()
27140            .await
27141            .observe_replicas(super::PoolMetaReplicaState {
27142                needs_repair: true,
27143                repair_write_safe: false,
27144            });
27145
27146        let err = store
27147            .reserve_decommission_routines(&CancellationToken::new(), &[0])
27148            .await
27149            .err()
27150            .expect("sticky pool metadata state must block worker reservation");
27151
27152        assert!(
27153            err.to_string()
27154                .contains("restart after all replicas are readable and consistent")
27155        );
27156        assert!(store.decommission_cancelers.read().await[0].is_none());
27157    }
27158
27159    #[tokio::test]
27160    async fn test_v1_unresolved_ledger_rejection_keeps_live_state_and_write_gate_safe() {
27161        let generation = OffsetDateTime::UNIX_EPOCH;
27162        let status = decommission_test_pool_status(
27163            0,
27164            Some(PoolDecommissionInfo {
27165                start_time: Some(generation),
27166                ..Default::default()
27167            }),
27168        );
27169        let last_update = status.last_update;
27170        let store = decommission_worker_test_store(
27171            PoolMeta {
27172                version: POOL_META_V1_VERSION,
27173                pools: vec![status],
27174                ..Default::default()
27175            },
27176            vec![None],
27177        );
27178        let entry = DecommissionUnresolvedEntry {
27179            bucket: "bucket-a".to_string(),
27180            object: "directory/".to_string(),
27181            pool_index: 0,
27182            set_index: 0,
27183            source_generation: generation,
27184            candidate_count: 1,
27185            disk_error_count: 0,
27186            observed_at: generation,
27187            reason: "metadata_resolution_failed".to_string(),
27188        };
27189
27190        let err = store
27191            .persist_decommission_unresolved_entry(0, generation, entry)
27192            .await
27193            .expect_err("V1 must reject the ledger before changing live state");
27194
27195        assert!(matches!(err, Error::InvalidArgument(..)));
27196        let pool_meta = store.pool_meta.read().await;
27197        let status = &pool_meta.pools[0];
27198        assert_eq!(status.last_update, last_update);
27199        assert!(
27200            status
27201                .decommission
27202                .as_ref()
27203                .expect("active decommission metadata should remain present")
27204                .unresolved_entries
27205                .is_empty()
27206        );
27207        drop(pool_meta);
27208        store
27209            .pool_meta_save_gate
27210            .lock()
27211            .await
27212            .ensure_write_safe("V1 unresolved-entry preflight")
27213            .expect("a deterministic capability rejection must not latch recovery");
27214    }
27215
27216    #[tokio::test]
27217    async fn test_v1_runtime_recovery_rejects_worker_but_keeps_cancel_persistable() {
27218        let generation = OffsetDateTime::UNIX_EPOCH;
27219        let store = decommission_worker_test_store(
27220            PoolMeta {
27221                version: POOL_META_V1_VERSION,
27222                pools: vec![decommission_test_pool_status(
27223                    0,
27224                    Some(PoolDecommissionInfo {
27225                        start_time: Some(generation),
27226                        ..Default::default()
27227                    }),
27228                )],
27229                ..Default::default()
27230            },
27231            vec![None],
27232        );
27233
27234        let err = store
27235            .reserve_decommission_routines(&CancellationToken::new(), &[0])
27236            .await
27237            .err()
27238            .expect("V1 recovery must not install a worker that cannot persist an unresolved ledger");
27239        assert!(matches!(err, Error::InvalidArgument(..)));
27240        assert!(store.decommission_cancelers.read().await[0].is_none());
27241
27242        let save_called = Arc::new(AtomicBool::new(false));
27243        store
27244            .decommission_cancel_with_owner_and_save(0, None, {
27245                let save_called = save_called.clone();
27246                move |snapshot, _| async move {
27247                    snapshot.encode_config_data_for_v2_gate(false)?;
27248                    save_called.store(true, Ordering::SeqCst);
27249                    Ok(())
27250                }
27251            })
27252            .await
27253            .expect("a rejected V1 recovery must remain cancelable without restart");
27254
27255        assert!(save_called.load(Ordering::SeqCst));
27256        let pool_meta = store.pool_meta.read().await;
27257        let info = pool_meta.pools[0]
27258            .decommission
27259            .as_ref()
27260            .expect("cancel metadata should remain present");
27261        assert!(info.canceled);
27262        assert!(!info.failed);
27263        assert!(!info.complete);
27264    }
27265
27266    #[tokio::test]
27267    async fn test_decommission_transition_waits_without_registered_canceler() {
27268        let store = decommission_worker_test_store(PoolMeta::default(), vec![None]);
27269        let operation_gate = store.ctx.data_movement_operation_gate();
27270        let operation_guard = operation_gate.read().await;
27271        let transition = tokio::spawn({
27272            let store = store.clone();
27273            async move { store.cancel_decommission_routines_and_wait(&[0]).await }
27274        });
27275
27276        tokio::task::yield_now().await;
27277        assert!(
27278            !transition.is_finished(),
27279            "a transition must wait for an in-flight side effect even after its canceler slot is gone"
27280        );
27281
27282        drop(operation_guard);
27283        tokio::time::timeout(StdDuration::from_secs(1), transition)
27284            .await
27285            .expect("transition should finish after the side effect")
27286            .expect("transition task should not panic");
27287    }
27288
27289    #[tokio::test]
27290    async fn test_join_error_quiesces_side_effects_before_failure_transition() {
27291        let canceler = DecommissionCanceler::new(CancellationToken::new());
27292        let store = decommission_worker_test_store(PoolMeta::default(), vec![Some(canceler.clone())]);
27293        let operation_gate = store.ctx.data_movement_operation_gate();
27294        let operation_guard = operation_gate.read().await;
27295        let transition = tokio::spawn({
27296            let store = store.clone();
27297            let canceler = canceler.clone();
27298            async move { store.quiesce_decommission_worker_after_join_error(&canceler).await }
27299        });
27300
27301        tokio::time::timeout(StdDuration::from_secs(1), canceler.token().cancelled())
27302            .await
27303            .expect("join error handling should cancel the detached worker");
27304        assert!(
27305            !transition.is_finished(),
27306            "failure transition must wait for the detached worker's in-flight side effect"
27307        );
27308
27309        drop(operation_guard);
27310        tokio::time::timeout(StdDuration::from_secs(1), transition)
27311            .await
27312            .expect("failure transition should finish after the side effect")
27313            .expect("failure transition task should not panic");
27314    }
27315
27316    #[tokio::test(start_paused = true)]
27317    async fn test_run_decommission_listing_with_retry_drains_before_each_retry() {
27318        let attempts = Arc::new(AtomicUsize::new(0));
27319        let drains = Arc::new(AtomicUsize::new(0));
27320        let err = run_decommission_listing_with_retry_and_drain(
27321            CancellationToken::new(),
27322            "bucket-a".to_string(),
27323            noop_decommission_list_callback(),
27324            1,
27325            2,
27326            2,
27327            {
27328                let attempts = attempts.clone();
27329                move |_| {
27330                    let attempts = attempts.clone();
27331                    async move {
27332                        attempts.fetch_add(1, Ordering::SeqCst);
27333                        Err(Error::SlowDown)
27334                    }
27335                }
27336            },
27337            {
27338                let drains = drains.clone();
27339                move || {
27340                    let drains = drains.clone();
27341                    async move {
27342                        drains.fetch_add(1, Ordering::SeqCst);
27343                        false
27344                    }
27345                }
27346            },
27347        )
27348        .await
27349        .expect_err("permanent listing failure must be returned");
27350
27351        assert!(err.to_string().contains("attempt 2/2"));
27352        assert_eq!(attempts.load(Ordering::SeqCst), 2);
27353        assert_eq!(drains.load(Ordering::SeqCst), 2);
27354    }
27355
27356    #[test]
27357    fn test_get_by_index_returns_value_when_in_range() {
27358        let values = vec!["a", "b", "c"];
27359        let value = get_by_index(values.as_slice(), 1, "fetch decommission status").expect("in-range index should return value");
27360        assert_eq!(*value, "b");
27361    }
27362
27363    #[test]
27364    fn test_get_by_index_returns_error_when_out_of_range() {
27365        let values = vec![1_u8];
27366        let err =
27367            get_by_index(values.as_slice(), 2, "load decommission background pool").expect_err("out-of-range index should fail");
27368        assert!(
27369            err.to_string()
27370                .contains("failed to load decommission background pool: invalid decommission pool index 2 for 1 pools")
27371        );
27372    }
27373
27374    #[test]
27375    fn test_pool_meta_is_suspended_returns_false_for_out_of_range() {
27376        let meta = PoolMeta::default();
27377        assert!(!meta.is_suspended(1));
27378    }
27379
27380    #[test]
27381    fn test_rollback_start_decommission_pool_meta_clears_active_state() {
27382        let previous = PoolMeta {
27383            pools: vec![PoolStatus {
27384                id: 0,
27385                cmd_line: "pool-0".to_string(),
27386                last_update: OffsetDateTime::UNIX_EPOCH,
27387                decommission: None,
27388            }],
27389            ..Default::default()
27390        };
27391        let mut active = previous.clone();
27392        let active_update = OffsetDateTime::UNIX_EPOCH + Duration::seconds(1);
27393        active.pools[0].last_update = active_update;
27394        active.pools[0].decommission = Some(PoolDecommissionInfo {
27395            start_time: Some(OffsetDateTime::UNIX_EPOCH),
27396            ..Default::default()
27397        });
27398        let mut peer = active.clone();
27399
27400        assert!(active.is_suspended(0));
27401        assert_eq!(
27402            decommission_start_pool_state(active.pools.first()),
27403            DecommissionStartPoolState::Decommissioning
27404        );
27405
27406        rollback_start_decommission_pool_meta(&mut active, &previous, &[0]);
27407
27408        assert!(!active.is_suspended(0));
27409        assert_eq!(decommission_start_pool_state(active.pools.first()), DecommissionStartPoolState::Active);
27410        assert!(active.pools[0].last_update > active_update);
27411        assert!(merge_pool_status_refresh(&mut peer, active, &[false]));
27412        assert!(peer.pools[0].decommission.is_none());
27413    }
27414
27415    #[test]
27416    fn test_pool_meta_queue_buckets_ignores_out_of_range_index() {
27417        let mut meta = PoolMeta {
27418            pools: vec![PoolStatus {
27419                id: 0,
27420                cmd_line: "pool-0".to_string(),
27421                last_update: OffsetDateTime::UNIX_EPOCH,
27422                decommission: Some(PoolDecommissionInfo::default()),
27423            }],
27424            ..Default::default()
27425        };
27426
27427        meta.queue_buckets(
27428            9,
27429            vec![DecomBucketInfo {
27430                name: "bucket-a".to_string(),
27431                prefix: String::new(),
27432            }],
27433        );
27434
27435        let queued = meta.pools[0]
27436            .decommission
27437            .as_ref()
27438            .expect("pool should have decommission info")
27439            .queued_buckets
27440            .clone();
27441        assert!(queued.is_empty());
27442    }
27443
27444    #[test]
27445    fn test_pool_meta_is_bucket_decommissioned_returns_false_for_out_of_range() {
27446        let meta = PoolMeta::default();
27447        assert!(!meta.is_bucket_decommissioned(7, "bucket-a".to_string()));
27448    }
27449
27450    #[test]
27451    fn test_resolve_decommission_bucket_state_rejects_out_of_range_index() {
27452        let meta = PoolMeta {
27453            pools: vec![PoolStatus {
27454                id: 0,
27455                cmd_line: "pool-0".to_string(),
27456                last_update: OffsetDateTime::UNIX_EPOCH,
27457                decommission: Some(PoolDecommissionInfo::default()),
27458            }],
27459            ..Default::default()
27460        };
27461
27462        let bucket = DecomBucketInfo {
27463            name: "bucket-a".to_string(),
27464            prefix: String::new(),
27465        };
27466        let err =
27467            resolve_decommission_bucket_state(&meta, 3, &bucket).expect_err("out-of-range index should return invalid argument");
27468        assert!(err.to_string().contains("invalid decommission pool index 3 for 1 pools"));
27469    }
27470
27471    #[test]
27472    fn test_resolve_decommission_bucket_state_rejects_missing_decommission_meta() {
27473        let meta = PoolMeta {
27474            pools: vec![PoolStatus {
27475                id: 0,
27476                cmd_line: "pool-0".to_string(),
27477                last_update: OffsetDateTime::UNIX_EPOCH,
27478                decommission: None,
27479            }],
27480            ..Default::default()
27481        };
27482
27483        let bucket = DecomBucketInfo {
27484            name: "bucket-a".to_string(),
27485            prefix: String::new(),
27486        };
27487        let err = resolve_decommission_bucket_state(&meta, 0, &bucket)
27488            .expect_err("missing decommission metadata should return explicit error");
27489        assert!(
27490            err.to_string()
27491                .contains("failed to resolve decommission bucket state: decommission metadata not initialized")
27492        );
27493    }
27494
27495    #[test]
27496    fn test_resolve_decommission_bucket_state_returns_true_for_done_bucket() {
27497        let meta = PoolMeta {
27498            pools: vec![PoolStatus {
27499                id: 0,
27500                cmd_line: "pool-0".to_string(),
27501                last_update: OffsetDateTime::UNIX_EPOCH,
27502                decommission: Some(PoolDecommissionInfo {
27503                    decommissioned_buckets: vec!["bucket-a".to_string()],
27504                    ..Default::default()
27505                }),
27506            }],
27507            ..Default::default()
27508        };
27509
27510        let bucket = DecomBucketInfo {
27511            name: "bucket-a".to_string(),
27512            prefix: String::new(),
27513        };
27514        let done = resolve_decommission_bucket_state(&meta, 0, &bucket).expect("valid state should resolve");
27515        assert!(done);
27516    }
27517
27518    #[test]
27519    fn test_decommission_capacity_publication_preserves_progress_and_checkpoint_deadline() {
27520        let mut current = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_MODEL_VERSION]);
27521        let start = OffsetDateTime::UNIX_EPOCH;
27522        current.pools[0].last_update = start;
27523        current.count_item(0, 17, false);
27524        let mut saved = current.clone();
27525        saved.pools[0]
27526            .decommission
27527            .as_mut()
27528            .expect("persisted source")
27529            .items_decommissioned = 0;
27530        saved.pools[0].decommission.as_mut().expect("persisted source").bytes_done = 0;
27531        for seconds in [10, 20, 29] {
27532            saved.pools[0].last_update = start + Duration::seconds(seconds);
27533            super::publish_decommission_capacity_update(&mut current, &saved, 0).expect("same-owner capacity publication");
27534        }
27535        let info = current.pools[0].decommission.as_ref().expect("live source");
27536        assert_eq!((info.items_decommissioned, info.bytes_done, info.progress_save_item_baseline), (1, 17, 0));
27537        assert_eq!(info.progress_save_last_at, Some(start));
27538        let checkpoint = current
27539            .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, start + Duration::seconds(30), None)
27540            .expect("checkpoint should remain due despite capacity writes")
27541            .expect("capacity updates must not postpone the 30-second progress checkpoint");
27542        current.count_item(0, 19, false);
27543        assert!(current.commit_decommission_progress_checkpoint(0, checkpoint));
27544        let info = current.pools[0].decommission.as_ref().expect("live source");
27545        assert_eq!(
27546            info.items_since_last_progress_save(),
27547            1,
27548            "only the committed snapshot advances the watermark"
27549        );
27550        assert_eq!(info.progress_save_last_at, Some(checkpoint.checkpoint_at));
27551        assert!(
27552            current
27553                .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, start + Duration::seconds(31), None)
27554                .expect("next checkpoint timing")
27555                .is_none()
27556        );
27557    }
27558
27559    #[test]
27560    fn test_decommission_capacity_publication_rejects_changed_identity_without_mutation() {
27561        let baseline = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_MODEL_VERSION]);
27562        for changed_field in [
27563            "layout",
27564            "start",
27565            "queued",
27566            "terminal",
27567            "operation",
27568            "generation",
27569            "owner",
27570            "model",
27571            "cleared",
27572        ] {
27573            let mut current = baseline.clone();
27574            let mut saved = baseline.clone();
27575            saved.pools[0].last_update += Duration::seconds(1);
27576            let pool = &mut current.pools[0];
27577            let info = pool.decommission.as_mut().expect("current source");
27578            match changed_field {
27579                "layout" => pool.cmd_line.push_str("-replaced"),
27580                "start" => info.start_time = Some(OffsetDateTime::UNIX_EPOCH + Duration::seconds(1)),
27581                "queued" => info.queued = true,
27582                "terminal" => info.canceled = true,
27583                "operation" => info.capacity_reservation.as_mut().expect("owner").operation_id = uuid::Uuid::new_v4(),
27584                "generation" => info.capacity_reservation.as_mut().expect("owner").generation += 1,
27585                "owner" => info.capacity_reservation.as_mut().expect("owner").owner_nonce = uuid::Uuid::new_v4(),
27586                "model" => info.capacity_reservation.as_mut().expect("owner").model_version += 1,
27587                "cleared" => pool.decommission = None,
27588                _ => unreachable!("fixed test matrix"),
27589            }
27590            let before = format!("{current:?}");
27591            let err = super::publish_decommission_capacity_update(&mut current, &saved, 0)
27592                .expect_err("changed operation identity must reject publication");
27593            assert!(matches!(err, Error::StalePoolMetadataUpdate { .. }), "{changed_field}: {err}");
27594            assert_eq!(format!("{current:?}"), before, "{changed_field}: rejected publication mutated live state");
27595        }
27596    }
27597
27598    #[test]
27599    fn test_decommission_capacity_revision_does_not_regress_when_clock_moves_backwards() {
27600        let mut meta = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_MODEL_VERSION, Vec::new(), 0, None);
27601        let revision = meta.pools[0].last_update;
27602        let mutation = uuid::Uuid::new_v4();
27603        reserve_decommission_target_pending(&mut meta, 0, 1, 1, mutation, OffsetDateTime::UNIX_EPOCH)
27604            .expect("clock rollback must not prevent a valid owned reservation");
27605        assert_eq!(meta.pools[0].last_update, revision);
27606        resolve_decommission_target_pending(&mut meta, 0, 1, 1, mutation).expect("resolve the exact intent");
27607        record_decommission_target_consumption(
27608            &mut meta,
27609            0,
27610            1,
27611            DecommissionTargetConsumption {
27612                committed_data_bytes: 1,
27613                target_physical_bytes: 1,
27614                observed_physical_bytes: 1,
27615            },
27616            mutation,
27617            OffsetDateTime::UNIX_EPOCH,
27618        )
27619        .expect("commit consumption during clock rollback");
27620        assert_eq!(meta.pools[0].last_update, revision);
27621    }
27622
27623    #[test]
27624    fn test_mark_decommission_bucket_done_is_idempotent_and_ignores_unknown_buckets() {
27625        let bucket = DecomBucketInfo {
27626            name: "bucket-a".to_string(),
27627            prefix: String::new(),
27628        };
27629        let unknown = DecomBucketInfo {
27630            name: "bucket-b".to_string(),
27631            prefix: String::new(),
27632        };
27633        let mut meta = PoolMeta {
27634            pools: vec![decommission_test_pool_status(
27635                0,
27636                Some(PoolDecommissionInfo {
27637                    queued_buckets: vec![bucket.to_string()],
27638                    ..Default::default()
27639                }),
27640            )],
27641            ..Default::default()
27642        };
27643        assert!(!mark_decommission_bucket_done(&mut meta, 0, &unknown).expect("an unknown bucket is not completed"));
27644        assert!(
27645            meta.pools[0]
27646                .decommission
27647                .as_ref()
27648                .expect("source progress")
27649                .decommissioned_buckets
27650                .is_empty()
27651        );
27652        assert!(mark_decommission_bucket_done(&mut meta, 0, &bucket).expect("complete the queued bucket once"));
27653        assert!(!mark_decommission_bucket_done(&mut meta, 0, &bucket).expect("repeat completion is a no-op"));
27654        let info = meta.pools[0].decommission.as_ref().expect("source progress");
27655        assert!(info.queued_buckets.is_empty());
27656        assert_eq!(info.decommissioned_buckets, vec![bucket.to_string()]);
27657    }
27658
27659    #[test]
27660    fn test_mark_decommission_bucket_done_rejects_missing_decommission_meta() {
27661        let mut meta = PoolMeta {
27662            pools: vec![PoolStatus {
27663                id: 0,
27664                cmd_line: "pool-0".to_string(),
27665                last_update: OffsetDateTime::UNIX_EPOCH,
27666                decommission: None,
27667            }],
27668            ..Default::default()
27669        };
27670
27671        let bucket = DecomBucketInfo {
27672            name: "bucket-a".to_string(),
27673            prefix: String::new(),
27674        };
27675        let err = mark_decommission_bucket_done(&mut meta, 0, &bucket)
27676            .expect_err("missing decommission metadata should return explicit error");
27677        assert!(
27678            err.to_string()
27679                .contains("failed to mark decommission bucket done: decommission metadata not initialized")
27680        );
27681    }
27682
27683    #[test]
27684    fn test_mark_decommission_bucket_done_rejects_out_of_range_index() {
27685        let mut meta = PoolMeta {
27686            pools: vec![PoolStatus {
27687                id: 0,
27688                cmd_line: "pool-0".to_string(),
27689                last_update: OffsetDateTime::UNIX_EPOCH,
27690                decommission: Some(PoolDecommissionInfo::default()),
27691            }],
27692            ..Default::default()
27693        };
27694
27695        let bucket = DecomBucketInfo {
27696            name: "bucket-a".to_string(),
27697            prefix: String::new(),
27698        };
27699        let err =
27700            mark_decommission_bucket_done(&mut meta, 1, &bucket).expect_err("out-of-range index should return invalid argument");
27701        assert!(err.to_string().contains("invalid decommission pool index 1 for 1 pools"));
27702    }
27703
27704    #[test]
27705    fn test_mark_decommission_bucket_done_pops_bucket_when_present() {
27706        let mut meta = PoolMeta {
27707            pools: vec![PoolStatus {
27708                id: 0,
27709                cmd_line: "pool-0".to_string(),
27710                last_update: OffsetDateTime::UNIX_EPOCH,
27711                decommission: Some(PoolDecommissionInfo {
27712                    queued_buckets: vec!["bucket-a".to_string()],
27713                    ..Default::default()
27714                }),
27715            }],
27716            ..Default::default()
27717        };
27718
27719        let bucket = DecomBucketInfo {
27720            name: "bucket-a".to_string(),
27721            prefix: String::new(),
27722        };
27723        let popped = mark_decommission_bucket_done(&mut meta, 0, &bucket).expect("valid state should mark bucket done");
27724        assert!(popped);
27725    }
27726
27727    #[test]
27728    fn test_count_decommission_item_rejects_missing_decommission_meta() {
27729        let mut meta = PoolMeta {
27730            pools: vec![PoolStatus {
27731                id: 0,
27732                cmd_line: "pool-0".to_string(),
27733                last_update: OffsetDateTime::UNIX_EPOCH,
27734                decommission: None,
27735            }],
27736            ..Default::default()
27737        };
27738
27739        let err = count_decommission_item(&mut meta, 0, 64, true)
27740            .expect_err("missing decommission metadata should return explicit error");
27741        assert!(
27742            err.to_string()
27743                .contains("failed to count decommission item: decommission metadata not initialized")
27744        );
27745    }
27746
27747    #[test]
27748    fn test_count_decommission_item_updates_done_and_failed_counters() {
27749        let mut meta = PoolMeta {
27750            pools: vec![PoolStatus {
27751                id: 0,
27752                cmd_line: "pool-0".to_string(),
27753                last_update: OffsetDateTime::UNIX_EPOCH,
27754                decommission: Some(PoolDecommissionInfo::default()),
27755            }],
27756            ..Default::default()
27757        };
27758
27759        count_decommission_item(&mut meta, 0, 32, false).expect("success counter should be updated");
27760        count_decommission_item(&mut meta, 0, 16, true).expect("failed counter should be updated");
27761
27762        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
27763        assert_eq!(info.items_decommissioned, 1);
27764        assert_eq!(info.bytes_done, 32);
27765        assert_eq!(info.items_decommission_failed, 1);
27766        assert_eq!(info.bytes_failed, 16);
27767    }
27768
27769    #[test]
27770    fn test_track_decommission_current_object_rejects_missing_decommission_meta() {
27771        let mut meta = PoolMeta {
27772            pools: vec![PoolStatus {
27773                id: 0,
27774                cmd_line: "pool-0".to_string(),
27775                last_update: OffsetDateTime::UNIX_EPOCH,
27776                decommission: None,
27777            }],
27778            ..Default::default()
27779        };
27780
27781        let err = track_decommission_current_object(&mut meta, 0, "bucket-a", "object-a")
27782            .expect_err("missing decommission metadata should return explicit error");
27783        assert!(
27784            err.to_string()
27785                .contains("failed to track decommission current object: decommission metadata not initialized")
27786        );
27787    }
27788
27789    #[test]
27790    fn test_track_decommission_current_object_updates_bucket_and_object() {
27791        let mut meta = PoolMeta {
27792            pools: vec![PoolStatus {
27793                id: 0,
27794                cmd_line: "pool-0".to_string(),
27795                last_update: OffsetDateTime::UNIX_EPOCH,
27796                decommission: Some(PoolDecommissionInfo::default()),
27797            }],
27798            ..Default::default()
27799        };
27800
27801        track_decommission_current_object(&mut meta, 0, "bucket-a", "object-a").expect("valid state should track bucket/object");
27802
27803        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
27804        assert_eq!(info.bucket, "bucket-a");
27805        assert_eq!(info.object, "object-a");
27806        assert!(info.stage.is_empty());
27807    }
27808
27809    #[test]
27810    fn test_track_decommission_current_object_stage_updates_stage() {
27811        let mut meta = PoolMeta {
27812            pools: vec![PoolStatus {
27813                id: 0,
27814                cmd_line: "pool-0".to_string(),
27815                last_update: OffsetDateTime::UNIX_EPOCH,
27816                decommission: Some(PoolDecommissionInfo::default()),
27817            }],
27818            ..Default::default()
27819        };
27820
27821        track_decommission_current_object_stage(&mut meta, 0, "bucket-a", "object-a", "cleanup_preflight")
27822            .expect("valid state should track bucket/object stage");
27823
27824        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
27825        assert_eq!(info.bucket, "bucket-a");
27826        assert_eq!(info.object, "object-a");
27827        assert_eq!(info.stage, "cleanup_preflight");
27828    }
27829
27830    #[test]
27831    fn test_resolve_decommission_update_after_result_passthrough_ok() {
27832        let ok = resolve_decommission_update_after_result(Ok(true)).expect("ok value should pass through");
27833        assert!(ok);
27834    }
27835
27836    #[test]
27837    fn test_resolve_decommission_update_after_result_wraps_error_context() {
27838        let err = resolve_decommission_update_after_result(ensure_valid_decommission_pool_index(0, 0).map(|_| false))
27839            .expect_err("invalid argument should be wrapped with context");
27840        assert!(err.to_string().contains("decommission metadata update failed"));
27841        assert!(err.to_string().contains("invalid decommission pool index 0 for 0 pools"));
27842    }
27843
27844    #[test]
27845    fn test_resolve_decommission_progress_save_result_returns_none_on_success() {
27846        assert!(resolve_decommission_progress_save_result(Ok(())).is_none());
27847    }
27848
27849    #[test]
27850    fn test_resolve_decommission_progress_save_result_returns_error_for_best_effort_failure() {
27851        let err = resolve_decommission_progress_save_result(Err(Error::SlowDown))
27852            .expect("progress save failure should be returned for logging");
27853
27854        assert!(err.to_string().contains("decommission progress save failed"));
27855        assert!(err.to_string().contains(Error::SlowDown.to_string().as_str()));
27856    }
27857
27858    #[test]
27859    fn test_resolve_decommission_preflight_heal_result_passthrough_ok() {
27860        assert!(resolve_decommission_preflight_heal_result::<()>("bucket-a", Ok(())).is_ok());
27861    }
27862
27863    #[test]
27864    fn test_resolve_decommission_preflight_heal_result_wraps_error_context() {
27865        let err = resolve_decommission_preflight_heal_result::<()>("bucket-a", Err(Error::SlowDown))
27866            .expect_err("heal failure should carry preflight context");
27867        assert!(
27868            err.to_string()
27869                .contains("decommission preflight heal failed for bucket bucket-a")
27870        );
27871    }
27872
27873    #[test]
27874    fn test_resolve_decommission_optional_bucket_config_result_passthrough() {
27875        let result = resolve_decommission_optional_bucket_config_result("bucket-a", "replication", Ok(42_u8))
27876            .expect("bucket config should pass through");
27877        assert_eq!(result, Some(42));
27878    }
27879
27880    #[test]
27881    fn test_resolve_decommission_optional_bucket_config_result_returns_none_for_missing_config() {
27882        let result =
27883            resolve_decommission_optional_bucket_config_result::<()>("bucket-a", "versioning", Err(Error::ConfigNotFound))
27884                .expect("missing bucket config should map to None");
27885        assert!(result.is_none());
27886    }
27887
27888    #[test]
27889    fn test_resolve_decommission_optional_bucket_config_result_wraps_other_errors() {
27890        let err = resolve_decommission_optional_bucket_config_result::<()>("bucket-a", "replication", Err(Error::SlowDown))
27891            .expect_err("unexpected bucket config errors should be wrapped with context");
27892        assert!(
27893            err.to_string()
27894                .contains("decommission replication config load failed for bucket bucket-a")
27895        );
27896    }
27897
27898    #[test]
27899    fn test_resolve_decommission_entry_cleanup_delete_result_passthrough_ok() {
27900        assert!(resolve_decommission_entry_cleanup_delete_result(Ok(()), "bucket-a", "obj.txt").is_ok());
27901    }
27902
27903    #[test]
27904    fn test_resolve_decommission_entry_cleanup_delete_result_ignores_not_found() {
27905        assert!(resolve_decommission_entry_cleanup_delete_result::<()>(Err(Error::FileNotFound), "bucket-a", "obj.txt").is_ok());
27906    }
27907
27908    #[test]
27909    fn test_resolve_decommission_entry_cleanup_delete_result_wraps_error_context() {
27910        let err = resolve_decommission_entry_cleanup_delete_result::<()>(Err(Error::SlowDown), "bucket-a", "obj.txt")
27911            .expect_err("cleanup delete failure should be wrapped with explicit context");
27912        assert!(
27913            err.to_string()
27914                .contains("decommission cleanup_delete_object failed for bucket-a/obj.txt")
27915        );
27916    }
27917
27918    #[test]
27919    fn test_resolve_decommission_entry_reload_result_passthrough_ok() {
27920        assert!(resolve_decommission_entry_reload_result(Ok(()), "bucket-a", "obj.txt").is_ok());
27921    }
27922
27923    #[test]
27924    fn test_resolve_decommission_entry_reload_result_wraps_error_context() {
27925        let err = resolve_decommission_entry_reload_result(Err(Error::SlowDown), "bucket-a", "obj.txt")
27926            .expect_err("reload failure should be wrapped with explicit context");
27927        assert!(
27928            err.to_string()
27929                .contains("decommission reload_pool_meta failed for bucket-a/obj.txt")
27930        );
27931    }
27932
27933    #[test]
27934    fn test_resolve_decommission_terminal_mark_result_passthrough_ok() {
27935        assert!(resolve_decommission_terminal_mark_result(Ok(()), "completed", "pool-a").is_ok());
27936    }
27937
27938    #[test]
27939    fn test_resolve_decommission_terminal_mark_result_wraps_error_context() {
27940        let err = resolve_decommission_terminal_mark_result(Err(Error::SlowDown), "failed", "pool-a")
27941            .expect_err("terminal mark failure should include stage and pool context");
27942        let message = err.to_string();
27943        assert!(message.contains("decommission terminal mark failed failed for pool pool-a"));
27944    }
27945
27946    #[test]
27947    fn test_resolve_decommission_terminal_mark_after_error_result_passthrough_ok() {
27948        assert!(resolve_decommission_terminal_mark_after_error_result(Ok(()), 3, &Error::SlowDown).is_ok());
27949    }
27950
27951    #[test]
27952    fn test_resolve_decommission_terminal_mark_after_error_result_wraps_error_context() {
27953        let err = resolve_decommission_terminal_mark_after_error_result(Err(Error::OperationCanceled), 3, &Error::SlowDown)
27954            .expect_err("terminal mark after-error failure should include both errors");
27955        let message = err.to_string();
27956        assert!(message.contains("decommission terminal mark failed after background error on pool 3"));
27957        assert!(message.contains("mark error"));
27958    }
27959
27960    #[test]
27961    fn test_observe_decommission_terminal_reload_result_returns_none_on_success() {
27962        assert!(observe_decommission_terminal_reload_result(Ok(()), "complete_decommission for pool 3").is_none());
27963    }
27964
27965    #[test]
27966    fn test_observe_decommission_terminal_reload_result_keeps_failure_for_logging() {
27967        let err = observe_decommission_terminal_reload_result(Err(Error::SlowDown), "decommission_failed for pool 3")
27968            .expect("reload failure should be observable");
27969        let message = err.to_string();
27970        assert!(message.contains("decommission terminal pool meta reload failed during decommission_failed for pool 3"));
27971        assert!(message.contains(Error::SlowDown.to_string().as_str()));
27972    }
27973
27974    #[test]
27975    fn test_decommission_item_size_converts_positive_values() {
27976        assert_eq!(decommission_item_size(42_i64), 42);
27977    }
27978
27979    #[test]
27980    fn test_decommission_item_size_clamps_negative_values_to_zero() {
27981        assert_eq!(decommission_item_size(-1_i64), 0);
27982    }
27983
27984    #[test]
27985    fn test_new_multipart_abort_flag_defaults_to_abort_enabled() {
27986        let flag = data_movement::new_multipart_abort_flag();
27987        assert!(data_movement::should_abort_multipart_upload(&flag));
27988    }
27989
27990    #[test]
27991    fn test_mark_multipart_upload_completed_disables_abort_cleanup() {
27992        let flag = data_movement::new_multipart_abort_flag();
27993        data_movement::mark_multipart_upload_completed(&flag);
27994        assert!(!data_movement::should_abort_multipart_upload(&flag));
27995    }
27996
27997    #[test]
27998    fn test_decode_part_index_returns_some_for_valid_payload() {
27999        let mut index = Index::new();
28000        index.add(0, 0).expect("first index entry should be accepted");
28001        index
28002            .add(2_097_152, 2_097_152)
28003            .expect("second index entry should advance totals");
28004
28005        let encoded = index.into_vec();
28006        let decoded = data_movement::decode_part_index(Some(&encoded)).expect("valid index payload should decode");
28007
28008        assert_eq!(decoded.total_uncompressed, 2_097_152);
28009        assert_eq!(decoded.total_compressed, 2_097_152);
28010    }
28011
28012    #[test]
28013    fn test_with_decommission_entry_context_formats_stage_bucket_and_object() {
28014        let err = with_decommission_entry_context("update_after", "bucket-a", "obj.txt", Error::SlowDown);
28015        let message = err.to_string();
28016        assert!(message.contains("decommission entry update_after failed"));
28017        assert!(message.contains("bucket bucket-a"));
28018        assert!(message.contains("object obj.txt"));
28019    }
28020
28021    #[test]
28022    fn test_load_decommission_entry_versions_wraps_parse_errors_with_context() {
28023        let entry = MetaCacheEntry {
28024            name: "obj.txt".to_string(),
28025            metadata: vec![1, 2, 3],
28026            cached: None,
28027            reusable: false,
28028        };
28029
28030        let err = load_decommission_entry_versions(&entry, "bucket-a", "check_after_decommission.file_info_versions")
28031            .expect_err("invalid metadata should fail");
28032        let message = err.to_string();
28033        assert!(message.contains("decommission entry check_after_decommission.file_info_versions failed"));
28034        assert!(message.contains("bucket bucket-a"));
28035        assert!(message.contains("object obj.txt"));
28036    }
28037
28038    #[test]
28039    fn test_resolve_decommission_entry_exact_versions_preserves_full_parts() {
28040        let entry = MetaCacheEntry {
28041            name: "obj.txt".to_string(),
28042            metadata: Vec::new(),
28043            cached: None,
28044            reusable: false,
28045        };
28046        let fivs = FileInfoVersions {
28047            volume: "bucket-a".to_string(),
28048            name: "obj.txt".to_string(),
28049            versions: vec![FileInfo {
28050                name: "obj.txt".to_string(),
28051                parts: vec![ObjectPartInfo {
28052                    number: 1,
28053                    etag: "part-etag".to_string(),
28054                    size: 128,
28055                    actual_size: 128,
28056                    ..Default::default()
28057                }],
28058                ..Default::default()
28059            }],
28060            ..Default::default()
28061        };
28062
28063        let resolved = resolve_decommission_entry_exact_versions(Ok(Some(fivs)), &entry, "bucket-a", "file_info_versions")
28064            .expect("exact versions should be preserved");
28065
28066        assert_eq!(resolved.versions[0].parts.len(), 1);
28067        assert_eq!(resolved.versions[0].parts[0].etag, "part-etag");
28068    }
28069
28070    #[test]
28071    fn test_resolve_decommission_entry_exact_versions_uses_empty_when_source_missing() {
28072        let entry = MetaCacheEntry {
28073            name: "obj.txt".to_string(),
28074            metadata: Vec::new(),
28075            cached: None,
28076            reusable: false,
28077        };
28078
28079        let resolved = resolve_decommission_entry_exact_versions(Ok(None), &entry, "bucket-a", "file_info_versions")
28080            .expect("missing source metadata should be treated as empty");
28081
28082        assert_eq!(resolved.volume, "bucket-a");
28083        assert_eq!(resolved.name, "obj.txt");
28084        assert!(resolved.versions.is_empty());
28085    }
28086
28087    #[test]
28088    fn test_resolve_decommission_check_after_list_result_prefers_entry_error() {
28089        let err = resolve_decommission_check_after_list_result(Err(Error::OperationCanceled), Some(Error::SlowDown))
28090            .expect_err("entry error should win over cancellation");
28091        assert!(matches!(err, Error::SlowDown));
28092    }
28093
28094    #[test]
28095    fn test_resolve_decommission_partial_listing_entry_rejects_unresolved_metadata() {
28096        let generation = OffsetDateTime::now_utc();
28097        let unresolved_entry = resolve_decommission_partial_listing_entry(
28098            MetaCacheEntries(vec![None]),
28099            MetadataResolutionParams {
28100                dir_quorum: 2,
28101                obj_quorum: 2,
28102                bucket: "bucket-a".to_string(),
28103                ..Default::default()
28104            },
28105            "bucket-a",
28106            "prefix/",
28107            1,
28108            2,
28109            3,
28110            generation,
28111        )
28112        .expect_err("unresolved partial listing must fail closed");
28113
28114        assert_eq!(unresolved_entry.bucket, "bucket-a");
28115        assert_eq!(unresolved_entry.object, "prefix/");
28116        assert_eq!(unresolved_entry.pool_index, 2);
28117        assert_eq!(unresolved_entry.set_index, 3);
28118        assert_eq!(unresolved_entry.source_generation, generation);
28119        assert_eq!(unresolved_entry.candidate_count, 0);
28120        assert_eq!(unresolved_entry.disk_error_count, 1);
28121        assert_eq!(unresolved_entry.reason, "metadata_resolution_failed");
28122
28123        let message = decommission_unresolved_listing_error(&unresolved_entry).to_string();
28124        assert!(message.contains("decommission listing could not resolve metadata"));
28125        assert!(message.contains("bucket-a/prefix/"));
28126        assert!(message.contains("pool 2 set 3"));
28127        assert!(message.contains("1 disk error(s)"));
28128    }
28129
28130    #[test]
28131    fn unresolved_entry_ledger_requires_individual_verification_before_completion() {
28132        let generation = OffsetDateTime::now_utc();
28133        let mut pool_meta = PoolMeta {
28134            version: POOL_META_VERSION,
28135            pools: vec![PoolStatus {
28136                id: 0,
28137                cmd_line: "/data/pool".to_string(),
28138                last_update: generation,
28139                decommission: Some(PoolDecommissionInfo {
28140                    start_time: Some(generation),
28141                    ..Default::default()
28142                }),
28143            }],
28144            dont_save: true,
28145        };
28146        let unresolved_entry = DecommissionUnresolvedEntry {
28147            bucket: "bucket-a".to_string(),
28148            object: "object-a".to_string(),
28149            pool_index: 0,
28150            set_index: 0,
28151            source_generation: generation,
28152            candidate_count: 1,
28153            disk_error_count: 0,
28154            observed_at: generation,
28155            reason: "metadata_resolution_failed".to_string(),
28156        };
28157
28158        assert!(
28159            record_decommission_unresolved_entry(
28160                &mut pool_meta,
28161                0,
28162                generation,
28163                unresolved_entry.clone(),
28164                generation + Duration::nanoseconds(1),
28165                None,
28166            )
28167            .expect("active generation should accept unresolved entry")
28168        );
28169        let err = reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, None, None)
28170            .expect_err("unverified completion must retain unresolved entries");
28171        assert!(err.to_string().contains("1 unresolved listing entries remain"));
28172        assert_eq!(
28173            pool_meta.pools[0]
28174                .decommission
28175                .as_ref()
28176                .expect("decommission should exist")
28177                .unresolved_entries
28178                .len(),
28179            1
28180        );
28181
28182        let err = reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), None)
28183            .expect_err("a same-generation sweep without entry proof must retain the ledger");
28184        assert!(err.to_string().contains("not individually verified"));
28185
28186        let stale_verified = vec![unresolved_entry.clone()];
28187        let mut replacement = unresolved_entry;
28188        replacement.disk_error_count = 2;
28189        replacement.observed_at = generation + Duration::seconds(1);
28190        assert!(
28191            record_decommission_unresolved_entry(
28192                &mut pool_meta,
28193                0,
28194                generation,
28195                replacement.clone(),
28196                generation + Duration::nanoseconds(2),
28197                None,
28198            )
28199            .expect("a newer observation should replace the ledger entry")
28200        );
28201        let err =
28202            reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), Some(&stale_verified))
28203                .expect_err("stale entry verification must not clear a concurrent replacement");
28204        assert!(err.to_string().contains("not individually verified"));
28205
28206        let verified = vec![replacement];
28207        reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), Some(&verified))
28208            .expect("individually verified entries should reconcile");
28209        assert!(
28210            pool_meta.pools[0]
28211                .decommission
28212                .as_ref()
28213                .expect("decommission should exist")
28214                .unresolved_entries
28215                .is_empty()
28216        );
28217    }
28218
28219    #[tokio::test]
28220    async fn final_sweep_persists_unresolved_entry_from_real_listing() {
28221        let (dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
28222        let bucket = "decommission-final-sweep-unresolved";
28223        let object = "corrupt-object";
28224        metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
28225        store
28226            .make_bucket(bucket, &MakeBucketOptions::default())
28227            .await
28228            .expect("test bucket should be created");
28229
28230        let generation = OffsetDateTime::now_utc();
28231        {
28232            let mut pool_meta = store.pool_meta.write().await;
28233            pool_meta.dont_save = false;
28234            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
28235                start_time: Some(generation),
28236                ..Default::default()
28237            });
28238        }
28239        store
28240            .save_current_pool_meta_for_test(&[0])
28241            .await
28242            .expect("active decommission metadata should be persisted before the final sweep");
28243
28244        for (disk_index, dir) in dirs.iter().enumerate() {
28245            let object_dir = dir.path().join(bucket).join(object);
28246            tokio::fs::create_dir_all(&object_dir)
28247                .await
28248                .expect("corrupt object directory should be created");
28249            tokio::fs::write(object_dir.join(STORAGE_FORMAT_FILE), format!("corrupt-xl-meta-{disk_index}").into_bytes())
28250                .await
28251                .expect("divergent corrupt metadata should be written");
28252        }
28253
28254        let err = store
28255            .check_after_decommission(0, &CancellationToken::new(), generation)
28256            .await
28257            .expect_err("real final sweep must fail closed on unresolved listing metadata");
28258        assert!(
28259            err.to_string().contains("decommission listing could not resolve metadata"),
28260            "unexpected final sweep error: {err:?}"
28261        );
28262
28263        let in_memory_entries = store.pool_meta.read().await.pools[0]
28264            .decommission
28265            .as_ref()
28266            .expect("decommission should remain active")
28267            .unresolved_entries
28268            .clone();
28269        assert_eq!(in_memory_entries.len(), 1);
28270        let unresolved_entry = &in_memory_entries[0];
28271        assert_eq!(unresolved_entry.bucket, bucket);
28272        assert_eq!(unresolved_entry.object, object);
28273        assert_eq!(unresolved_entry.pool_index, 0);
28274        assert_eq!(unresolved_entry.set_index, 0);
28275        assert_eq!(unresolved_entry.source_generation, generation);
28276        assert_eq!(unresolved_entry.candidate_count, 4);
28277        assert_eq!(unresolved_entry.disk_error_count, 0);
28278        assert_eq!(unresolved_entry.reason, "metadata_resolution_failed");
28279
28280        let mut restored = PoolMeta::default();
28281        restored
28282            .load_no_lock_from_replicas(vec![store.pools[0].clone()])
28283            .await
28284            .expect("persisted pool metadata should reload after the final-sweep failure");
28285        assert_eq!(
28286            restored.pools[0]
28287                .decommission
28288                .as_ref()
28289                .expect("reloaded decommission should exist")
28290                .unresolved_entries,
28291            in_memory_entries
28292        );
28293    }
28294
28295    #[tokio::test]
28296    async fn decommission_metadata_saves_stay_monotonic_across_clock_rollback_before_terminal_restart() {
28297        let (_dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
28298        let persisted_floor = store.pool_meta.read().await.pools[0].last_update;
28299        let generation = persisted_floor
28300            .checked_add(Duration::seconds(1))
28301            .expect("test generation should advance the initialized pool metadata");
28302        let earlier_tick = generation - Duration::nanoseconds(10);
28303        {
28304            let mut pool_meta = store.pool_meta.write().await;
28305            pool_meta.dont_save = false;
28306            pool_meta.pools[0].last_update = generation;
28307            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
28308                start_time: Some(generation),
28309                ..Default::default()
28310            });
28311        }
28312        store
28313            .save_current_pool_meta_for_test(&[0])
28314            .await
28315            .expect("active decommission metadata should persist before rollback checkpoints");
28316
28317        let unresolved_entry = DecommissionUnresolvedEntry {
28318            bucket: "bucket-a".to_string(),
28319            object: "object-a".to_string(),
28320            pool_index: 0,
28321            set_index: 0,
28322            source_generation: generation,
28323            candidate_count: 1,
28324            disk_error_count: 1,
28325            observed_at: earlier_tick,
28326            reason: "metadata_resolution_failed".to_string(),
28327        };
28328        {
28329            let mut pool_meta = store.pool_meta.write().await;
28330            assert!(
28331                record_decommission_unresolved_entry(&mut pool_meta, 0, generation, unresolved_entry, earlier_tick, None)
28332                    .expect("unresolved entry should record under active generation")
28333            );
28334            assert_eq!(pool_meta.pools[0].last_update, generation + Duration::nanoseconds(1));
28335        }
28336        store
28337            .save_current_pool_meta_for_test(&[0])
28338            .await
28339            .expect("rollback unresolved-entry save should not stale-reject");
28340        store.pool_meta.write().await.pools[0]
28341            .decommission
28342            .as_mut()
28343            .expect("decommission metadata should exist")
28344            .items_decommissioned = DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD;
28345
28346        assert!(
28347            store
28348                .save_decommission_progress_checkpoint_at(0, generation, earlier_tick)
28349                .await
28350                .expect("rollback progress checkpoint should use a monotonic durable identity")
28351        );
28352        let progress_at = generation + Duration::nanoseconds(2);
28353        assert_eq!(store.pool_meta.read().await.pools[0].last_update, progress_at);
28354
28355        let terminal_at = generation + Duration::nanoseconds(3);
28356        {
28357            let mut pool_meta = store.pool_meta.write().await;
28358            assert!(pool_meta.decommission_failed_at_for_test(0, earlier_tick, None));
28359            assert_eq!(pool_meta.pools[0].last_update, terminal_at);
28360        }
28361        store
28362            .save_current_pool_meta_for_test(&[0])
28363            .await
28364            .expect("terminal failure after rollback checkpoints should persist");
28365
28366        let mut restored = PoolMeta::default();
28367        restored
28368            .load_no_lock_from_replicas(vec![store.pools[0].clone()])
28369            .await
28370            .expect("terminal metadata should reload after restart");
28371        assert_eq!(restored.pools[0].last_update, terminal_at);
28372        assert!(
28373            restored.pools[0]
28374                .decommission
28375                .as_ref()
28376                .is_some_and(|info| info.failed && !info.complete && !info.canceled)
28377        );
28378
28379        let restarted = decommission_worker_test_store(restored, Vec::new());
28380        let status = restarted.scanner_data_movement_pause_status().await;
28381        let expected_generation =
28382            u64::try_from(terminal_at.unix_timestamp_nanos()).expect("fixed positive timestamp should fit generation");
28383
28384        assert_eq!(status.movement_generation, expected_generation);
28385        assert_eq!(status.reasons, vec![crate::store::ScannerDataMovementPauseReason::DecommissionFailed]);
28386        assert_eq!(restarted.scanner_data_movement_generation(), expected_generation);
28387    }
28388
28389    #[tokio::test]
28390    async fn unresolved_entry_probe_verifies_absence_on_every_source_disk() {
28391        let (_dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
28392        let bucket = "decommission-final-sweep-absent-ledger";
28393        store
28394            .peer_sys
28395            .make_bucket(bucket, &MakeBucketOptions::default())
28396            .await
28397            .expect("test bucket should be created");
28398        metadata_sys::init_bucket_metadata_sys(store.clone(), vec![bucket.to_string()]).await;
28399
28400        let generation = OffsetDateTime::now_utc();
28401        let unresolved_entry = DecommissionUnresolvedEntry {
28402            bucket: bucket.to_string(),
28403            object: "absent-object".to_string(),
28404            pool_index: 0,
28405            set_index: 0,
28406            source_generation: generation,
28407            candidate_count: 1,
28408            disk_error_count: 1,
28409            observed_at: generation,
28410            reason: "metadata_resolution_failed".to_string(),
28411        };
28412        {
28413            let mut pool_meta = store.pool_meta.write().await;
28414            pool_meta.version = POOL_META_VERSION;
28415            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
28416                start_time: Some(generation),
28417                unresolved_entries: vec![unresolved_entry.clone()],
28418                ..Default::default()
28419            });
28420        }
28421
28422        assert!(
28423            store.pools[0].disk_set[0]
28424                .decommission_unresolved_entry_absent_on_all_disks(0, &unresolved_entry)
28425                .await
28426                .expect("all source disks should be readable")
28427        );
28428        let verified = vec![unresolved_entry.clone()];
28429
28430        let mut pool_meta = store.pool_meta.write().await;
28431        reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), Some(&verified))
28432            .expect("the individually verified entry should reconcile");
28433        assert!(
28434            pool_meta.pools[0]
28435                .decommission
28436                .as_ref()
28437                .expect("decommission should remain present")
28438                .unresolved_entries
28439                .is_empty()
28440        );
28441    }
28442
28443    #[tokio::test]
28444    async fn test_record_decommission_entry_error_cancels_listing_and_preserves_first_error() {
28445        let entry_error = Arc::new(tokio::sync::Mutex::new(None));
28446        let rx = CancellationToken::new();
28447
28448        assert!(record_decommission_entry_error(&entry_error, &rx, Error::SlowDown).await);
28449        assert!(!record_decommission_entry_error(&entry_error, &rx, Error::OperationCanceled).await);
28450
28451        assert!(rx.is_cancelled());
28452        assert!(matches!(*entry_error.lock().await, Some(Error::SlowDown)));
28453    }
28454
28455    #[tokio::test]
28456    async fn test_record_decommission_entry_error_ignores_already_canceled_listing() {
28457        let entry_error = Arc::new(tokio::sync::Mutex::new(None));
28458        let rx = CancellationToken::new();
28459        rx.cancel();
28460
28461        assert!(!record_decommission_entry_error(&entry_error, &rx, Error::SlowDown).await);
28462
28463        assert!(entry_error.lock().await.is_none());
28464    }
28465
28466    #[test]
28467    fn unresolved_entry_verification_requires_every_source_disk() {
28468        assert!(ensure_decommission_unresolved_verification_disk_count(4, 4, 0, 1).is_ok());
28469
28470        let err = ensure_decommission_unresolved_verification_disk_count(4, 3, 0, 1)
28471            .expect_err("an offline source disk must block ledger reconciliation");
28472        assert!(err.to_string().contains("requires all 4 source disks, but only 3 are online"));
28473    }
28474
28475    #[test]
28476    fn test_resolve_decommission_listing_error_preserves_real_listing_failure() {
28477        let err = resolve_decommission_listing_error(Some(Error::SlowDown), Some(Error::OperationCanceled))
28478            .expect("listing failure should be returned");
28479        assert!(matches!(err, Error::SlowDown));
28480
28481        let err = resolve_decommission_listing_error(Some(Error::OperationCanceled), Some(Error::SlowDown))
28482            .expect("entry failure should be returned");
28483        assert!(matches!(err, Error::SlowDown));
28484    }
28485
28486    #[test]
28487    fn test_resolve_decommission_check_after_list_result_returns_list_result_without_entry_error() {
28488        let err = resolve_decommission_check_after_list_result(Err(Error::OperationCanceled), None)
28489            .expect_err("list result should be preserved without entry error");
28490        assert!(matches!(err, Error::OperationCanceled));
28491    }
28492
28493    #[test]
28494    fn test_resolve_decommission_pool_meta_reload_result_passthrough_ok() {
28495        assert!(resolve_decommission_pool_meta_reload_result(Ok(()), "start_decommission").is_ok());
28496    }
28497
28498    #[test]
28499    fn test_resolve_decommission_pool_meta_reload_result_wraps_error_context() {
28500        let err = resolve_decommission_pool_meta_reload_result(Err(Error::SlowDown), "decommission_failed for pool 3")
28501            .expect_err("reload failure should be wrapped with stage context");
28502        let message = err.to_string();
28503        assert!(message.contains("decommission pool meta reload failed during decommission_failed for pool 3"));
28504        assert!(message.contains(Error::SlowDown.to_string().as_str()));
28505    }
28506
28507    #[test]
28508    fn test_resolve_start_decommission_pool_meta_reload_result_returns_failure() {
28509        let err = resolve_start_decommission_pool_meta_reload_result(Err(Error::other(
28510            "reload_pool_meta encountered 1 failure(s): peer[0] reload_pool_meta failed",
28511        )))
28512        .expect_err("start_decommission must fail when peer pool meta reload fails");
28513        let message = err.to_string();
28514
28515        assert!(message.contains("decommission pool meta reload failed during start_decommission"));
28516        assert!(message.contains("reload_pool_meta encountered 1 failure(s)"));
28517        assert!(message.contains("peer[0]"));
28518    }
28519
28520    #[test]
28521    fn test_resolve_decommission_listing_worker_result_passthrough_ok() {
28522        assert!(resolve_decommission_listing_worker_result(2, Ok(Ok(()))).is_ok());
28523    }
28524
28525    #[test]
28526    fn test_resolve_decommission_listing_worker_result_passthrough_worker_error() {
28527        let err = resolve_decommission_listing_worker_result(2, Ok(Err(Error::SlowDown)))
28528            .expect_err("listing worker error should be returned");
28529
28530        assert!(matches!(err, Error::SlowDown));
28531    }
28532
28533    #[tokio::test]
28534    async fn test_resolve_decommission_listing_worker_result_wraps_join_error_context() {
28535        let join_error = tokio::spawn(async {
28536            panic!("listing worker panic");
28537        })
28538        .await
28539        .expect_err("panic task should return JoinError");
28540
28541        let err = resolve_decommission_listing_worker_result(4, Err(join_error))
28542            .expect_err("join error should be wrapped with context");
28543        let message = err.to_string();
28544        assert!(message.contains("decommission listing worker 4 task join error"));
28545        assert!(message.contains("panic"));
28546    }
28547
28548    #[test]
28549    fn test_should_retry_decommission_listing_respects_attempt_limit_and_bucket_missing() {
28550        assert!(should_retry_decommission_listing(&Error::SlowDown, 0, 2));
28551        assert!(!should_retry_decommission_listing(&Error::SlowDown, 1, 2));
28552        assert!(!should_retry_decommission_listing(
28553            &StorageError::BucketNotFound("bucket".to_string()),
28554            0,
28555            2
28556        ));
28557    }
28558
28559    #[tokio::test]
28560    async fn test_wait_decommission_retry_backoff_reports_canceled_without_sleeping() {
28561        let token = CancellationToken::new();
28562        token.cancel();
28563
28564        assert!(wait_decommission_retry_backoff(&token, StdDuration::from_secs(30)).await);
28565    }
28566
28567    #[test]
28568    fn test_decommission_retry_backoff_ceiling_grows_exponentially_and_caps() {
28569        let base = StdDuration::from_millis(100);
28570
28571        assert_eq!(decommission_retry_backoff_ceiling(base, 1), StdDuration::from_millis(100));
28572        assert_eq!(decommission_retry_backoff_ceiling(base, 2), StdDuration::from_millis(200));
28573        assert_eq!(decommission_retry_backoff_ceiling(base, 3), StdDuration::from_millis(400));
28574        assert_eq!(decommission_retry_backoff_ceiling(base, 4), StdDuration::from_millis(800));
28575        assert_eq!(decommission_retry_backoff_ceiling(base, 5), DECOMMISSION_RETRY_BACKOFF_CAP);
28576        assert_eq!(decommission_retry_backoff_ceiling(base, usize::MAX), DECOMMISSION_RETRY_BACKOFF_CAP);
28577    }
28578
28579    #[test]
28580    fn test_decommission_retry_backoff_delay_stays_within_jittered_ceiling() {
28581        let base = StdDuration::from_millis(100);
28582
28583        for attempt in 1..=6 {
28584            let ceiling = decommission_retry_backoff_ceiling(base, attempt);
28585            let delay = decommission_retry_backoff_delay(base, attempt);
28586            assert!(delay <= ceiling, "delay {delay:?} must not exceed ceiling {ceiling:?}");
28587        }
28588        assert_eq!(decommission_retry_backoff_delay(StdDuration::ZERO, 3), StdDuration::ZERO);
28589    }
28590
28591    #[test]
28592    fn test_source_changed_exhaustion_fails_only_after_pool_limit() {
28593        assert!(!should_fail_decommission_pool_after_exhausted_source_changed(
28594            DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT
28595        ));
28596        assert!(should_fail_decommission_pool_after_exhausted_source_changed(
28597            DECOMMISSION_SOURCE_CHANGED_EXHAUSTION_LIMIT + 1
28598        ));
28599    }
28600
28601    #[tokio::test(start_paused = true)]
28602    async fn test_run_decommission_listing_with_retry_stops_after_attempt_limit() {
28603        let attempts = Arc::new(AtomicUsize::new(0));
28604        let err = run_decommission_listing_with_retry(
28605            CancellationToken::new(),
28606            "bucket-a".to_string(),
28607            noop_decommission_list_callback(),
28608            1,
28609            2,
28610            3,
28611            {
28612                let attempts = attempts.clone();
28613                move |_| {
28614                    let attempts = attempts.clone();
28615                    async move {
28616                        attempts.fetch_add(1, Ordering::SeqCst);
28617                        Err(Error::SlowDown)
28618                    }
28619                }
28620            },
28621        )
28622        .await
28623        .expect_err("permanent listing failure must not retry forever");
28624
28625        assert_eq!(attempts.load(Ordering::SeqCst), 3);
28626        assert!(err.to_string().contains("attempt 3/3"));
28627    }
28628
28629    #[tokio::test]
28630    async fn test_run_decommission_listing_with_retry_treats_bucket_missing_as_complete() {
28631        let attempts = Arc::new(AtomicUsize::new(0));
28632        run_decommission_listing_with_retry(
28633            CancellationToken::new(),
28634            "bucket-a".to_string(),
28635            noop_decommission_list_callback(),
28636            1,
28637            2,
28638            3,
28639            {
28640                let attempts = attempts.clone();
28641                move |_| {
28642                    let attempts = attempts.clone();
28643                    async move {
28644                        attempts.fetch_add(1, Ordering::SeqCst);
28645                        Err(StorageError::BucketNotFound("bucket-a".to_string()))
28646                    }
28647                }
28648            },
28649        )
28650        .await
28651        .expect("missing bucket should keep previous decommission listing behavior");
28652
28653        assert_eq!(attempts.load(Ordering::SeqCst), 1);
28654    }
28655
28656    #[test]
28657    fn test_should_count_decommission_version_complete_for_cleanup_safe_ignored_result() {
28658        assert!(should_count_decommission_version_complete(true, true, false));
28659    }
28660
28661    #[test]
28662    fn test_should_count_decommission_version_complete_rejects_skip_only_ignored_result() {
28663        assert!(!should_count_decommission_version_complete(true, false, false));
28664    }
28665
28666    #[test]
28667    fn test_should_count_decommission_version_complete_for_completed_result() {
28668        assert!(should_count_decommission_version_complete(false, false, false));
28669    }
28670
28671    #[test]
28672    fn test_should_count_decommission_version_complete_rejects_failed_result() {
28673        assert!(!should_count_decommission_version_complete(false, false, true));
28674    }
28675
28676    #[test]
28677    fn test_should_cleanup_decommission_source_entry_accepts_all_versions_completed() {
28678        assert!(should_cleanup_decommission_source_entry(3, 3, 0));
28679    }
28680
28681    #[test]
28682    fn test_should_cleanup_decommission_source_entry_accepts_migrated_and_safely_expired_versions() {
28683        assert!(should_cleanup_decommission_source_entry(1, 2, 1));
28684    }
28685
28686    #[test]
28687    fn test_should_cleanup_decommission_source_entry_accepts_versions_only_safely_expired_by_lifecycle() {
28688        assert!(should_cleanup_decommission_source_entry(0, 2, 2));
28689    }
28690
28691    #[test]
28692    fn test_should_cleanup_decommission_source_entry_rejects_object_lock_retained_version() {
28693        assert!(!should_cleanup_decommission_source_entry(1, 2, 0));
28694    }
28695
28696    #[test]
28697    fn test_should_cleanup_decommission_source_entry_rejects_replication_pending_version() {
28698        assert!(!should_cleanup_decommission_source_entry(2, 3, 0));
28699    }
28700
28701    #[test]
28702    fn test_should_cleanup_decommission_source_entry_rejects_counter_overrun() {
28703        assert!(!should_cleanup_decommission_source_entry(2, 2, 1));
28704    }
28705
28706    #[test]
28707    fn test_pool_meta_update_after_rejects_out_of_range_index() {
28708        let mut meta = PoolMeta::default();
28709        let err = meta
28710            .update_after(1, Duration::seconds(1))
28711            .expect_err("out-of-range index should fail");
28712        assert!(err.to_string().contains("invalid decommission pool index 1 for 0 pools"));
28713    }
28714
28715    #[test]
28716    fn test_pool_meta_update_after_rejects_when_decommission_missing() {
28717        let mut meta = PoolMeta {
28718            pools: vec![PoolStatus {
28719                id: 0,
28720                cmd_line: "pool-0".to_string(),
28721                last_update: OffsetDateTime::UNIX_EPOCH,
28722                decommission: None,
28723            }],
28724            ..Default::default()
28725        };
28726
28727        let err = meta
28728            .update_after(0, Duration::seconds(1))
28729            .expect_err("pool without decommission should fail");
28730        assert!(
28731            err.to_string()
28732                .contains("failed to update decommission metadata timestamp: decommission metadata not initialized")
28733        );
28734    }
28735
28736    #[test]
28737    fn test_track_decommission_stage_does_not_advance_checkpoint_state() {
28738        let mut meta = PoolMeta {
28739            pools: vec![PoolStatus {
28740                id: 0,
28741                cmd_line: "pool-0".to_string(),
28742                last_update: OffsetDateTime::UNIX_EPOCH,
28743                decommission: Some(PoolDecommissionInfo {
28744                    items_decommissioned: 3,
28745                    items_decommission_failed: 2,
28746                    ..Default::default()
28747                }),
28748            }],
28749            ..Default::default()
28750        };
28751
28752        track_decommission_current_object_stage(&mut meta, 0, "bucket", "object", "migrate_object")
28753            .expect("valid decommission progress should be tracked");
28754
28755        assert_eq!(meta.pools[0].last_update, OffsetDateTime::UNIX_EPOCH);
28756        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
28757        assert_eq!(info.items_since_last_progress_save(), 5);
28758        assert_eq!(info.stage, "migrate_object");
28759    }
28760
28761    #[test]
28762    fn test_pool_meta_update_after_skips_before_time_and_item_thresholds() {
28763        let mut meta = PoolMeta {
28764            pools: vec![PoolStatus {
28765                id: 0,
28766                cmd_line: "pool-0".to_string(),
28767                last_update: OffsetDateTime::now_utc(),
28768                decommission: Some(PoolDecommissionInfo {
28769                    items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD - 1,
28770                    ..Default::default()
28771                }),
28772            }],
28773            ..Default::default()
28774        };
28775
28776        let saved = meta
28777            .update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
28778            .expect("valid decommission state should update");
28779
28780        assert!(!saved);
28781        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
28782        assert_eq!(info.items_since_last_progress_save(), DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD - 1);
28783    }
28784
28785    #[test]
28786    fn test_pool_meta_update_after_requests_save_when_item_threshold_reached() {
28787        let mut meta = PoolMeta {
28788            pools: vec![PoolStatus {
28789                id: 0,
28790                cmd_line: "pool-0".to_string(),
28791                last_update: OffsetDateTime::now_utc(),
28792                decommission: Some(PoolDecommissionInfo {
28793                    items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
28794                    ..Default::default()
28795                }),
28796            }],
28797            ..Default::default()
28798        };
28799
28800        let saved = meta
28801            .update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
28802            .expect("item threshold should save progress");
28803
28804        assert!(saved);
28805        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
28806        assert_eq!(info.items_since_last_progress_save(), DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD);
28807    }
28808
28809    #[test]
28810    fn test_pool_meta_update_after_requests_save_when_time_threshold_reached() {
28811        let mut meta = PoolMeta {
28812            pools: vec![PoolStatus {
28813                id: 0,
28814                cmd_line: "pool-0".to_string(),
28815                last_update: OffsetDateTime::now_utc() - DECOMMISSION_PROGRESS_SAVE_INTERVAL,
28816                decommission: Some(PoolDecommissionInfo {
28817                    items_decommissioned: 1,
28818                    ..Default::default()
28819                }),
28820            }],
28821            ..Default::default()
28822        };
28823
28824        let saved = meta
28825            .update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
28826            .expect("time threshold should save progress");
28827
28828        assert!(saved);
28829        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
28830        assert_eq!(info.items_since_last_progress_save(), 1);
28831    }
28832
28833    #[test]
28834    fn test_pool_meta_update_after_does_not_advance_last_update_before_save() {
28835        let last_update = OffsetDateTime::UNIX_EPOCH;
28836        let mut meta = PoolMeta {
28837            pools: vec![PoolStatus {
28838                id: 0,
28839                cmd_line: "pool-0".to_string(),
28840                last_update,
28841                decommission: Some(PoolDecommissionInfo {
28842                    start_time: Some(last_update),
28843                    items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
28844                    ..Default::default()
28845                }),
28846            }],
28847            ..Default::default()
28848        };
28849
28850        assert!(
28851            meta.update_after(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL)
28852                .expect("item threshold should request a checkpoint")
28853        );
28854        assert_eq!(meta.pools[0].last_update, last_update);
28855    }
28856
28857    #[test]
28858    fn test_decommission_progress_checkpoint_commits_exact_snapshot_watermark() {
28859        let start_time = OffsetDateTime::UNIX_EPOCH;
28860        let checkpoint_at = start_time + Duration::seconds(30);
28861        let mut meta = PoolMeta {
28862            pools: vec![PoolStatus {
28863                id: 0,
28864                cmd_line: "pool-0".to_string(),
28865                last_update: start_time,
28866                decommission: Some(PoolDecommissionInfo {
28867                    start_time: Some(start_time),
28868                    items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
28869                    ..Default::default()
28870                }),
28871            }],
28872            ..Default::default()
28873        };
28874
28875        let checkpoint = meta
28876            .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None)
28877            .expect("valid decommission state should produce a checkpoint")
28878            .expect("item threshold should produce a checkpoint");
28879        meta.count_item(0, 1, false);
28880
28881        assert!(meta.commit_decommission_progress_checkpoint(0, checkpoint));
28882        let info = meta.pools[0].decommission.as_ref().expect("decommission info should exist");
28883        assert_eq!(info.progress_save_item_baseline, checkpoint.counted_items);
28884        assert_eq!(info.items_since_last_progress_save(), 1);
28885        assert_eq!(meta.pools[0].last_update, checkpoint_at);
28886    }
28887
28888    #[test]
28889    fn test_decommission_progress_checkpoint_backoff_does_not_advance_baseline() {
28890        let start_time = OffsetDateTime::UNIX_EPOCH;
28891        let checkpoint_at = start_time + Duration::seconds(30);
28892        let retry_after = checkpoint_at + DECOMMISSION_PROGRESS_SAVE_RETRY_BACKOFF;
28893        let mut meta = PoolMeta {
28894            pools: vec![PoolStatus {
28895                id: 0,
28896                cmd_line: "pool-0".to_string(),
28897                last_update: start_time,
28898                decommission: Some(PoolDecommissionInfo {
28899                    start_time: Some(start_time),
28900                    items_decommissioned: DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD,
28901                    ..Default::default()
28902                }),
28903            }],
28904            ..Default::default()
28905        };
28906
28907        let checkpoint = meta
28908            .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None)
28909            .expect("valid decommission state should produce a checkpoint")
28910            .expect("item threshold should produce a checkpoint");
28911        meta.defer_decommission_progress_checkpoint(0, checkpoint, retry_after);
28912
28913        assert!(
28914            meta.decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None)
28915                .expect("retry backoff check should succeed")
28916                .is_none()
28917        );
28918        assert_eq!(meta.pools[0].last_update, start_time);
28919        assert_eq!(
28920            meta.pools[0]
28921                .decommission
28922                .as_ref()
28923                .expect("decommission info should exist")
28924                .progress_save_item_baseline,
28925            0
28926        );
28927    }
28928
28929    #[test]
28930    fn test_decommission_progress_checkpoint_count_scales_with_threshold() {
28931        let start_time = OffsetDateTime::UNIX_EPOCH;
28932        let checkpoint_at = start_time;
28933        let mut meta = PoolMeta {
28934            pools: vec![PoolStatus {
28935                id: 0,
28936                cmd_line: "pool-0".to_string(),
28937                last_update: start_time,
28938                decommission: Some(PoolDecommissionInfo {
28939                    start_time: Some(start_time),
28940                    ..Default::default()
28941                }),
28942            }],
28943            ..Default::default()
28944        };
28945        let mut checkpoint_count = 0;
28946
28947        for _ in 0..(DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD * 10) {
28948            meta.count_item(0, 1, false);
28949            if let Some(checkpoint) = meta
28950                .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None)
28951                .expect("valid decommission state should produce a checkpoint")
28952            {
28953                checkpoint_count += 1;
28954                assert!(meta.commit_decommission_progress_checkpoint(0, checkpoint));
28955            }
28956        }
28957
28958        assert_eq!(checkpoint_count, 10);
28959    }
28960
28961    #[test]
28962    fn test_ensure_decommission_not_rebalancing_rejects_running_rebalance() {
28963        let err = ensure_decommission_not_rebalancing(true).expect_err("rebalance running should be rejected");
28964        assert!(matches!(err, Error::RebalanceAlreadyRunning));
28965    }
28966
28967    #[test]
28968    fn test_ensure_decommission_not_rebalancing_allows_idle() {
28969        assert!(ensure_decommission_not_rebalancing(false).is_ok());
28970    }
28971
28972    #[test]
28973    fn test_ensure_decommission_start_rebalance_meta_allowed_rejects_active_rebalance() {
28974        let meta = RebalanceMeta {
28975            pool_stats: vec![RebalanceStats {
28976                participating: true,
28977                info: RebalanceInfo {
28978                    status: RebalStatus::Started,
28979                    ..Default::default()
28980                },
28981                ..Default::default()
28982            }],
28983            ..Default::default()
28984        };
28985
28986        let err = ensure_decommission_start_rebalance_meta_allowed(Some(&meta))
28987            .expect_err("persisted active rebalance should block decommission start");
28988
28989        assert!(matches!(err, Error::RebalanceAlreadyRunning));
28990    }
28991
28992    #[test]
28993    fn test_ensure_decommission_start_rebalance_meta_allowed_rejects_stopping_rebalance() {
28994        let meta = RebalanceMeta {
28995            pool_stats: vec![RebalanceStats {
28996                info: RebalanceInfo {
28997                    status: RebalStatus::Started,
28998                    stopping: true,
28999                    ..Default::default()
29000                },
29001                ..Default::default()
29002            }],
29003            ..Default::default()
29004        };
29005
29006        let err = ensure_decommission_start_rebalance_meta_allowed(Some(&meta))
29007            .expect_err("persisted stopping rebalance should block decommission start");
29008
29009        assert!(matches!(err, Error::RebalanceAlreadyRunning));
29010    }
29011
29012    #[test]
29013    fn test_ensure_decommission_start_rebalance_meta_allowed_allows_terminal_or_missing_rebalance() {
29014        let terminal_meta = RebalanceMeta {
29015            pool_stats: vec![RebalanceStats {
29016                participating: true,
29017                info: RebalanceInfo {
29018                    status: RebalStatus::Completed,
29019                    ..Default::default()
29020                },
29021                ..Default::default()
29022            }],
29023            ..Default::default()
29024        };
29025
29026        assert!(ensure_decommission_start_rebalance_meta_allowed(Some(&terminal_meta)).is_ok());
29027        assert!(ensure_decommission_start_rebalance_meta_allowed(None).is_ok());
29028    }
29029
29030    #[test]
29031    fn test_ensure_local_decommission_pool_leaders_allows_local_first_endpoint() {
29032        let endpoints = EndpointServerPools::from(vec![
29033            decommission_test_pool_endpoint(0, false),
29034            decommission_test_pool_endpoint(1, true),
29035        ]);
29036
29037        assert!(ensure_local_decommission_pool_leaders(&endpoints, &[1]).is_ok());
29038    }
29039
29040    #[test]
29041    fn test_ensure_local_decommission_pool_leaders_rejects_remote_first_endpoint() {
29042        let endpoints = EndpointServerPools::from(vec![decommission_test_pool_endpoint(0, false)]);
29043
29044        let err = ensure_local_decommission_pool_leaders(&endpoints, &[0])
29045            .expect_err("remote first endpoint should reject local decommission start");
29046
29047        assert!(err.to_string().contains("must run on the pool first endpoint"));
29048    }
29049
29050    #[test]
29051    fn test_ensure_local_decommission_pool_leaders_rejects_empty_endpoints() {
29052        let endpoints = EndpointServerPools::from(vec![PoolEndpoints {
29053            legacy: false,
29054            set_count: 1,
29055            drives_per_set: 1,
29056            endpoints: Endpoints::from(Vec::<Endpoint>::new()),
29057            cmd_line: "pool-0".to_string(),
29058            platform: String::new(),
29059        }]);
29060
29061        let err = ensure_local_decommission_pool_leaders(&endpoints, &[0])
29062            .expect_err("pool without endpoints should reject local decommission start");
29063
29064        assert!(err.to_string().contains("has no configured endpoints"));
29065    }
29066
29067    #[test]
29068    fn test_decommission_meta_bucket_options_are_idempotent() {
29069        let opts = decommission_meta_bucket_options();
29070
29071        assert!(opts.force_create);
29072    }
29073
29074    #[test]
29075    fn test_is_decommission_active_true_only_when_not_terminal() {
29076        assert!(is_decommission_active(false, false, false));
29077        assert!(!is_decommission_active(true, false, false));
29078        assert!(!is_decommission_active(false, true, false));
29079        assert!(!is_decommission_active(false, false, true));
29080    }
29081
29082    #[test]
29083    fn test_ensure_decommission_generation_rejects_stale_or_queued_workers() {
29084        let generation = OffsetDateTime::UNIX_EPOCH;
29085        let mut meta = PoolMeta {
29086            pools: vec![PoolStatus {
29087                id: 0,
29088                cmd_line: "pool-0".to_string(),
29089                last_update: generation,
29090                decommission: Some(PoolDecommissionInfo {
29091                    start_time: Some(generation),
29092                    ..Default::default()
29093                }),
29094            }],
29095            ..Default::default()
29096        };
29097
29098        assert!(ensure_decommission_generation(&meta, 0, generation).is_ok());
29099        assert!(ensure_decommission_generation(&meta, 0, generation + Duration::seconds(1)).is_err());
29100
29101        meta.pools[0]
29102            .decommission
29103            .as_mut()
29104            .expect("decommission metadata should exist")
29105            .queued = true;
29106        assert!(ensure_decommission_generation(&meta, 0, generation).is_err());
29107
29108        let replacement_generation = generation + Duration::seconds(2);
29109        let info = meta.pools[0]
29110            .decommission
29111            .as_mut()
29112            .expect("decommission metadata should exist");
29113        info.queued = false;
29114        info.start_time = Some(replacement_generation);
29115        assert!(ensure_decommission_generation(&meta, 0, generation).is_err());
29116        assert!(ensure_decommission_generation(&meta, 0, replacement_generation).is_ok());
29117    }
29118
29119    #[test]
29120    fn test_pool_meta_has_active_decommission_counts_running_and_queued_states() {
29121        let active_meta = PoolMeta {
29122            pools: vec![PoolStatus {
29123                id: 0,
29124                cmd_line: "pool-0".to_string(),
29125                last_update: OffsetDateTime::UNIX_EPOCH,
29126                decommission: Some(PoolDecommissionInfo {
29127                    start_time: Some(OffsetDateTime::UNIX_EPOCH),
29128                    ..Default::default()
29129                }),
29130            }],
29131            ..Default::default()
29132        };
29133        let queued_meta = PoolMeta {
29134            pools: vec![PoolStatus {
29135                id: 0,
29136                cmd_line: "pool-0".to_string(),
29137                last_update: OffsetDateTime::UNIX_EPOCH,
29138                decommission: Some(PoolDecommissionInfo {
29139                    queued: true,
29140                    ..Default::default()
29141                }),
29142            }],
29143            ..Default::default()
29144        };
29145
29146        assert!(pool_meta_has_active_decommission(&active_meta));
29147        assert!(pool_meta_has_active_decommission(&queued_meta));
29148    }
29149
29150    #[test]
29151    fn test_pool_meta_has_active_decommission_ignores_capacity_placeholder() {
29152        let meta = PoolMeta {
29153            pools: vec![PoolStatus {
29154                id: 0,
29155                cmd_line: "pool-0".to_string(),
29156                last_update: OffsetDateTime::UNIX_EPOCH,
29157                decommission: Some(PoolDecommissionInfo {
29158                    total_size: 100,
29159                    current_size: 75,
29160                    ..Default::default()
29161                }),
29162            }],
29163            ..Default::default()
29164        };
29165
29166        assert!(!pool_meta_has_active_decommission(&meta));
29167        assert!(!meta.is_suspended(0));
29168        assert_eq!(decommission_start_pool_state(meta.pools.first()), DecommissionStartPoolState::Active);
29169    }
29170
29171    #[test]
29172    fn test_pool_meta_has_active_decommission_ignores_terminal_states() {
29173        let terminal_meta = PoolMeta {
29174            pools: vec![
29175                PoolStatus {
29176                    id: 0,
29177                    cmd_line: "pool-0".to_string(),
29178                    last_update: OffsetDateTime::UNIX_EPOCH,
29179                    decommission: Some(PoolDecommissionInfo {
29180                        complete: true,
29181                        ..Default::default()
29182                    }),
29183                },
29184                PoolStatus {
29185                    id: 1,
29186                    cmd_line: "pool-1".to_string(),
29187                    last_update: OffsetDateTime::UNIX_EPOCH,
29188                    decommission: Some(PoolDecommissionInfo {
29189                        failed: true,
29190                        ..Default::default()
29191                    }),
29192                },
29193                PoolStatus {
29194                    id: 2,
29195                    cmd_line: "pool-2".to_string(),
29196                    last_update: OffsetDateTime::UNIX_EPOCH,
29197                    decommission: Some(PoolDecommissionInfo {
29198                        canceled: true,
29199                        ..Default::default()
29200                    }),
29201                },
29202            ],
29203            ..Default::default()
29204        };
29205
29206        assert!(!pool_meta_has_active_decommission(&terminal_meta));
29207    }
29208
29209    #[test]
29210    fn test_decommission_request_rejections_preserve_invalid_argument_type() {
29211        for result in [
29212            ensure_decommission_start_allowed(DecommissionStartPoolState::Missing),
29213            ensure_decommission_start_allowed(DecommissionStartPoolState::Decommissioned),
29214            ensure_decommission_start_allowed(DecommissionStartPoolState::Blocked),
29215            ensure_decommission_cancel_allowed(false, false, false),
29216            ensure_decommission_clear_allowed(false, false, false, false, false, 0),
29217            ensure_decommission_clear_allowed(true, true, false, true, false, 1),
29218            ensure_decommission_terminal_operation_supported(true, "cancel decommission"),
29219            validate_start_decommission_request(&[], false),
29220            validate_start_decommission_request(&[0], true),
29221            ensure_decommission_start_keeps_active_pool(&PoolMeta::default(), &[]),
29222        ] {
29223            let err = result.expect_err("invalid lifecycle requests must be rejected before mutation");
29224            assert!(matches!(&err, Error::InvalidArgument(_, _, reason) if !reason.is_empty()), "{err:?}");
29225        }
29226    }
29227
29228    #[test]
29229    fn test_ensure_decommission_start_allowed_rejects_missing_pool() {
29230        let err =
29231            ensure_decommission_start_allowed(DecommissionStartPoolState::Missing).expect_err("missing pool should be invalid");
29232        assert!(
29233            err.to_string()
29234                .contains("failed to start decommission: target pool was not found")
29235        );
29236    }
29237
29238    #[test]
29239    fn test_ensure_decommission_start_allowed_rejects_running_state() {
29240        let err = ensure_decommission_start_allowed(DecommissionStartPoolState::Decommissioning)
29241            .expect_err("active decommission should be rejected");
29242        assert!(matches!(err, Error::DecommissionAlreadyRunning));
29243    }
29244
29245    #[test]
29246    fn test_ensure_decommission_start_allowed_rejects_completed_state() {
29247        let err = ensure_decommission_start_allowed(DecommissionStartPoolState::Decommissioned)
29248            .expect_err("completed decommission should be rejected");
29249        assert!(err.to_string().contains("target pool is already decommissioned"));
29250    }
29251
29252    #[test]
29253    fn test_ensure_decommission_start_allowed_rejects_blocked_state() {
29254        let err = ensure_decommission_start_allowed(DecommissionStartPoolState::Blocked)
29255            .expect_err("blocked decommission should be rejected");
29256        assert!(err.to_string().contains("target pool decommission is blocked"));
29257    }
29258
29259    #[test]
29260    fn test_ensure_decommission_start_allowed_allows_active_state() {
29261        assert!(ensure_decommission_start_allowed(DecommissionStartPoolState::Active).is_ok());
29262    }
29263
29264    #[test]
29265    fn test_ensure_decommission_start_allowed_allows_retryable_state() {
29266        assert!(ensure_decommission_start_allowed(DecommissionStartPoolState::Retryable).is_ok());
29267    }
29268
29269    #[test]
29270    fn test_decommission_start_pool_state_reports_missing_pool() {
29271        assert_eq!(decommission_start_pool_state(None), DecommissionStartPoolState::Missing);
29272    }
29273
29274    #[test]
29275    fn test_decommission_start_pool_state_reports_idle_pool_without_decommission_info() {
29276        let pool = PoolStatus {
29277            id: 0,
29278            cmd_line: "pool-0".to_string(),
29279            last_update: OffsetDateTime::UNIX_EPOCH,
29280            decommission: None,
29281        };
29282
29283        assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Active);
29284    }
29285
29286    #[test]
29287    fn test_decommission_start_pool_state_reports_decommissioning_pool_when_not_terminal() {
29288        let pool = PoolStatus {
29289            id: 0,
29290            cmd_line: "pool-0".to_string(),
29291            last_update: OffsetDateTime::UNIX_EPOCH,
29292            decommission: Some(PoolDecommissionInfo {
29293                start_time: Some(OffsetDateTime::UNIX_EPOCH),
29294                complete: false,
29295                failed: false,
29296                canceled: false,
29297                ..Default::default()
29298            }),
29299        };
29300
29301        assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Decommissioning);
29302    }
29303
29304    #[test]
29305    fn test_decommission_start_pool_state_reports_canceled_pool_as_blocked() {
29306        let pool = PoolStatus {
29307            id: 0,
29308            cmd_line: "pool-0".to_string(),
29309            last_update: OffsetDateTime::UNIX_EPOCH,
29310            decommission: Some(PoolDecommissionInfo {
29311                complete: false,
29312                failed: false,
29313                canceled: true,
29314                ..Default::default()
29315            }),
29316        };
29317
29318        assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Blocked);
29319    }
29320
29321    #[test]
29322    fn test_decommission_start_pool_state_reports_failed_pool_as_blocked() {
29323        let pool = PoolStatus {
29324            id: 0,
29325            cmd_line: "pool-0".to_string(),
29326            last_update: OffsetDateTime::UNIX_EPOCH,
29327            decommission: Some(PoolDecommissionInfo {
29328                failed: true,
29329                ..Default::default()
29330            }),
29331        };
29332
29333        assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Blocked);
29334    }
29335
29336    #[test]
29337    fn test_decommission_start_pool_state_reports_terminal_pool_with_unresolved_entries_as_retryable() {
29338        let pool = PoolStatus {
29339            id: 0,
29340            cmd_line: "pool-0".to_string(),
29341            last_update: OffsetDateTime::UNIX_EPOCH,
29342            decommission: Some(PoolDecommissionInfo {
29343                failed: true,
29344                unresolved_entries: vec![DecommissionUnresolvedEntry {
29345                    bucket: "bucket-a".to_string(),
29346                    object: "object-a".to_string(),
29347                    pool_index: 0,
29348                    set_index: 0,
29349                    source_generation: OffsetDateTime::UNIX_EPOCH,
29350                    candidate_count: 1,
29351                    disk_error_count: 1,
29352                    observed_at: OffsetDateTime::UNIX_EPOCH,
29353                    reason: "metadata_resolution_failed".to_string(),
29354                }],
29355                ..Default::default()
29356            }),
29357        };
29358
29359        assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Retryable);
29360    }
29361
29362    #[test]
29363    fn test_decommission_start_pool_state_reports_completed_pool() {
29364        let pool = PoolStatus {
29365            id: 0,
29366            cmd_line: "pool-0".to_string(),
29367            last_update: OffsetDateTime::UNIX_EPOCH,
29368            decommission: Some(PoolDecommissionInfo {
29369                complete: true,
29370                ..Default::default()
29371            }),
29372        };
29373
29374        assert_eq!(decommission_start_pool_state(Some(&pool)), DecommissionStartPoolState::Decommissioned);
29375    }
29376
29377    #[test]
29378    fn test_ensure_decommission_start_keeps_active_pool_rejects_last_active_pool() {
29379        let meta = PoolMeta {
29380            pools: vec![PoolStatus {
29381                id: 0,
29382                cmd_line: "pool-0".to_string(),
29383                last_update: OffsetDateTime::UNIX_EPOCH,
29384                decommission: None,
29385            }],
29386            ..Default::default()
29387        };
29388
29389        let err = ensure_decommission_start_keeps_active_pool(&meta, &[0]).expect_err("last active pool should be rejected");
29390
29391        assert!(err.to_string().contains("at least one active pool must remain"));
29392    }
29393
29394    #[test]
29395    fn decommission_capacity_model_selection_uses_v2_only_with_a_live_fleet_proof() {
29396        let meta = PoolMeta::default();
29397
29398        assert_eq!(
29399            select_decommission_capacity_model(&meta, false).expect("a proofless empty cohort should remain compatible"),
29400            DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION
29401        );
29402        assert_eq!(
29403            select_decommission_capacity_model(&meta, true).expect("an all-v4 proof should authorize the target fence"),
29404            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
29405        );
29406    }
29407
29408    #[test]
29409    fn decommission_capacity_model_selection_keeps_an_active_v1_cohort_sticky() {
29410        let meta = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION]);
29411
29412        assert_eq!(
29413            select_decommission_capacity_model(&meta, true).expect("a fleet upgrade must not change an active v1 lock model"),
29414            DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION
29415        );
29416    }
29417
29418    #[test]
29419    fn decommission_capacity_model_selection_never_downgrades_an_active_v2_cohort() {
29420        let meta = decommission_test_active_model_meta(&[DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION]);
29421
29422        assert_eq!(
29423            select_decommission_capacity_model(&meta, true).expect("a live proof should admit another v2 reservation"),
29424            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION
29425        );
29426        let err = select_decommission_capacity_model(&meta, false)
29427            .expect_err("proof loss must block new admission instead of falling back to the global lock model");
29428        assert!(err.to_string().contains("active per-target capacity cohort"));
29429    }
29430
29431    #[test]
29432    fn decommission_capacity_model_selection_rejects_a_mixed_active_cohort() {
29433        let meta = decommission_test_active_model_meta(&[
29434            DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION,
29435            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
29436        ]);
29437
29438        let err = select_decommission_capacity_model(&meta, true)
29439            .expect_err("mixed global and per-target lock models cannot be made safe by a fleet proof");
29440        assert!(err.to_string().contains("mixed lock models"));
29441    }
29442
29443    #[test]
29444    fn test_ensure_decommission_start_target_capacity_allows_sufficient_free_space() {
29445        let meta = PoolMeta {
29446            version: POOL_META_VERSION,
29447            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
29448            ..Default::default()
29449        };
29450        let capacity_infos = vec![
29451            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 100, 700, 600),
29452            DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 1_600, 2_000, 400),
29453        ];
29454
29455        assert!(ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos, true).is_ok());
29456    }
29457
29458    #[test]
29459    fn test_ensure_decommission_start_target_capacity_rejects_insufficient_free_space() {
29460        let meta = PoolMeta {
29461            version: POOL_META_VERSION,
29462            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
29463            ..Default::default()
29464        };
29465        let capacity_infos = vec![
29466            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 100, 700, 600),
29467            DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 1_599, 2_000, 401),
29468        ];
29469
29470        let err = ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos, true)
29471            .expect_err("target physical free capacity below the modeled peak should be rejected");
29472
29473        assert!(err.to_string().contains("insufficient reserved physical target capacity"));
29474        assert!(err.to_string().contains("requires 1600 bytes, but 1599 bytes are available"));
29475    }
29476
29477    #[test]
29478    fn test_ensure_decommission_start_target_capacity_ignores_non_active_target_pool() {
29479        let meta = PoolMeta {
29480            version: POOL_META_VERSION,
29481            pools: vec![
29482                decommission_test_pool_status(0, None),
29483                decommission_test_pool_status(
29484                    1,
29485                    Some(PoolDecommissionInfo {
29486                        complete: true,
29487                        ..Default::default()
29488                    }),
29489                ),
29490                decommission_test_pool_status(2, None),
29491            ],
29492            ..Default::default()
29493        };
29494        let capacity_infos = vec![
29495            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 100, 700, 600),
29496            DecommissionPoolCapacityInfo::for_test(1, DecommissionErasureLayout { data: 4, parity: 4 }, 10_000, 10_000, 0),
29497            DecommissionPoolCapacityInfo::for_test(2, DecommissionErasureLayout { data: 4, parity: 4 }, 1_599, 2_000, 401),
29498        ];
29499
29500        let err = ensure_decommission_start_target_capacity(&meta, &[0], &capacity_infos, true)
29501            .expect_err("completed pools must not contribute target free capacity");
29502
29503        assert!(err.to_string().contains("requires 1600 bytes, but 1599 bytes are available"));
29504    }
29505
29506    #[test]
29507    fn decommission_capacity_model_converts_different_source_and_target_parity() {
29508        let now = OffsetDateTime::UNIX_EPOCH;
29509        let high_target_parity = build_decommission_capacity_reservation(
29510            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 2 }, 0, 600, 600),
29511            DecommissionErasureLayout { data: 4, parity: 4 },
29512            uuid::Uuid::new_v4(),
29513            1,
29514            now,
29515        )
29516        .expect("the source and target layouts should be valid");
29517        assert_eq!(high_target_parity.source_data_equivalent_bytes, 400);
29518        assert_eq!(high_target_parity.predicted_physical_bytes, 800);
29519        assert_eq!(high_target_parity.temporary_physical_bytes, 800);
29520        assert_eq!(high_target_parity.peak_physical_bytes, 1_600);
29521
29522        let low_target_parity = build_decommission_capacity_reservation(
29523            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 4, parity: 4 }, 0, 800, 800),
29524            DecommissionErasureLayout { data: 4, parity: 2 },
29525            uuid::Uuid::new_v4(),
29526            2,
29527            now,
29528        )
29529        .expect("the inverse source and target layouts should be valid");
29530        assert_eq!(low_target_parity.source_data_equivalent_bytes, 400);
29531        assert_eq!(low_target_parity.predicted_physical_bytes, 600);
29532        assert_eq!(low_target_parity.peak_physical_bytes, 1_200);
29533    }
29534
29535    #[test]
29536    fn decommission_batch_persists_one_replayable_operation_identity_and_generation() {
29537        let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(1);
29538        let operation_id = uuid::Uuid::new_v4();
29539        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
29540        let capacity_infos = vec![
29541            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 10, 10),
29542            DecommissionPoolCapacityInfo::for_test(1, layout, 0, 20, 20),
29543            DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
29544        ];
29545        let mut meta = PoolMeta {
29546            version: POOL_META_VERSION,
29547            pools: vec![
29548                decommission_test_pool_status(0, None),
29549                decommission_test_pool_status(1, None),
29550                decommission_test_pool_status(2, None),
29551            ],
29552            ..Default::default()
29553        };
29554        meta.decommission(0, capacity_infos[0].space).unwrap();
29555        meta.queue_decommission(1, capacity_infos[1].space).unwrap();
29556
29557        reserve_decommission_start_target_capacity(
29558            &mut meta,
29559            &[0, 1],
29560            &capacity_infos,
29561            operation_id,
29562            17,
29563            now,
29564            DECOMMISSION_CAPACITY_MODEL_VERSION,
29565        )
29566        .expect("the batch reservation should fit exactly");
29567        for idx in [0, 1] {
29568            let reservation = meta.pools[idx]
29569                .decommission
29570                .as_ref()
29571                .and_then(|info| info.capacity_reservation.as_ref())
29572                .expect("each source pool should carry the batch reservation identity");
29573            assert_eq!(reservation.operation_id, operation_id);
29574            assert_eq!(reservation.generation, 17);
29575        }
29576    }
29577
29578    #[test]
29579    fn decommission_capacity_reservation_grows_after_stale_empty_snapshot() {
29580        let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(1);
29581        let layout = DecommissionErasureLayout { data: 3, parity: 1 };
29582        let capacity_infos = vec![
29583            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 0, 0),
29584            DecommissionPoolCapacityInfo::for_test(1, layout, 64, 64, 0),
29585        ];
29586        let mut meta = PoolMeta {
29587            version: POOL_META_VERSION,
29588            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
29589            ..Default::default()
29590        };
29591        meta.decommission(0, capacity_infos[0].space).unwrap();
29592        reserve_decommission_start_target_capacity(
29593            &mut meta,
29594            &[0],
29595            &capacity_infos,
29596            uuid::Uuid::new_v4(),
29597            1,
29598            now,
29599            DECOMMISSION_CAPACITY_MODEL_VERSION,
29600        )
29601        .expect("an empty scanner snapshot should still start decommission");
29602        let grew = grow_decommission_target_reservation(&mut meta, &capacity_infos, 0, 1, 1, uuid::Uuid::new_v4(), now)
29603            .expect("the reservation should grow against current target capacity");
29604        assert!(grew);
29605        let reservation = meta.pools[0]
29606            .decommission
29607            .as_ref()
29608            .and_then(|info| info.capacity_reservation.as_ref())
29609            .unwrap();
29610        assert_eq!(reservation.source_data_equivalent_bytes, 2);
29611        assert_eq!(reservation.predicted_physical_bytes, 3);
29612        assert_eq!(reservation.targets.len(), 1);
29613        assert_eq!(reservation.targets[0].reserved_physical_bytes, 6);
29614    }
29615
29616    #[test]
29617    fn decommission_capacity_model_reserves_versions_delete_markers_and_temporary_copies_from_physical_usage() {
29618        let reservation = build_decommission_capacity_reservation(
29619            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 2, parity: 2 }, 0, 1_024, 1_024),
29620            DecommissionErasureLayout { data: 2, parity: 2 },
29621            uuid::Uuid::new_v4(),
29622            1,
29623            OffsetDateTime::UNIX_EPOCH,
29624        )
29625        .expect("physical source usage should produce a reservation");
29626
29627        assert_eq!(reservation.source_physical_bytes, 1_024);
29628        assert_eq!(reservation.predicted_physical_bytes, 1_024);
29629        assert_eq!(reservation.temporary_copies, 1);
29630        assert_eq!(reservation.peak_physical_bytes, 2_048);
29631    }
29632
29633    #[test]
29634    fn decommission_runtime_capacity_tracks_own_target_consumption_but_rejects_external_drop() {
29635        let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
29636        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
29637        let initial = vec![
29638            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
29639            DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
29640        ];
29641        let mut meta = PoolMeta {
29642            version: POOL_META_VERSION,
29643            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
29644            ..Default::default()
29645        };
29646        meta.decommission(0, initial[0].space).unwrap();
29647        reserve_decommission_start_target_capacity(
29648            &mut meta,
29649            &[0],
29650            &initial,
29651            uuid::Uuid::new_v4(),
29652            1,
29653            now,
29654            DECOMMISSION_CAPACITY_MODEL_VERSION,
29655        )
29656        .expect("the initial target capacity should fit exactly");
29657
29658        let mutation_id = uuid::Uuid::from_u128(1);
29659        reserve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id, now + Duration::seconds(1))
29660            .expect("the target intent should be durable before its write");
29661        let own_consumption = vec![initial[0], DecommissionPoolCapacityInfo::for_test(1, layout, 50, 60, 10)];
29662        ensure_decommission_capacity_reservations_available(&meta, &own_consumption, "restart")
29663            .expect("a persisted target intent must recognize its own write after restart");
29664        resolve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id)
29665            .expect("the persisted target intent should resolve at commit");
29666        record_decommission_target_consumption(
29667            &mut meta,
29668            0,
29669            1,
29670            DecommissionTargetConsumption {
29671                committed_data_bytes: 10,
29672                target_physical_bytes: 10,
29673                observed_physical_bytes: 10,
29674            },
29675            mutation_id,
29676            now + Duration::seconds(1),
29677        )
29678        .expect("the operation's own target consumption should be persisted");
29679        ensure_decommission_capacity_reservations_available(&meta, &own_consumption, "migration")
29680            .expect("the operation's own committed target bytes must not look like external capacity loss");
29681
29682        let dropped = vec![initial[0], DecommissionPoolCapacityInfo::for_test(1, layout, 39, 60, 21)];
29683        let err = ensure_decommission_capacity_reservations_available(&meta, &dropped, "migration")
29684            .expect_err("runtime migration must stop after a sudden competing write consumes reserved capacity");
29685        assert!(
29686            err.to_string()
29687                .contains("requires 40 physical bytes, but only 39 bytes remain")
29688        );
29689        assert!(is_decommission_capacity_blocked_error(&err));
29690    }
29691
29692    #[test]
29693    fn decommission_target_capacity_reuses_only_matching_pending_intent() {
29694        let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
29695        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
29696        let initial = vec![
29697            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
29698            DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
29699        ];
29700        let mut meta = PoolMeta {
29701            version: POOL_META_VERSION,
29702            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
29703            ..Default::default()
29704        };
29705        meta.decommission(0, initial[0].space).unwrap();
29706        reserve_decommission_start_target_capacity(
29707            &mut meta,
29708            &[0],
29709            &initial,
29710            uuid::Uuid::new_v4(),
29711            1,
29712            now,
29713            DECOMMISSION_CAPACITY_MODEL_VERSION,
29714        )
29715        .expect("the initial target capacity should fit exactly");
29716
29717        let first_mutation_id = uuid::Uuid::from_u128(1);
29718        let second_mutation_id = uuid::Uuid::from_u128(2);
29719        reserve_decommission_target_pending(&mut meta, 0, 1, 10, first_mutation_id, now + Duration::seconds(1))
29720            .expect("the first mutation should persist its target intent");
29721        assert_eq!(
29722            reserve_decommission_target_pending(&mut meta, 0, 1, 10, first_mutation_id, now + Duration::seconds(2))
29723                .expect("the same mutation should reuse its persisted target intent"),
29724            0
29725        );
29726        let err = reserve_decommission_target_pending(&mut meta, 0, 1, 10, second_mutation_id, now + Duration::seconds(2))
29727            .expect_err("a second mutation must not reuse the first mutation's pending intent");
29728        assert!(is_decommission_capacity_blocked_error(&err));
29729        assert!(err.to_string().contains("unresolved target capacity intent"));
29730
29731        resolve_decommission_target_pending(&mut meta, 0, 1, 10, first_mutation_id)
29732            .expect("the first mutation should finalize its intent");
29733        record_decommission_target_consumption(
29734            &mut meta,
29735            0,
29736            1,
29737            DecommissionTargetConsumption {
29738                committed_data_bytes: 10,
29739                target_physical_bytes: 10,
29740                observed_physical_bytes: 10,
29741            },
29742            first_mutation_id,
29743            now + Duration::seconds(2),
29744        )
29745        .expect("the first mutation's consumption should be durable");
29746        reserve_decommission_target_pending(&mut meta, 0, 1, 10, second_mutation_id, now + Duration::seconds(3))
29747            .expect("the second mutation should retry only after the first intent is finalized");
29748        assert_eq!(
29749            meta.pools[0]
29750                .decommission
29751                .as_ref()
29752                .and_then(|info| info.capacity_reservation.as_ref())
29753                .expect("the reservation should remain present")
29754                .pending_target_physical_bytes,
29755            10
29756        );
29757    }
29758
29759    #[test]
29760    fn decommission_target_capacity_allows_same_mutation_to_grow_its_pending_stage() {
29761        let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
29762        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
29763        let initial = vec![
29764            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
29765            DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
29766        ];
29767        let mut meta = PoolMeta {
29768            version: POOL_META_VERSION,
29769            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
29770            ..Default::default()
29771        };
29772        meta.decommission(0, initial[0].space).unwrap();
29773        reserve_decommission_start_target_capacity(
29774            &mut meta,
29775            &[0],
29776            &initial,
29777            uuid::Uuid::new_v4(),
29778            1,
29779            now,
29780            DECOMMISSION_CAPACITY_MODEL_VERSION,
29781        )
29782        .expect("the initial target capacity should fit exactly");
29783
29784        let mutation_id = uuid::Uuid::from_u128(1);
29785        assert_eq!(
29786            reserve_decommission_target_pending(&mut meta, 0, 1, 1, mutation_id, now + Duration::seconds(1))
29787                .expect("the initial multipart stage should persist its one-byte intent"),
29788            1
29789        );
29790        assert_eq!(
29791            reserve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id, now + Duration::seconds(2))
29792                .expect("the same mutation should grow its intent for the commit stage"),
29793            9
29794        );
29795        assert_eq!(
29796            meta.pools[0]
29797                .decommission
29798                .as_ref()
29799                .and_then(|info| info.capacity_reservation.as_ref())
29800                .expect("the reservation should remain present")
29801                .targets[0]
29802                .pending_physical_bytes,
29803            10
29804        );
29805        assert_eq!(
29806            reserve_decommission_target_pending(&mut meta, 0, 1, 1, mutation_id, now + Duration::seconds(3))
29807                .expect("a retry of an earlier stage must reuse the larger same-mutation intent"),
29808            0
29809        );
29810    }
29811
29812    #[test]
29813    fn decommission_capacity_mutation_identity_survives_lease_nonce_rotation() {
29814        let owner = DecommissionCapacityOwner {
29815            source_pool_index: 0,
29816            operation_id: uuid::Uuid::from_u128(1),
29817            generation: 2,
29818            owner_nonce: uuid::Uuid::from_u128(3),
29819            mutation_id: None,
29820        };
29821        let recovered_owner = DecommissionCapacityOwner {
29822            owner_nonce: uuid::Uuid::from_u128(4),
29823            ..owner
29824        };
29825        let first = decommission_capacity_mutation_id(owner, "bucket", "object", Some("version"), false, None);
29826        let recovered = decommission_capacity_mutation_id(recovered_owner, "bucket", "object", Some("version"), false, None);
29827        assert_eq!(first, recovered, "a lease nonce rotation must not change the mutation identity");
29828    }
29829
29830    #[test]
29831    fn exact_delete_capacity_plan_requires_identity_and_exact_size() {
29832        let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
29833        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
29834        let capacity_infos = vec![
29835            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
29836            DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
29837        ];
29838        let mut meta = PoolMeta {
29839            version: POOL_META_VERSION,
29840            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
29841            ..Default::default()
29842        };
29843        meta.decommission(0, capacity_infos[0].space).unwrap();
29844        reserve_decommission_start_target_capacity(
29845            &mut meta,
29846            &[0],
29847            &capacity_infos,
29848            uuid::Uuid::new_v4(),
29849            1,
29850            now,
29851            DECOMMISSION_CAPACITY_MODEL_VERSION,
29852        )
29853        .expect("the exact-delete test reservation should fit");
29854        let exact = ObjectInfo {
29855            bucket: "bucket".to_string(),
29856            name: "object".to_string(),
29857            version_id: Some(uuid::Uuid::from_u128(7)),
29858            mod_time: Some(now),
29859            size: 10,
29860            ..Default::default()
29861        };
29862        let owner = {
29863            let reservation = meta.pools[0]
29864                .decommission
29865                .as_ref()
29866                .and_then(|info| info.capacity_reservation.as_ref())
29867                .expect("the exact-delete test reservation should exist");
29868            DecommissionCapacityOwner {
29869                source_pool_index: 0,
29870                operation_id: reservation.operation_id,
29871                generation: reservation.generation,
29872                owner_nonce: reservation.owner_nonce,
29873                mutation_id: None,
29874            }
29875        };
29876        let version_id = exact.version_id.map(|version_id| version_id.to_string());
29877        let mutation_id = decommission_capacity_mutation_id(
29878            owner,
29879            &exact.bucket,
29880            &exact.name,
29881            version_id.as_deref(),
29882            exact.delete_marker,
29883            exact.mod_time,
29884        );
29885        reserve_decommission_target_pending(&mut meta, 0, 1, 10, mutation_id, now + Duration::seconds(1))
29886            .expect("the exact-delete test intent should be reserved");
29887
29888        let plan = plan_exact_delete_capacity_reconciliations(&meta, &exact.name, &exact)
29889            .expect("the exact identity should match the pending intent");
29890        assert_eq!(plan.len(), 1);
29891        assert_eq!(plan[0].source_pool_index, 0);
29892        assert_eq!(plan[0].target_pool_index, 1);
29893        assert_eq!(plan[0].expected_data_bytes, 10);
29894        assert_eq!(plan[0].expected_target_physical_bytes, 10);
29895
29896        let mismatched_size = ObjectInfo {
29897            size: 9,
29898            ..exact.clone()
29899        };
29900        let mismatched_size = plan_exact_delete_capacity_reconciliations(&meta, &mismatched_size.name, &mismatched_size)
29901            .expect_err("a different exact size must not consume the pending intent");
29902        assert!(mismatched_size.to_string().contains("does not match the exact object size"));
29903
29904        let directory_exact = ObjectInfo {
29905            name: "directory/".to_string(),
29906            ..exact.clone()
29907        };
29908        let internal_directory = rustfs_utils::path::encode_dir_object(&directory_exact.name);
29909        let internal_directory_mutation_id = decommission_capacity_mutation_id(
29910            owner,
29911            &directory_exact.bucket,
29912            &internal_directory,
29913            version_id.as_deref(),
29914            directory_exact.delete_marker,
29915            directory_exact.mod_time,
29916        );
29917        meta.pools[0]
29918            .decommission
29919            .as_mut()
29920            .and_then(|info| info.capacity_reservation.as_mut())
29921            .expect("the exact-delete test reservation should exist")
29922            .targets[0]
29923            .pending_mutation_id = Some(internal_directory_mutation_id);
29924        let directory_plan = plan_exact_delete_capacity_reconciliations(&meta, &internal_directory, &directory_exact)
29925            .expect("an internally encoded directory intent should match its logical exact object");
29926        assert_eq!(directory_plan[0].mutation_id, internal_directory_mutation_id);
29927
29928        let logical_directory_mutation_id = decommission_capacity_mutation_id(
29929            owner,
29930            &directory_exact.bucket,
29931            &directory_exact.name,
29932            version_id.as_deref(),
29933            directory_exact.delete_marker,
29934            directory_exact.mod_time,
29935        );
29936        meta.pools[0]
29937            .decommission
29938            .as_mut()
29939            .and_then(|info| info.capacity_reservation.as_mut())
29940            .expect("the exact-delete test reservation should exist")
29941            .targets[0]
29942            .pending_mutation_id = Some(logical_directory_mutation_id);
29943        let directory_plan = plan_exact_delete_capacity_reconciliations(&meta, &internal_directory, &directory_exact)
29944            .expect("a logical directory intent should match its internally encoded delete path");
29945        assert_eq!(directory_plan[0].mutation_id, logical_directory_mutation_id);
29946
29947        meta.pools[0]
29948            .decommission
29949            .as_mut()
29950            .and_then(|info| info.capacity_reservation.as_mut())
29951            .expect("the exact-delete test reservation should exist")
29952            .targets[0]
29953            .pending_mutation_id = None;
29954        let unidentified = plan_exact_delete_capacity_reconciliations(&meta, &exact.name, &exact)
29955            .expect_err("an unidentified pending intent must fail closed");
29956        assert!(unidentified.to_string().contains("without an object identity"));
29957
29958        let mut opts = ObjectOptions::default();
29959        let unfenced = ensure_exact_delete_capacity_namespace_fences(&opts, &exact.bucket, &exact.name)
29960            .expect_err("capacity reconciliation must reject a missing object namespace fence");
29961        assert!(unfenced.to_string().contains("requires an object namespace fence"));
29962        opts.ensure_namespace_lock_fence();
29963        ensure_exact_delete_capacity_namespace_fences(&opts, &exact.bucket, &exact.name)
29964            .expect("a live object namespace fence should admit capacity reconciliation");
29965    }
29966
29967    #[test]
29968    fn ordinary_write_admission_shares_a_reserved_target_without_becoming_its_owner() {
29969        let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
29970        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
29971        let capacity_infos = vec![
29972            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
29973            DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
29974        ];
29975        let mut meta = PoolMeta {
29976            version: POOL_META_VERSION,
29977            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
29978            ..Default::default()
29979        };
29980        meta.decommission(0, capacity_infos[0].space).unwrap();
29981        reserve_decommission_start_target_capacity(
29982            &mut meta,
29983            &[0],
29984            &capacity_infos,
29985            uuid::Uuid::new_v4(),
29986            1,
29987            now,
29988            DECOMMISSION_CAPACITY_MODEL_VERSION,
29989        )
29990        .expect("the decommission reservation should fit");
29991
29992        for admission in [
29993            DecommissionCapacityAdmission::Mutation,
29994            DecommissionCapacityAdmission::BatchDelete,
29995            DecommissionCapacityAdmission::ScannerBacklog,
29996        ] {
29997            ensure_external_decommission_target_admission(&meta, 1, admission)
29998                .expect("a healthy target must remain writable while sharing capacity with migration");
29999        }
30000        assert!(matches!(
30001            ensure_external_decommission_target_admission(&meta, 1, DecommissionCapacityAdmission::Heal),
30002            Err(Error::SlowDown)
30003        ));
30004        let rebalance_opts = ObjectOptions {
30005            data_movement: true,
30006            src_pool_idx: 0,
30007            ..Default::default()
30008        };
30009        assert!(
30010            DecommissionCapacityOwner::from_options(&rebalance_opts).is_none(),
30011            "rebalance data movement must remain an external capacity consumer"
30012        );
30013        let reservation = meta.pools[0]
30014            .decommission
30015            .as_ref()
30016            .and_then(|info| info.capacity_reservation.as_ref())
30017            .expect("the active reservation should remain available");
30018        let expected_owner = DecommissionCapacityOwner {
30019            source_pool_index: 0,
30020            operation_id: reservation.operation_id,
30021            generation: reservation.generation,
30022            owner_nonce: reservation.owner_nonce,
30023            mutation_id: None,
30024        };
30025        ensure_decommission_target_owner_admission(&meta, expected_owner, 1, 10, now)
30026            .expect("the reservation owner should consume its own allocation under the shared boundary");
30027        let mut decommission_opts = rebalance_opts;
30028        expected_owner.apply_to(&mut decommission_opts);
30029        assert_eq!(DecommissionCapacityOwner::from_options(&decommission_opts), Some(expected_owner));
30030
30031        meta.pools[0]
30032            .decommission
30033            .as_mut()
30034            .expect("active source")
30035            .capacity_reservation = None;
30036        for admission in [
30037            DecommissionCapacityAdmission::Mutation,
30038            DecommissionCapacityAdmission::BatchDelete,
30039            DecommissionCapacityAdmission::ScannerBacklog,
30040        ] {
30041            assert!(
30042                matches!(ensure_external_decommission_target_admission(&meta, 1, admission), Err(Error::SlowDown)),
30043                "shared capacity must not bypass an active source's missing durable ledger"
30044            );
30045        }
30046    }
30047
30048    #[test]
30049    fn external_decommission_admission_fences_suspended_sources_but_preserves_repair() {
30050        let now = OffsetDateTime::UNIX_EPOCH + Duration::minutes(2);
30051        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
30052        let capacity_infos = vec![
30053            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
30054            DecommissionPoolCapacityInfo::for_test(1, layout, 100, 100, 0),
30055        ];
30056        let mut active = PoolMeta {
30057            version: POOL_META_VERSION,
30058            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
30059            ..Default::default()
30060        };
30061        active
30062            .decommission(0, capacity_infos[0].space)
30063            .expect("start the source admission fixture");
30064        reserve_decommission_start_target_capacity(
30065            &mut active,
30066            &[0],
30067            &capacity_infos,
30068            uuid::Uuid::new_v4(),
30069            1,
30070            now,
30071            DECOMMISSION_CAPACITY_MODEL_VERSION,
30072        )
30073        .expect("the active source must have a valid reservation to isolate its write fence");
30074
30075        for (state, queued, failed, canceled, complete) in [
30076            ("running", false, false, false, false),
30077            ("queued", true, false, false, false),
30078            ("failed", false, true, false, false),
30079            ("canceled", false, false, true, false),
30080            ("completed", false, false, false, true),
30081        ] {
30082            let mut meta = active.clone();
30083            let info = meta.pools[0].decommission.as_mut().expect("the source fixture must exist");
30084            info.queued = queued;
30085            info.failed = failed;
30086            info.canceled = canceled;
30087            info.complete = complete;
30088            if queued || failed || canceled || complete {
30089                info.start_time = None;
30090            }
30091            for admission in [
30092                DecommissionCapacityAdmission::Mutation,
30093                DecommissionCapacityAdmission::BatchDelete,
30094            ] {
30095                assert!(
30096                    matches!(ensure_external_decommission_target_admission(&meta, 0, admission), Err(Error::SlowDown)),
30097                    "{state} source must reject new publication until its decommission metadata is cleared"
30098                );
30099            }
30100            let existing_multipart =
30101                ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::ExistingMultipart);
30102            if active_decommission_source_indices(&meta).contains(&0) {
30103                existing_multipart
30104                    .unwrap_or_else(|err| panic!("{state} source must allow an existing multipart upload to drain: {err}"));
30105            } else {
30106                assert!(
30107                    matches!(existing_multipart, Err(Error::SlowDown)),
30108                    "{state} terminal source must reject an existing multipart publication"
30109                );
30110            }
30111            ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::Heal)
30112                .unwrap_or_else(|err| panic!("{state} source repair must retain its capacity-only admission: {err}"));
30113            let scanner_result =
30114                ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::ScannerBacklog);
30115            assert_eq!(
30116                meta.scanner_pause_backlog_pool_writable(0),
30117                failed || canceled,
30118                "{state} scanner selection"
30119            );
30120            if failed || canceled {
30121                scanner_result.unwrap_or_else(|err| panic!("{state} scanner membership repair must remain writable: {err}"));
30122            } else {
30123                assert!(
30124                    matches!(scanner_result, Err(Error::SlowDown)),
30125                    "{state} scanner publication must reject its source"
30126                );
30127            }
30128            meta.pools[0].decommission = None;
30129            ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::Mutation)
30130                .unwrap_or_else(|err| panic!("cleared {state} source must become writable again: {err}"));
30131            ensure_external_decommission_target_admission(&meta, 0, DecommissionCapacityAdmission::ScannerBacklog)
30132                .unwrap_or_else(|err| panic!("cleared {state} scanner source must rejoin membership: {err}"));
30133        }
30134        assert!(!active.scanner_pause_backlog_pool_writable(active.pools.len()));
30135        assert!(matches!(
30136            ensure_external_decommission_target_admission(
30137                &active,
30138                active.pools.len(),
30139                DecommissionCapacityAdmission::ScannerBacklog
30140            ),
30141            Err(Error::SlowDown)
30142        ));
30143    }
30144
30145    #[test]
30146    fn decommission_physical_capacity_uses_per_set_bottleneck_and_widest_usage() {
30147        let disks = [10_u64, 20, 30, 40]
30148            .into_iter()
30149            .enumerate()
30150            .map(|(disk_index, available_space)| rustfs_madmin::Disk {
30151                endpoint: format!("http://node-{disk_index}"),
30152                drive_path: format!("/disk-{disk_index}"),
30153                state: "ok".to_string(),
30154                total_space: 100,
30155                used_space: 100 - available_space,
30156                available_space,
30157                pool_index: 0,
30158                set_index: 0,
30159                disk_index: disk_index as i32,
30160                ..Default::default()
30161            })
30162            .collect::<Vec<_>>();
30163        let capacity = decommission_physical_pool_capacity(
30164            &disks,
30165            0,
30166            DecommissionErasureLayout { data: 2, parity: 2 },
30167            PoolSpaceInfo {
30168                free: 0,
30169                total: 0,
30170                used: 0,
30171            },
30172            None,
30173        );
30174
30175        assert_eq!(capacity, (400, 40, 360));
30176    }
30177
30178    #[test]
30179    fn decommission_physical_capacity_prefers_rustfs_owned_usage_over_filesystem_usage() {
30180        let disks = [10_u64, 20, 30, 40]
30181            .into_iter()
30182            .enumerate()
30183            .map(|(disk_index, available_space)| rustfs_madmin::Disk {
30184                endpoint: format!("http://node-{disk_index}"),
30185                drive_path: format!("/disk-{disk_index}"),
30186                state: "ok".to_string(),
30187                total_space: 100,
30188                used_space: 100 - available_space,
30189                available_space,
30190                pool_index: 0,
30191                set_index: 0,
30192                disk_index: disk_index as i32,
30193                ..Default::default()
30194            })
30195            .collect::<Vec<_>>();
30196        let owned = HashMap::from([(0, 12)]);
30197        let capacity = decommission_physical_pool_capacity(
30198            &disks,
30199            0,
30200            DecommissionErasureLayout { data: 2, parity: 2 },
30201            PoolSpaceInfo {
30202                free: 0,
30203                total: 0,
30204                used: 0,
30205            },
30206            Some(&owned),
30207        );
30208
30209        assert_eq!(capacity, (400, 40, 12));
30210    }
30211
30212    #[test]
30213    fn decommission_terminal_transitions_release_capacity_reservations() {
30214        let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(1);
30215        let reservation = build_decommission_capacity_reservation(
30216            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 1, parity: 0 }, 0, 50, 50),
30217            DecommissionErasureLayout { data: 1, parity: 0 },
30218            uuid::Uuid::new_v4(),
30219            7,
30220            now,
30221        )
30222        .expect("test reservation should be valid");
30223        let active = PoolMeta {
30224            version: POOL_META_VERSION,
30225            pools: vec![decommission_test_pool_status(
30226                0,
30227                Some(PoolDecommissionInfo {
30228                    start_time: Some(now),
30229                    capacity_reservation: Some(reservation),
30230                    ..Default::default()
30231                }),
30232            )],
30233            ..Default::default()
30234        };
30235
30236        for (reason, transition) in [
30237            (
30238                DECOMMISSION_CAPACITY_RELEASE_CANCELED,
30239                PoolMeta::decommission_cancel as fn(&mut PoolMeta, usize) -> bool,
30240            ),
30241            (DECOMMISSION_CAPACITY_RELEASE_FAILED, PoolMeta::decommission_failed),
30242            (DECOMMISSION_CAPACITY_RELEASE_COMPLETED, PoolMeta::decommission_complete),
30243        ] {
30244            let mut meta = active.clone();
30245            assert!(transition(&mut meta, 0));
30246            let released = meta.pools[0]
30247                .decommission
30248                .as_ref()
30249                .and_then(|info| info.capacity_reservation.as_ref())
30250                .expect("terminal metadata should retain reservation observability");
30251            assert!(released.released_at.is_some());
30252            assert_eq!(released.release_reason.as_deref(), Some(reason));
30253            assert_eq!(
30254                released.operation_id,
30255                active.pools[0]
30256                    .decommission
30257                    .as_ref()
30258                    .unwrap()
30259                    .capacity_reservation
30260                    .as_ref()
30261                    .unwrap()
30262                    .operation_id
30263            );
30264        }
30265    }
30266
30267    #[test]
30268    fn decommission_capacity_target_round_trip_preserves_temporary_mutation_without_pending_intent() {
30269        let mutation_id = uuid::Uuid::new_v4();
30270        let target = DecommissionCapacityTarget {
30271            pool_index: 1,
30272            layout: DecommissionErasureLayout { data: 2, parity: 2 },
30273            physical_total_at_reservation: 200,
30274            physical_free_at_reservation: 200,
30275            reserved_physical_bytes: 200,
30276            consumed_physical_bytes: 0,
30277            observed_physical_bytes: 1,
30278            inflight_physical_bytes: 1,
30279            pending_physical_bytes: 0,
30280            pending_mutation_id: None,
30281            temporary_mutations: vec![DecommissionCapacityTemporaryMutation {
30282                mutation_id,
30283                physical_bytes: 1,
30284            }],
30285        };
30286        let mut encoded = Vec::new();
30287        target
30288            .serialize(&mut Serializer::new(&mut encoded))
30289            .expect("capacity target should serialize");
30290
30291        let restored: DecommissionCapacityTarget =
30292            rmp_serde::from_slice(&encoded).expect("capacity target with a released pending intent should deserialize");
30293
30294        assert_eq!(restored, target);
30295    }
30296
30297    #[test]
30298    fn temporary_cleanup_releases_only_its_exact_scoped_mutation_and_is_idempotent() {
30299        let mutation_id = uuid::Uuid::new_v4();
30300        let foreign_id = uuid::Uuid::new_v4();
30301        let mut meta = decommission_test_cleanup_meta(
30302            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
30303            vec![
30304                DecommissionCapacityTemporaryMutation {
30305                    mutation_id,
30306                    physical_bytes: 10,
30307                },
30308                DecommissionCapacityTemporaryMutation {
30309                    mutation_id: foreign_id,
30310                    physical_bytes: 20,
30311                },
30312            ],
30313            30,
30314            None,
30315        );
30316
30317        assert!(
30318            release_decommission_target_inflight(
30319                &mut meta,
30320                0,
30321                1,
30322                0,
30323                mutation_id,
30324                DecommissionCapacityReleaseProof::confirmed_absence(true),
30325                OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
30326            )
30327            .expect("confirmed absence should release the exact mutation")
30328        );
30329        let reservation = meta.pools[0]
30330            .decommission
30331            .as_ref()
30332            .and_then(|info| info.capacity_reservation.as_ref())
30333            .expect("cleanup reservation should remain present");
30334        assert_eq!(reservation.inflight_target_physical_bytes, 20);
30335        assert_eq!(
30336            reservation.targets[0].temporary_mutations,
30337            vec![DecommissionCapacityTemporaryMutation {
30338                mutation_id: foreign_id,
30339                physical_bytes: 20,
30340            }]
30341        );
30342
30343        let before_replay = reservation.clone();
30344        assert!(
30345            !release_decommission_target_inflight(
30346                &mut meta,
30347                0,
30348                1,
30349                0,
30350                mutation_id,
30351                DecommissionCapacityReleaseProof::confirmed_absence(true),
30352                OffsetDateTime::UNIX_EPOCH + Duration::seconds(2),
30353            )
30354            .expect("repeated confirmed absence should be a metadata no-op")
30355        );
30356        let after_replay = meta.pools[0]
30357            .decommission
30358            .as_ref()
30359            .and_then(|info| info.capacity_reservation.as_ref())
30360            .expect("cleanup reservation should remain present");
30361        assert_eq!(after_replay, &before_replay);
30362    }
30363
30364    #[test]
30365    fn temporary_cleanup_uses_aggregate_fallback_only_for_legacy_unscoped_state() {
30366        let mutation_id = uuid::Uuid::new_v4();
30367        let mut meta = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, Vec::new(), 30, None);
30368
30369        assert!(
30370            release_decommission_target_inflight(
30371                &mut meta,
30372                0,
30373                1,
30374                12,
30375                mutation_id,
30376                DecommissionCapacityReleaseProof::confirmed_absence(true),
30377                OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
30378            )
30379            .expect("legacy unscoped cleanup should use its observed release delta")
30380        );
30381        let reservation = meta.pools[0]
30382            .decommission
30383            .as_ref()
30384            .and_then(|info| info.capacity_reservation.as_ref())
30385            .expect("legacy cleanup reservation should remain present");
30386        assert_eq!(reservation.inflight_target_physical_bytes, 18);
30387        assert_eq!(reservation.targets[0].inflight_physical_bytes, 18);
30388    }
30389
30390    #[test]
30391    fn temporary_cleanup_clears_only_a_matching_pending_identity_after_confirmed_absence() {
30392        let mutation_id = uuid::Uuid::new_v4();
30393        let mut meta = decommission_test_cleanup_meta(
30394            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
30395            Vec::new(),
30396            0,
30397            Some((mutation_id, 15)),
30398        );
30399
30400        assert!(
30401            release_decommission_target_inflight(
30402                &mut meta,
30403                0,
30404                1,
30405                7,
30406                mutation_id,
30407                DecommissionCapacityReleaseProof::confirmed_absence(true),
30408                OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
30409            )
30410            .expect("a confirmed absent upload should clear its matching pending identity")
30411        );
30412        let reservation = meta.pools[0]
30413            .decommission
30414            .as_ref()
30415            .and_then(|info| info.capacity_reservation.as_ref())
30416            .expect("cleanup reservation should remain present");
30417        assert_eq!(reservation.pending_target_physical_bytes, 0);
30418        assert_eq!(reservation.targets[0].pending_mutation_id, None);
30419        assert_eq!(reservation.targets[0].pending_physical_bytes, 0);
30420    }
30421
30422    #[test]
30423    fn temporary_cleanup_preserves_pending_for_an_already_published_target() {
30424        let mutation_id = uuid::Uuid::new_v4();
30425        let mut meta = decommission_test_cleanup_meta(
30426            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
30427            vec![DecommissionCapacityTemporaryMutation {
30428                mutation_id,
30429                physical_bytes: 5,
30430            }],
30431            5,
30432            Some((mutation_id, 15)),
30433        );
30434
30435        assert!(
30436            release_decommission_target_inflight(
30437                &mut meta,
30438                0,
30439                1,
30440                0,
30441                mutation_id,
30442                DecommissionCapacityReleaseProof::confirmed_absence(false),
30443                OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
30444            )
30445            .expect("published-target cleanup should release only its temporary staging state")
30446        );
30447        let reservation = meta.pools[0]
30448            .decommission
30449            .as_ref()
30450            .and_then(|info| info.capacity_reservation.as_ref())
30451            .expect("cleanup reservation should remain present");
30452        assert_eq!(reservation.inflight_target_physical_bytes, 0);
30453        assert_eq!(reservation.pending_target_physical_bytes, 15);
30454        assert_eq!(reservation.targets[0].pending_mutation_id, Some(mutation_id));
30455        assert_eq!(reservation.targets[0].pending_physical_bytes, 15);
30456        assert!(reservation.targets[0].temporary_mutations.is_empty());
30457    }
30458
30459    #[test]
30460    fn published_pending_ignores_a_delayed_release_observation() {
30461        let mutation_id = uuid::Uuid::new_v4();
30462        let mut meta = decommission_test_cleanup_meta(
30463            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
30464            Vec::new(),
30465            0,
30466            Some((mutation_id, 15)),
30467        );
30468        let before = meta.pools[0]
30469            .decommission
30470            .as_ref()
30471            .and_then(|info| info.capacity_reservation.as_ref())
30472            .expect("published-target reservation should remain present")
30473            .clone();
30474
30475        assert!(
30476            !release_decommission_target_inflight(
30477                &mut meta,
30478                0,
30479                1,
30480                7,
30481                mutation_id,
30482                DecommissionCapacityReleaseProof::confirmed_absence(false),
30483                OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
30484            )
30485            .expect("a delayed statfs release must not block published-target reconciliation")
30486        );
30487        let after = meta.pools[0]
30488            .decommission
30489            .as_ref()
30490            .and_then(|info| info.capacity_reservation.as_ref())
30491            .expect("published-target reservation should remain present");
30492        assert_eq!(after, &before);
30493    }
30494
30495    #[test]
30496    fn temporary_cleanup_fails_closed_on_a_foreign_scoped_release_delta() {
30497        let foreign_id = uuid::Uuid::new_v4();
30498        let mut meta = decommission_test_cleanup_meta(
30499            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
30500            vec![DecommissionCapacityTemporaryMutation {
30501                mutation_id: foreign_id,
30502                physical_bytes: 20,
30503            }],
30504            20,
30505            None,
30506        );
30507        let before = meta.pools[0]
30508            .decommission
30509            .as_ref()
30510            .and_then(|info| info.capacity_reservation.as_ref())
30511            .expect("cleanup reservation should remain present")
30512            .clone();
30513
30514        let err = release_decommission_target_inflight(
30515            &mut meta,
30516            0,
30517            1,
30518            5,
30519            uuid::Uuid::new_v4(),
30520            DecommissionCapacityReleaseProof::confirmed_absence(true),
30521            OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
30522        )
30523        .expect_err("an observed release cannot be charged to a foreign scoped mutation");
30524        assert!(err.to_string().contains("cannot be attributed"));
30525        let after = meta.pools[0]
30526            .decommission
30527            .as_ref()
30528            .and_then(|info| info.capacity_reservation.as_ref())
30529            .expect("cleanup reservation should remain present");
30530        assert_eq!(after, &before);
30531    }
30532
30533    #[test]
30534    fn temporary_cleanup_missing_identity_and_nonzero_delta_is_a_metadata_noop() {
30535        let mut meta = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, Vec::new(), 0, None);
30536        let before = meta.pools[0]
30537            .decommission
30538            .as_ref()
30539            .and_then(|info| info.capacity_reservation.as_ref())
30540            .expect("cleanup reservation should remain present")
30541            .clone();
30542
30543        assert!(
30544            !release_decommission_target_inflight(
30545                &mut meta,
30546                0,
30547                1,
30548                7,
30549                uuid::Uuid::new_v4(),
30550                DecommissionCapacityReleaseProof::confirmed_absence(true),
30551                OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
30552            )
30553            .expect("confirmed absence without tracked state should ignore a delayed capacity observation")
30554        );
30555        let after = meta.pools[0]
30556            .decommission
30557            .as_ref()
30558            .and_then(|info| info.capacity_reservation.as_ref())
30559            .expect("cleanup reservation should remain present");
30560        assert_eq!(after, &before);
30561    }
30562
30563    #[test]
30564    fn temporary_cleanup_missing_identity_fails_closed_on_foreign_pending_state() {
30565        let foreign_id = uuid::Uuid::new_v4();
30566        let mut meta = decommission_test_cleanup_meta(
30567            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
30568            Vec::new(),
30569            0,
30570            Some((foreign_id, 11)),
30571        );
30572        let before = meta.pools[0]
30573            .decommission
30574            .as_ref()
30575            .and_then(|info| info.capacity_reservation.as_ref())
30576            .expect("cleanup reservation should remain present")
30577            .clone();
30578
30579        let err = release_decommission_target_inflight(
30580            &mut meta,
30581            0,
30582            1,
30583            7,
30584            uuid::Uuid::new_v4(),
30585            DecommissionCapacityReleaseProof::confirmed_absence(true),
30586            OffsetDateTime::UNIX_EPOCH + Duration::seconds(1),
30587        )
30588        .expect_err("a delayed release observation must not clear a foreign pending mutation");
30589        assert!(err.to_string().contains("cannot be attributed"));
30590        let after = meta.pools[0]
30591            .decommission
30592            .as_ref()
30593            .and_then(|info| info.capacity_reservation.as_ref())
30594            .expect("cleanup reservation should remain present");
30595        assert_eq!(after, &before);
30596    }
30597
30598    #[test]
30599    fn pool_meta_v2_round_trip_preserves_each_supported_capacity_lock_model() {
30600        for model_version in [
30601            DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION,
30602            DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION,
30603        ] {
30604            let meta = decommission_test_cleanup_meta(model_version, Vec::new(), 0, None);
30605            let encoded = meta
30606                .encode_config_data_for_test()
30607                .expect("supported capacity lock model should encode in pool metadata V2");
30608            let mut restored = PoolMeta::default();
30609            restored
30610                .load_from_config_data(encoded)
30611                .expect("supported capacity lock model should decode from pool metadata V2");
30612            assert_eq!(
30613                restored.pools[0]
30614                    .decommission
30615                    .as_ref()
30616                    .and_then(|info| info.capacity_reservation.as_ref())
30617                    .expect("round-tripped reservation should remain present")
30618                    .model_version,
30619                model_version
30620            );
30621        }
30622    }
30623
30624    #[test]
30625    fn pool_meta_v2_decode_rejects_mixed_active_capacity_lock_models() {
30626        let mut mixed = decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_LEGACY_MODEL_VERSION, Vec::new(), 0, None);
30627        let mut target_fence_source =
30628            decommission_test_cleanup_meta(DECOMMISSION_CAPACITY_TARGET_FENCE_MODEL_VERSION, Vec::new(), 0, None)
30629                .pools
30630                .remove(0);
30631        target_fence_source.id = 2;
30632        target_fence_source.cmd_line = "pool-2".to_string();
30633        target_fence_source
30634            .decommission
30635            .as_mut()
30636            .and_then(|info| info.capacity_reservation.as_mut())
30637            .expect("second active reservation should remain present")
30638            .source_pool_index = 2;
30639        mixed.pools.push(target_fence_source);
30640
30641        let encoded = mixed
30642            .encode_config_data_for_test()
30643            .expect("the decode test must be able to construct a mixed persisted payload");
30644        let mut restored = PoolMeta::default();
30645        let err = restored
30646            .load_from_config_data(encoded)
30647            .expect_err("mixed active lock models must fail closed while loading pool metadata");
30648        assert!(err.to_string().contains("mixed lock models"));
30649    }
30650
30651    #[test]
30652    fn decommission_capacity_reservation_recovers_expired_lease_after_restart_round_trip() {
30653        let created_at = OffsetDateTime::UNIX_EPOCH + Duration::hours(1);
30654        let recovered_at = created_at + DECOMMISSION_CAPACITY_RESERVATION_TTL + Duration::seconds(1);
30655        let mut reservation = build_decommission_capacity_reservation(
30656            DecommissionPoolCapacityInfo::for_test(0, DecommissionErasureLayout { data: 2, parity: 2 }, 0, 100, 100),
30657            DecommissionErasureLayout { data: 2, parity: 2 },
30658            uuid::Uuid::new_v4(),
30659            41,
30660            created_at,
30661        )
30662        .expect("test reservation should be valid");
30663        reservation.targets.push(DecommissionCapacityTarget {
30664            pool_index: 1,
30665            layout: DecommissionErasureLayout { data: 2, parity: 2 },
30666            physical_total_at_reservation: 200,
30667            physical_free_at_reservation: 200,
30668            reserved_physical_bytes: 200,
30669            consumed_physical_bytes: 0,
30670            observed_physical_bytes: 0,
30671            inflight_physical_bytes: 0,
30672            pending_physical_bytes: 0,
30673            pending_mutation_id: None,
30674            temporary_mutations: Vec::new(),
30675        });
30676        let operation_id = reservation.operation_id;
30677        let owner_nonce = reservation.owner_nonce;
30678        let mut meta = PoolMeta {
30679            version: POOL_META_VERSION,
30680            pools: vec![decommission_test_pool_status(
30681                0,
30682                Some(PoolDecommissionInfo {
30683                    start_time: Some(created_at),
30684                    capacity_reservation: Some(reservation),
30685                    ..Default::default()
30686                }),
30687            )],
30688            ..Default::default()
30689        };
30690
30691        let encoded = meta
30692            .encode_config_data_for_test()
30693            .expect("capacity reservation should persist in V2");
30694        let mut restored = PoolMeta::default();
30695        restored
30696            .load_from_config_data(encoded)
30697            .expect("capacity reservation should survive a restart round trip");
30698        let info = restored.pools[0]
30699            .decommission
30700            .as_mut()
30701            .expect("active decommission should restore");
30702        let recovered = info.capacity_reservation.as_mut().expect("reservation should remain present");
30703        assert!(renew_decommission_capacity_reservation(recovered, recovered_at, true));
30704        assert_eq!(recovered.operation_id, operation_id);
30705        assert_eq!(recovered.generation, 41);
30706        assert_ne!(recovered.owner_nonce, owner_nonce);
30707        assert_eq!(recovered.recovered_at, Some(recovered_at));
30708        assert_eq!(recovered.expires_at, recovered_at + DECOMMISSION_CAPACITY_RESERVATION_TTL);
30709        assert_eq!(next_decommission_capacity_generation(&restored).unwrap(), 42);
30710
30711        meta.decommission_failed(0);
30712        let terminal = meta.pools[0]
30713            .decommission
30714            .as_ref()
30715            .and_then(|info| info.capacity_reservation.as_ref())
30716            .expect("failed metadata should retain the released reservation");
30717        assert!(!terminal.active());
30718    }
30719
30720    #[test]
30721    fn decommission_restart_reconstructs_a_missing_capacity_reservation_fail_closed() {
30722        let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(2);
30723        let mut meta = PoolMeta {
30724            version: POOL_META_VERSION,
30725            pools: vec![
30726                decommission_test_pool_status(
30727                    0,
30728                    Some(PoolDecommissionInfo {
30729                        start_time: Some(now - Duration::minutes(1)),
30730                        ..Default::default()
30731                    }),
30732                ),
30733                decommission_test_pool_status(1, None),
30734            ],
30735            ..Default::default()
30736        };
30737        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
30738        let capacity_infos = vec![
30739            DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
30740            DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
30741        ];
30742
30743        let recovered_indices = recover_decommission_capacity_reservations(&mut meta, &capacity_infos, now, true)
30744            .expect("restart recovery should rebuild the missing reservation while capacity still fits");
30745        assert_eq!(recovered_indices, vec![0]);
30746        let reservation = meta.pools[0]
30747            .decommission
30748            .as_ref()
30749            .and_then(|info| info.capacity_reservation.as_ref())
30750            .expect("restart recovery should persist a replacement reservation");
30751        assert_eq!(reservation.peak_physical_bytes, 60);
30752        assert_eq!(reservation.recovered_at, Some(now));
30753        assert_eq!(reservation.generation, 1);
30754    }
30755
30756    #[test]
30757    fn test_ensure_decommission_start_pool_states_rejects_blocked_pool() {
30758        let meta = PoolMeta {
30759            pools: vec![
30760                PoolStatus {
30761                    id: 0,
30762                    cmd_line: "pool-0".to_string(),
30763                    last_update: OffsetDateTime::UNIX_EPOCH,
30764                    decommission: Some(PoolDecommissionInfo {
30765                        failed: true,
30766                        ..Default::default()
30767                    }),
30768                },
30769                PoolStatus {
30770                    id: 1,
30771                    cmd_line: "pool-1".to_string(),
30772                    last_update: OffsetDateTime::UNIX_EPOCH,
30773                    decommission: None,
30774                },
30775            ],
30776            ..Default::default()
30777        };
30778
30779        let err = ensure_decommission_start_pool_states(&meta, &[0]).expect_err("blocked pool should be rejected");
30780
30781        assert!(err.to_string().contains("target pool decommission is blocked"));
30782    }
30783
30784    #[test]
30785    fn test_ensure_decommission_start_pool_states_allows_active_pool_with_remaining_active_pool() {
30786        let meta = PoolMeta {
30787            pools: vec![
30788                PoolStatus {
30789                    id: 0,
30790                    cmd_line: "pool-0".to_string(),
30791                    last_update: OffsetDateTime::UNIX_EPOCH,
30792                    decommission: None,
30793                },
30794                PoolStatus {
30795                    id: 1,
30796                    cmd_line: "pool-1".to_string(),
30797                    last_update: OffsetDateTime::UNIX_EPOCH,
30798                    decommission: None,
30799                },
30800            ],
30801            ..Default::default()
30802        };
30803
30804        assert!(ensure_decommission_start_pool_states(&meta, &[0]).is_ok());
30805    }
30806
30807    #[test]
30808    fn test_ensure_decommission_start_pool_states_allows_retryable_pool_with_active_peer() {
30809        let meta = PoolMeta {
30810            pools: vec![
30811                PoolStatus {
30812                    id: 0,
30813                    cmd_line: "pool-0".to_string(),
30814                    last_update: OffsetDateTime::UNIX_EPOCH,
30815                    decommission: Some(PoolDecommissionInfo {
30816                        failed: true,
30817                        unresolved_entries: vec![DecommissionUnresolvedEntry {
30818                            bucket: "bucket-a".to_string(),
30819                            object: "object-a".to_string(),
30820                            pool_index: 0,
30821                            set_index: 0,
30822                            source_generation: OffsetDateTime::UNIX_EPOCH,
30823                            candidate_count: 1,
30824                            disk_error_count: 1,
30825                            observed_at: OffsetDateTime::UNIX_EPOCH,
30826                            reason: "metadata_resolution_failed".to_string(),
30827                        }],
30828                        ..Default::default()
30829                    }),
30830                },
30831                PoolStatus {
30832                    id: 1,
30833                    cmd_line: "pool-1".to_string(),
30834                    last_update: OffsetDateTime::UNIX_EPOCH,
30835                    decommission: None,
30836                },
30837            ],
30838            ..Default::default()
30839        };
30840
30841        assert!(ensure_decommission_start_pool_states(&meta, &[0]).is_ok());
30842    }
30843
30844    #[test]
30845    fn test_ensure_valid_decommission_pool_index_accepts_in_range_index() {
30846        assert!(ensure_valid_decommission_pool_index(4, 3).is_ok());
30847    }
30848
30849    #[test]
30850    fn test_ensure_valid_decommission_pool_index_rejects_out_of_range_index() {
30851        let err = ensure_valid_decommission_pool_index(2, 2).expect_err("out-of-range index should fail");
30852        assert!(err.to_string().contains("invalid decommission pool index 2 for 2 pools"));
30853    }
30854
30855    #[test]
30856    fn test_ensure_valid_decommission_pool_index_rejects_when_pool_count_zero() {
30857        let err = ensure_valid_decommission_pool_index(0, 0).expect_err("empty pool list should reject all indices");
30858        assert!(err.to_string().contains("invalid decommission pool index 0 for 0 pools"));
30859    }
30860
30861    #[test]
30862    fn test_classify_decommission_terminal_state_completed_when_no_failures() {
30863        assert_eq!(classify_decommission_terminal_state(false), DecommissionTerminalState::Completed);
30864    }
30865
30866    #[test]
30867    fn test_classify_decommission_terminal_state_failed_when_failures_present() {
30868        assert_eq!(classify_decommission_terminal_state(true), DecommissionTerminalState::Failed);
30869    }
30870
30871    #[test]
30872    fn test_should_preserve_decommission_canceled_state_when_meta_canceled() {
30873        assert!(should_preserve_decommission_canceled_state(true, false));
30874    }
30875
30876    #[test]
30877    fn test_should_preserve_decommission_canceled_state_when_signal_canceled() {
30878        assert!(!should_preserve_decommission_canceled_state(false, true));
30879    }
30880
30881    #[test]
30882    fn test_should_preserve_decommission_canceled_state_when_not_canceled() {
30883        assert!(!should_preserve_decommission_canceled_state(false, false));
30884    }
30885
30886    #[test]
30887    fn test_should_continue_decommission_queue_requires_clean_completion() {
30888        let meta = PoolMeta {
30889            pools: vec![
30890                PoolStatus {
30891                    id: 0,
30892                    cmd_line: "pool-0".to_string(),
30893                    last_update: OffsetDateTime::UNIX_EPOCH,
30894                    decommission: Some(PoolDecommissionInfo {
30895                        complete: true,
30896                        ..Default::default()
30897                    }),
30898                },
30899                PoolStatus {
30900                    id: 1,
30901                    cmd_line: "pool-1".to_string(),
30902                    last_update: OffsetDateTime::UNIX_EPOCH,
30903                    decommission: Some(PoolDecommissionInfo::default()),
30904                },
30905                PoolStatus {
30906                    id: 2,
30907                    cmd_line: "pool-2".to_string(),
30908                    last_update: OffsetDateTime::UNIX_EPOCH,
30909                    decommission: Some(PoolDecommissionInfo {
30910                        failed: true,
30911                        ..Default::default()
30912                    }),
30913                },
30914                PoolStatus {
30915                    id: 3,
30916                    cmd_line: "pool-3".to_string(),
30917                    last_update: OffsetDateTime::UNIX_EPOCH,
30918                    decommission: Some(PoolDecommissionInfo {
30919                        canceled: true,
30920                        ..Default::default()
30921                    }),
30922                },
30923                PoolStatus {
30924                    id: 4,
30925                    cmd_line: "pool-4".to_string(),
30926                    last_update: OffsetDateTime::UNIX_EPOCH,
30927                    decommission: None,
30928                },
30929            ],
30930            ..Default::default()
30931        };
30932
30933        assert!(should_continue_decommission_queue(&meta, 0));
30934        assert!(!should_continue_decommission_queue(&meta, 1));
30935        assert!(!should_continue_decommission_queue(&meta, 2));
30936        assert!(!should_continue_decommission_queue(&meta, 3));
30937        assert!(!should_continue_decommission_queue(&meta, 4));
30938        assert!(!should_continue_decommission_queue(&meta, 5));
30939    }
30940
30941    #[test]
30942    fn test_decommission_cancel_signal_result_returns_err_when_canceled() {
30943        let err = decommission_cancel_signal_result(true).expect_err("canceled signal should return operation-canceled");
30944        assert!(matches!(err, Error::OperationCanceled));
30945    }
30946
30947    #[test]
30948    fn test_decommission_cancel_signal_result_returns_ok_when_not_canceled() {
30949        assert!(decommission_cancel_signal_result(false).is_ok());
30950    }
30951
30952    #[test]
30953    fn test_is_decommission_cancel_requested_accepts_signal_or_metadata() {
30954        let pool = PoolStatus {
30955            id: 0,
30956            cmd_line: "pool-0".to_string(),
30957            last_update: OffsetDateTime::UNIX_EPOCH,
30958            decommission: Some(PoolDecommissionInfo {
30959                canceled: true,
30960                ..Default::default()
30961            }),
30962        };
30963
30964        assert!(is_decommission_cancel_requested(false, Some(&pool)));
30965        assert!(is_decommission_cancel_requested(true, None));
30966    }
30967
30968    #[test]
30969    fn test_is_decommission_cancel_requested_rejects_active_without_signal() {
30970        let pool = PoolStatus {
30971            id: 0,
30972            cmd_line: "pool-0".to_string(),
30973            last_update: OffsetDateTime::UNIX_EPOCH,
30974            decommission: Some(PoolDecommissionInfo::default()),
30975        };
30976
30977        assert!(!is_decommission_cancel_requested(false, Some(&pool)));
30978        assert!(!is_decommission_cancel_requested(false, None));
30979    }
30980
30981    #[test]
30982    fn test_skip_canceled_decommission_routine_only_for_terminal_canceled_state() {
30983        let canceled = PoolStatus {
30984            id: 0,
30985            cmd_line: "pool-0".to_string(),
30986            last_update: OffsetDateTime::UNIX_EPOCH,
30987            decommission: Some(PoolDecommissionInfo {
30988                canceled: true,
30989                ..Default::default()
30990            }),
30991        };
30992        let active = PoolStatus {
30993            id: 1,
30994            cmd_line: "pool-1".to_string(),
30995            last_update: OffsetDateTime::UNIX_EPOCH,
30996            decommission: Some(PoolDecommissionInfo::default()),
30997        };
30998
30999        assert!(should_skip_canceled_decommission_routine(true, Some(&canceled)));
31000        assert!(!should_skip_canceled_decommission_routine(false, Some(&canceled)));
31001        assert!(!should_skip_canceled_decommission_routine(true, Some(&active)));
31002        assert!(!should_skip_canceled_decommission_routine(true, None));
31003    }
31004
31005    #[test]
31006    fn test_ensure_decommission_cancel_allowed_rejects_missing_pool() {
31007        let err = ensure_decommission_cancel_allowed(false, false, false).expect_err("missing pool should be invalid");
31008        assert!(
31009            err.to_string()
31010                .contains("failed to cancel decommission: target pool was not found")
31011        );
31012    }
31013
31014    #[test]
31015    fn test_should_reject_decommission_cancel_as_terminal_true_when_completed() {
31016        assert!(should_reject_decommission_cancel_as_terminal(true, false));
31017    }
31018
31019    #[test]
31020    fn test_should_reject_decommission_cancel_as_terminal_true_when_failed() {
31021        assert!(should_reject_decommission_cancel_as_terminal(false, true));
31022    }
31023
31024    #[test]
31025    fn test_should_reject_decommission_cancel_as_terminal_false_when_active_or_canceled() {
31026        assert!(!should_reject_decommission_cancel_as_terminal(false, false));
31027    }
31028
31029    #[test]
31030    fn test_should_retry_decommission_cancel_reload_when_changed_or_already_canceled() {
31031        assert!(should_retry_decommission_cancel_reload(true, false));
31032        assert!(should_retry_decommission_cancel_reload(false, true));
31033        assert!(!should_retry_decommission_cancel_reload(false, false));
31034    }
31035
31036    #[test]
31037    fn test_ensure_decommission_cancel_allowed_rejects_not_started() {
31038        let err =
31039            ensure_decommission_cancel_allowed(true, false, false).expect_err("not-started decommission should be rejected");
31040        assert!(matches!(err, Error::DecommissionNotStarted));
31041    }
31042
31043    #[test]
31044    fn test_ensure_decommission_cancel_allowed_rejects_terminal() {
31045        let err = ensure_decommission_cancel_allowed(true, true, true).expect_err("terminal decommission should be rejected");
31046        assert!(matches!(err, Error::DecommissionNotStarted));
31047    }
31048
31049    #[test]
31050    fn test_ensure_decommission_cancel_allowed_allows_active() {
31051        assert!(ensure_decommission_cancel_allowed(true, true, false).is_ok());
31052    }
31053
31054    #[test]
31055    fn test_ensure_decommission_clear_allowed_allows_failed_or_canceled() {
31056        assert!(ensure_decommission_clear_allowed(true, true, false, true, false, 0).is_ok());
31057        assert!(ensure_decommission_clear_allowed(true, true, false, false, true, 0).is_ok());
31058    }
31059
31060    #[test]
31061    fn test_ensure_decommission_clear_allowed_rejects_active_or_completed() {
31062        let active = ensure_decommission_clear_allowed(true, true, false, false, false, 0)
31063            .expect_err("active decommission should not be clearable");
31064        assert!(matches!(active, Error::DecommissionAlreadyRunning));
31065
31066        let complete = ensure_decommission_clear_allowed(true, true, true, false, false, 0)
31067            .expect_err("completed decommission should not be clearable");
31068        assert!(matches!(complete, Error::DecommissionNotStarted));
31069    }
31070
31071    #[test]
31072    fn test_ensure_decommission_clear_allowed_rejects_unresolved_entries() {
31073        let err = ensure_decommission_clear_allowed(true, true, false, true, false, 1)
31074            .expect_err("unresolved entries must survive until a retry reconciles them");
31075
31076        assert!(err.to_string().contains("must be reconciled by retrying decommission"));
31077    }
31078
31079    #[test]
31080    fn test_pool_meta_clear_decommission_restores_failed_or_canceled_pool() {
31081        for decommission in [
31082            PoolDecommissionInfo {
31083                failed: true,
31084                ..Default::default()
31085            },
31086            PoolDecommissionInfo {
31087                canceled: true,
31088                ..Default::default()
31089            },
31090        ] {
31091            let mut meta = PoolMeta {
31092                pools: vec![PoolStatus {
31093                    id: 0,
31094                    cmd_line: "pool-0".to_string(),
31095                    last_update: OffsetDateTime::UNIX_EPOCH,
31096                    decommission: Some(decommission),
31097                }],
31098                ..Default::default()
31099            };
31100
31101            assert!(meta.is_suspended(0));
31102            assert!(meta.clear_decommission(0).expect("terminal decommission should clear"));
31103            assert!(
31104                meta.pools[0]
31105                    .decommission
31106                    .as_ref()
31107                    .is_some_and(|info| !info.has_decommission_state())
31108            );
31109            assert!(!meta.is_suspended(0));
31110        }
31111    }
31112
31113    #[test]
31114    fn test_pool_meta_clear_decommission_preserves_unresolved_entries_for_retry() {
31115        let generation = OffsetDateTime::UNIX_EPOCH;
31116        let unresolved_entry = DecommissionUnresolvedEntry {
31117            bucket: "bucket-a".to_string(),
31118            object: "object-a".to_string(),
31119            pool_index: 0,
31120            set_index: 0,
31121            source_generation: generation,
31122            candidate_count: 1,
31123            disk_error_count: 1,
31124            observed_at: generation,
31125            reason: "metadata_resolution_failed".to_string(),
31126        };
31127        let mut meta = PoolMeta {
31128            pools: vec![PoolStatus {
31129                id: 0,
31130                cmd_line: "pool-0".to_string(),
31131                last_update: generation,
31132                decommission: Some(PoolDecommissionInfo {
31133                    failed: true,
31134                    unresolved_entries: vec![unresolved_entry.clone()],
31135                    ..Default::default()
31136                }),
31137            }],
31138            ..Default::default()
31139        };
31140
31141        let err = meta
31142            .clear_decommission(0)
31143            .expect_err("clear must not discard the unresolved-entry recovery ledger");
31144
31145        assert!(err.to_string().contains("must be reconciled by retrying decommission"));
31146        assert_eq!(
31147            meta.pools[0]
31148                .decommission
31149                .as_ref()
31150                .expect("failed state should remain retryable")
31151                .unresolved_entries,
31152            vec![unresolved_entry]
31153        );
31154    }
31155
31156    #[test]
31157    fn test_pool_meta_clear_decommission_rejects_active_or_completed_pool() {
31158        for decommission in [
31159            PoolDecommissionInfo::default(),
31160            PoolDecommissionInfo {
31161                complete: true,
31162                ..Default::default()
31163            },
31164        ] {
31165            let mut meta = PoolMeta {
31166                pools: vec![PoolStatus {
31167                    id: 0,
31168                    cmd_line: "pool-0".to_string(),
31169                    last_update: OffsetDateTime::UNIX_EPOCH,
31170                    decommission: Some(decommission),
31171                }],
31172                ..Default::default()
31173            };
31174
31175            assert!(meta.clear_decommission(0).is_err());
31176            assert!(meta.pools[0].decommission.is_some());
31177        }
31178    }
31179
31180    #[test]
31181    fn test_contextualized_decommission_terminal_operation_supported_rejects_single_pool() {
31182        let err = ensure_decommission_terminal_operation_supported(true, "complete decommission")
31183            .expect_err("single-pool decommission terminal operations should be rejected");
31184        assert!(
31185            err.to_string()
31186                .contains("failed to complete decommission: single pool deployments do not support decommission")
31187        );
31188    }
31189
31190    #[test]
31191    fn test_contextualized_decommission_terminal_operation_supported_allows_multi_pool() {
31192        assert!(ensure_decommission_terminal_operation_supported(false, "mark decommission failed").is_ok());
31193    }
31194
31195    #[test]
31196    fn test_contextualized_decommission_start_request_rejects_empty_indices() {
31197        let err = validate_start_decommission_request(&[], false).expect_err("empty decommission target list should be rejected");
31198        assert!(
31199            err.to_string()
31200                .contains("failed to start decommission: no target pools were provided")
31201        );
31202    }
31203
31204    #[test]
31205    fn test_contextualized_decommission_start_request_rejects_single_pool() {
31206        let err = validate_start_decommission_request(&[0], true)
31207            .expect_err("single-pool deployments should reject decommission start");
31208        assert!(
31209            err.to_string()
31210                .contains("failed to start decommission: single pool deployments do not support decommission")
31211        );
31212    }
31213
31214    #[test]
31215    fn test_contextualized_decommission_start_request_allows_multiple_target_pools() {
31216        assert!(validate_start_decommission_request(&[0, 1], false).is_ok());
31217    }
31218
31219    #[test]
31220    fn test_contextualized_decommission_start_request_allows_one_target_pool() {
31221        assert!(validate_start_decommission_request(&[0], false).is_ok());
31222    }
31223
31224    #[test]
31225    fn test_decommission_retry_preserves_and_rebinds_unresolved_entries() {
31226        let previous_generation = OffsetDateTime::UNIX_EPOCH;
31227        let unresolved_entry = DecommissionUnresolvedEntry {
31228            bucket: "bucket-a".to_string(),
31229            object: "object-a".to_string(),
31230            pool_index: 0,
31231            set_index: 0,
31232            source_generation: previous_generation,
31233            candidate_count: 1,
31234            disk_error_count: 1,
31235            observed_at: previous_generation,
31236            reason: "metadata_resolution_failed".to_string(),
31237        };
31238        let mut meta = PoolMeta {
31239            pools: vec![PoolStatus {
31240                id: 0,
31241                cmd_line: "pool-0".to_string(),
31242                last_update: previous_generation,
31243                decommission: Some(PoolDecommissionInfo {
31244                    failed: true,
31245                    decommissioned_buckets: vec!["bucket-a".to_string()],
31246                    items_decommissioned: 7,
31247                    bytes_done: 1024,
31248                    unresolved_entries: vec![unresolved_entry],
31249                    ..Default::default()
31250                }),
31251            }],
31252            ..Default::default()
31253        };
31254
31255        meta.decommission(
31256            0,
31257            PoolSpaceInfo {
31258                total: 200,
31259                free: 50,
31260                used: 150,
31261            },
31262        )
31263        .expect("failed decommission with unresolved entries should retry atomically");
31264
31265        let info = meta.pools[0]
31266            .decommission
31267            .as_ref()
31268            .expect("retried decommission metadata should exist");
31269        let next_generation = info.start_time.expect("retry should assign a new generation");
31270        assert!(next_generation > previous_generation);
31271        assert_eq!(info.decommissioned_buckets, vec!["bucket-a".to_string()]);
31272        assert_eq!(info.items_decommissioned, 7);
31273        assert_eq!(info.bytes_done, 1024);
31274        assert_eq!(info.unresolved_entries.len(), 1);
31275        assert_eq!(info.unresolved_entries[0].source_generation, next_generation);
31276    }
31277
31278    #[test]
31279    fn test_queued_decommission_retry_rebinds_unresolved_entries_when_promoted() {
31280        let previous_generation = OffsetDateTime::UNIX_EPOCH;
31281        let mut meta = PoolMeta {
31282            pools: vec![PoolStatus {
31283                id: 0,
31284                cmd_line: "pool-0".to_string(),
31285                last_update: previous_generation,
31286                decommission: Some(PoolDecommissionInfo {
31287                    canceled: true,
31288                    unresolved_entries: vec![DecommissionUnresolvedEntry {
31289                        bucket: "bucket-a".to_string(),
31290                        object: "object-a".to_string(),
31291                        pool_index: 0,
31292                        set_index: 0,
31293                        source_generation: previous_generation,
31294                        candidate_count: 1,
31295                        disk_error_count: 1,
31296                        observed_at: previous_generation,
31297                        reason: "metadata_resolution_failed".to_string(),
31298                    }],
31299                    ..Default::default()
31300                }),
31301            }],
31302            ..Default::default()
31303        };
31304
31305        meta.queue_decommission(
31306            0,
31307            PoolSpaceInfo {
31308                total: 100,
31309                free: 25,
31310                used: 75,
31311            },
31312        )
31313        .expect("canceled decommission with unresolved entries should queue an atomic retry");
31314
31315        assert!(meta.is_suspended(0));
31316        assert!(meta.promote_queued_decommission(0));
31317        let promoted = meta.pools[0]
31318            .decommission
31319            .as_ref()
31320            .expect("promoted decommission metadata should exist");
31321        let generation = promoted.start_time.expect("promotion should assign a generation");
31322        assert_eq!(promoted.unresolved_entries.len(), 1);
31323        assert_eq!(promoted.unresolved_entries[0].source_generation, generation);
31324    }
31325
31326    #[test]
31327    fn test_queued_decommission_promotion_advances_generation_after_clock_rollback() {
31328        let queued_at = OffsetDateTime::from_unix_timestamp(1_260).expect("fixed timestamp should be valid");
31329        let earlier_tick = queued_at - Duration::nanoseconds(10);
31330        let rebalance_floor = queued_at + Duration::nanoseconds(5);
31331        let rebalance = RebalanceMeta {
31332            stopped_at: Some(rebalance_floor),
31333            id: "completed-rebalance".to_string(),
31334            ..Default::default()
31335        };
31336        let mut meta = PoolMeta {
31337            pools: vec![PoolStatus {
31338                id: 0,
31339                cmd_line: "pool-0".to_string(),
31340                last_update: queued_at,
31341                decommission: Some(PoolDecommissionInfo {
31342                    queued: true,
31343                    ..Default::default()
31344                }),
31345            }],
31346            ..Default::default()
31347        };
31348
31349        assert!(meta.promote_queued_decommission_at_for_test(0, earlier_tick, Some(&rebalance)));
31350        let expected_generation = rebalance_floor + Duration::nanoseconds(1);
31351        let promoted = meta.pools[0]
31352            .decommission
31353            .as_ref()
31354            .expect("promoted decommission metadata should exist");
31355        assert_eq!(meta.pools[0].last_update, expected_generation);
31356        assert_eq!(promoted.start_time, Some(expected_generation));
31357        assert!(!promoted.queued);
31358    }
31359
31360    #[test]
31361    fn test_pool_meta_queued_decommission_is_suspended_to_preserve_reserved_capacity() {
31362        let mut meta = PoolMeta {
31363            pools: vec![PoolStatus {
31364                id: 0,
31365                cmd_line: "pool-0".to_string(),
31366                last_update: OffsetDateTime::UNIX_EPOCH,
31367                decommission: None,
31368            }],
31369            ..Default::default()
31370        };
31371
31372        meta.queue_decommission(
31373            0,
31374            PoolSpaceInfo {
31375                total: 100,
31376                free: 10,
31377                used: 90,
31378            },
31379        )
31380        .expect("queued decommission should be stored");
31381
31382        assert!(meta.is_suspended(0));
31383        assert!(meta.promote_queued_decommission(0));
31384        assert!(meta.is_suspended(0));
31385    }
31386
31387    #[test]
31388    fn test_pool_meta_promoted_queued_decommission_can_be_canceled() {
31389        let mut meta = PoolMeta {
31390            pools: vec![PoolStatus {
31391                id: 0,
31392                cmd_line: "pool-0".to_string(),
31393                last_update: OffsetDateTime::UNIX_EPOCH,
31394                decommission: None,
31395            }],
31396            ..Default::default()
31397        };
31398
31399        meta.queue_decommission(
31400            0,
31401            PoolSpaceInfo {
31402                total: 100,
31403                free: 10,
31404                used: 90,
31405            },
31406        )
31407        .expect("queued decommission should be stored");
31408
31409        assert!(pool_meta_has_active_decommission(&meta));
31410        assert!(meta.promote_queued_decommission(0));
31411        assert!(meta.decommission_cancel(0));
31412
31413        let info = meta.pools[0]
31414            .decommission
31415            .as_ref()
31416            .expect("canceled decommission state should be kept for clear");
31417        assert!(info.canceled);
31418        assert!(!info.queued);
31419        assert!(!info.failed);
31420        assert!(!info.complete);
31421        assert!(!pool_meta_has_active_decommission(&meta));
31422    }
31423
31424    #[test]
31425    fn test_pool_meta_failed_decommission_requires_clear_before_restart() {
31426        let mut meta = PoolMeta {
31427            pools: vec![PoolStatus {
31428                id: 0,
31429                cmd_line: "pool-0".to_string(),
31430                last_update: OffsetDateTime::UNIX_EPOCH,
31431                decommission: Some(PoolDecommissionInfo {
31432                    failed: true,
31433                    decommissioned_buckets: vec!["bucket-done".to_string()],
31434                    queued_buckets: vec!["bucket-pending".to_string()],
31435                    bucket: "bucket-pending".to_string(),
31436                    prefix: "prefix".to_string(),
31437                    object: "object.txt".to_string(),
31438                    items_decommissioned: 7,
31439                    items_decommission_failed: 3,
31440                    bytes_done: 1024,
31441                    bytes_failed: 256,
31442                    progress_save_item_baseline: 10,
31443                    ..Default::default()
31444                }),
31445            }],
31446            ..Default::default()
31447        };
31448
31449        let err = meta
31450            .decommission(
31451                0,
31452                PoolSpaceInfo {
31453                    total: 200,
31454                    free: 50,
31455                    used: 150,
31456                },
31457            )
31458            .expect_err("failed decommission should be blocked until cleared");
31459        assert!(err.to_string().contains("target pool decommission is blocked"));
31460        let blocked = meta.pools[0]
31461            .decommission
31462            .as_ref()
31463            .expect("blocked metadata should remain until clear");
31464        assert!(blocked.failed);
31465        assert_eq!(blocked.decommissioned_buckets, vec!["bucket-done".to_string()]);
31466        assert_eq!(blocked.items_decommissioned, 7);
31467        assert_eq!(blocked.bytes_done, 1024);
31468
31469        assert!(meta.clear_decommission(0).expect("failed decommission should clear"));
31470        assert!(
31471            meta.pools[0]
31472                .decommission
31473                .as_ref()
31474                .is_some_and(|info| !info.has_decommission_state())
31475        );
31476
31477        meta.decommission(
31478            0,
31479            PoolSpaceInfo {
31480                total: 200,
31481                free: 50,
31482                used: 150,
31483            },
31484        )
31485        .expect("cleared decommission should be restartable");
31486        meta.queue_buckets(
31487            0,
31488            vec![
31489                DecomBucketInfo {
31490                    name: "bucket-done".to_string(),
31491                    prefix: String::new(),
31492                },
31493                DecomBucketInfo {
31494                    name: "bucket-pending".to_string(),
31495                    prefix: String::new(),
31496                },
31497            ],
31498        );
31499
31500        let info = meta.pools[0]
31501            .decommission
31502            .as_ref()
31503            .expect("decommission info should be rebuilt");
31504        assert!(!info.failed);
31505        assert!(!info.canceled);
31506        assert!(!info.complete);
31507        assert!(info.decommissioned_buckets.is_empty());
31508        assert_eq!(info.queued_buckets, vec!["bucket-done".to_string(), "bucket-pending".to_string()]);
31509        assert_eq!(info.items_decommissioned, 0);
31510        assert_eq!(info.items_decommission_failed, 0);
31511        assert_eq!(info.bytes_done, 0);
31512        assert_eq!(info.bytes_failed, 0);
31513        assert_eq!(info.items_since_last_progress_save(), 0);
31514        assert_eq!(info.start_size, 50);
31515        assert_eq!(info.total_size, 200);
31516        assert_eq!(info.current_size, 50);
31517        assert_eq!(info.bucket, "bucket-pending");
31518        assert!(info.prefix.is_empty());
31519        assert!(info.object.is_empty());
31520        assert!(info.start_time.is_some());
31521    }
31522
31523    #[test]
31524    fn test_pool_meta_canceled_queued_decommission_requires_clear_before_restart() {
31525        let mut meta = PoolMeta {
31526            pools: vec![PoolStatus {
31527                id: 0,
31528                cmd_line: "pool-0".to_string(),
31529                last_update: OffsetDateTime::UNIX_EPOCH,
31530                decommission: Some(PoolDecommissionInfo {
31531                    canceled: true,
31532                    decommissioned_buckets: vec!["bucket-done".to_string()],
31533                    items_decommissioned: 5,
31534                    bytes_done: 512,
31535                    ..Default::default()
31536                }),
31537            }],
31538            ..Default::default()
31539        };
31540
31541        let err = meta
31542            .queue_decommission(
31543                0,
31544                PoolSpaceInfo {
31545                    total: 100,
31546                    free: 25,
31547                    used: 75,
31548                },
31549            )
31550            .expect_err("canceled queued decommission should be blocked until cleared");
31551        assert!(err.to_string().contains("target pool decommission is blocked"));
31552        let blocked = meta.pools[0]
31553            .decommission
31554            .as_ref()
31555            .expect("blocked metadata should remain until clear");
31556        assert!(blocked.canceled);
31557        assert_eq!(blocked.decommissioned_buckets, vec!["bucket-done".to_string()]);
31558        assert_eq!(blocked.items_decommissioned, 5);
31559        assert_eq!(blocked.bytes_done, 512);
31560
31561        assert!(meta.clear_decommission(0).expect("canceled decommission should clear"));
31562        assert!(
31563            meta.pools[0]
31564                .decommission
31565                .as_ref()
31566                .is_some_and(|info| !info.has_decommission_state())
31567        );
31568
31569        meta.queue_decommission(
31570            0,
31571            PoolSpaceInfo {
31572                total: 100,
31573                free: 25,
31574                used: 75,
31575            },
31576        )
31577        .expect("cleared queued decommission should be restartable");
31578
31579        let info = meta.pools[0]
31580            .decommission
31581            .as_ref()
31582            .expect("decommission info should be rebuilt");
31583        assert!(info.queued);
31584        assert!(info.start_time.is_none());
31585        assert!(info.decommissioned_buckets.is_empty());
31586        assert_eq!(info.items_decommissioned, 0);
31587        assert_eq!(info.bytes_done, 0);
31588    }
31589
31590    #[test]
31591    fn test_contextualized_decommission_listing_disks_available_rejects_empty_set() {
31592        let err = ensure_decommission_listing_disks_available(false, "bucket-a")
31593            .expect_err("missing online disks should be reported with bucket context");
31594        assert!(
31595            err.to_string()
31596                .contains("failed to list objects to decommission for bucket bucket-a: no disks available")
31597        );
31598    }
31599
31600    #[test]
31601    fn test_contextualized_decommission_listing_disks_available_allows_online_disks() {
31602        assert!(ensure_decommission_listing_disks_available(true, "bucket-a").is_ok());
31603    }
31604
31605    #[test]
31606    fn test_require_decommission_store_returns_value_when_present() {
31607        let store = require_decommission_store(Some(7_u8), "start decommission").expect("present store should be returned");
31608        assert_eq!(store, 7);
31609    }
31610
31611    #[test]
31612    fn test_require_decommission_store_returns_error_when_missing() {
31613        let err = require_decommission_store::<u8>(None, "start decommission").expect_err("missing store should return error");
31614        assert!(
31615            err.to_string()
31616                .contains("failed to start decommission: store not initialized")
31617        );
31618    }
31619
31620    #[test]
31621    fn test_bind_decommission_cancelers_binds_existing_slots_only() {
31622        let parent = CancellationToken::new();
31623        let mut cancelers = vec![None, None];
31624
31625        let bound = bind_decommission_cancelers(&[0, 3, 1], &parent, cancelers.as_mut_slice());
31626
31627        assert_eq!(bound.len(), 2);
31628        assert_eq!(bound[0].0, 0);
31629        assert_eq!(bound[1].0, 1);
31630        assert!(cancelers[0].is_some());
31631        assert!(cancelers[1].is_some());
31632    }
31633
31634    #[test]
31635    fn test_bind_decommission_cancelers_child_tokens_follow_parent_cancel() {
31636        let parent = CancellationToken::new();
31637        let mut cancelers = vec![None];
31638
31639        let bound = bind_decommission_cancelers(&[0], &parent, cancelers.as_mut_slice());
31640        assert_eq!(bound.len(), 1);
31641        assert!(!bound[0].1.is_cancelled());
31642
31643        parent.cancel();
31644        assert!(bound[0].1.is_cancelled());
31645    }
31646
31647    #[test]
31648    fn test_bind_decommission_cancelers_replaces_existing_slot() {
31649        let parent = CancellationToken::new();
31650        let existing = DecommissionCanceler::new(CancellationToken::new());
31651        let mut cancelers = vec![Some(existing.clone())];
31652
31653        let bound = bind_decommission_cancelers(&[0], &parent, cancelers.as_mut_slice());
31654
31655        assert_eq!(bound.len(), 1);
31656        assert_eq!(bound[0].0, 0);
31657        assert!(existing.is_cancelled());
31658        let replacement = cancelers[0].as_ref().expect("replacement token should be stored");
31659        assert!(!replacement.is_cancelled());
31660        parent.cancel();
31661        assert!(replacement.is_cancelled());
31662    }
31663
31664    #[test]
31665    fn test_bind_missing_decommission_cancelers_stops_at_existing_slot() {
31666        let parent = CancellationToken::new();
31667        let existing = DecommissionCanceler::new(CancellationToken::new());
31668        let mut cancelers = vec![None, Some(existing.clone()), None];
31669
31670        let bound = bind_missing_decommission_cancelers(&[0, 1, 2], &parent, cancelers.as_mut_slice());
31671
31672        assert_eq!(bound.len(), 1);
31673        assert_eq!(bound[0].0, 0);
31674        assert!(cancelers[0].is_some());
31675        assert!(cancelers[1].is_some());
31676        assert!(cancelers[2].is_none());
31677        assert!(!existing.is_cancelled());
31678    }
31679
31680    #[test]
31681    fn test_serialized_decommission_double_start_preserves_first_operation() {
31682        let mut pool_meta = PoolMeta {
31683            pools: vec![decommission_test_pool_status(0, None), decommission_test_pool_status(1, None)],
31684            ..Default::default()
31685        };
31686        let first_parent = CancellationToken::new();
31687        let second_parent = CancellationToken::new();
31688        let mut cancelers = vec![None, None];
31689
31690        let first = reserve_decommission_start_cancelers(&pool_meta, &[0], &[0], &first_parent, cancelers.as_mut_slice())
31691            .expect("first start should reserve its worker");
31692        pool_meta
31693            .decommission(
31694                0,
31695                PoolSpaceInfo {
31696                    total: 100,
31697                    free: 40,
31698                    used: 60,
31699                },
31700            )
31701            .expect("first start should install active metadata");
31702
31703        let second = reserve_decommission_start_cancelers(&pool_meta, &[0], &[0], &second_parent, cancelers.as_mut_slice());
31704
31705        assert!(matches!(second, Err(Error::DecommissionAlreadyRunning)));
31706        let current = cancelers[0].as_ref().expect("first operation should retain the slot");
31707        assert!(current.owns_same_operation(first[0].1.canceler()));
31708        assert!(current.is_active());
31709        assert!(!first_parent.is_cancelled());
31710    }
31711
31712    #[test]
31713    fn test_local_decommission_queue_prefix_stops_at_remote_leader() {
31714        let endpoints = EndpointServerPools::from(vec![
31715            decommission_test_pool_endpoint(0, true),
31716            decommission_test_pool_endpoint(1, true),
31717            decommission_test_pool_endpoint(2, false),
31718            decommission_test_pool_endpoint(3, true),
31719        ]);
31720
31721        let local = local_decommission_queue_prefix(&endpoints, &[0, 1, 2, 3]).expect("prefix should resolve");
31722
31723        assert_eq!(local, vec![0, 1]);
31724    }
31725
31726    #[test]
31727    fn test_local_decommission_queue_prefix_empty_when_first_leader_remote() {
31728        let endpoints = EndpointServerPools::from(vec![
31729            decommission_test_pool_endpoint(0, false),
31730            decommission_test_pool_endpoint(1, true),
31731        ]);
31732
31733        let local = local_decommission_queue_prefix(&endpoints, &[0, 1]).expect("prefix should resolve");
31734
31735        assert!(local.is_empty());
31736    }
31737
31738    #[test]
31739    fn test_decommission_start_local_leader_allows_remote_queued_pool() {
31740        let endpoints = EndpointServerPools::from(vec![
31741            decommission_test_pool_endpoint(0, true),
31742            decommission_test_pool_endpoint(1, false),
31743        ]);
31744
31745        assert!(ensure_decommission_start_local_leader(&endpoints, &[0, 1]).is_ok());
31746    }
31747
31748    #[test]
31749    fn test_decommission_start_local_leader_rejects_remote_active_pool() {
31750        let endpoints = EndpointServerPools::from(vec![decommission_test_pool_endpoint(0, false)]);
31751
31752        let err = ensure_decommission_start_local_leader(&endpoints, &[0]).expect_err("remote active pool should be rejected");
31753
31754        assert!(
31755            err.to_string()
31756                .contains("decommission for pool 0 must run on the pool first endpoint")
31757        );
31758    }
31759
31760    #[test]
31761    fn test_missing_decommission_worker_prefix_stops_at_active_worker() {
31762        let cancelers = vec![None, Some(DecommissionCanceler::new(CancellationToken::new())), None];
31763
31764        let missing = missing_decommission_worker_prefix(&[0, 1, 2], cancelers.as_slice());
31765
31766        assert_eq!(missing, vec![0]);
31767    }
31768
31769    #[test]
31770    fn test_resumable_decommission_queue_indices_skip_terminal_predecessors() {
31771        let meta = PoolMeta {
31772            pools: vec![
31773                decommission_test_pool_status(
31774                    0,
31775                    Some(PoolDecommissionInfo {
31776                        complete: true,
31777                        ..Default::default()
31778                    }),
31779                ),
31780                decommission_test_pool_status(
31781                    1,
31782                    Some(PoolDecommissionInfo {
31783                        canceled: true,
31784                        ..Default::default()
31785                    }),
31786                ),
31787                decommission_test_pool_status(
31788                    2,
31789                    Some(PoolDecommissionInfo {
31790                        failed: true,
31791                        ..Default::default()
31792                    }),
31793                ),
31794                decommission_test_pool_status(
31795                    3,
31796                    Some(PoolDecommissionInfo {
31797                        queued: true,
31798                        ..Default::default()
31799                    }),
31800                ),
31801                decommission_test_pool_status(4, None),
31802            ],
31803            ..Default::default()
31804        };
31805
31806        assert_eq!(resumable_decommission_queue_indices(&meta), vec![3]);
31807    }
31808
31809    #[test]
31810    fn test_resumable_decommission_queue_indices_preserve_active_predecessor_order() {
31811        let meta = PoolMeta {
31812            pools: vec![
31813                decommission_test_pool_status(
31814                    0,
31815                    Some(PoolDecommissionInfo {
31816                        complete: true,
31817                        ..Default::default()
31818                    }),
31819                ),
31820                decommission_test_pool_status(
31821                    1,
31822                    Some(PoolDecommissionInfo {
31823                        start_time: Some(OffsetDateTime::UNIX_EPOCH),
31824                        ..Default::default()
31825                    }),
31826                ),
31827                decommission_test_pool_status(
31828                    2,
31829                    Some(PoolDecommissionInfo {
31830                        queued: true,
31831                        ..Default::default()
31832                    }),
31833                ),
31834            ],
31835            ..Default::default()
31836        };
31837
31838        assert_eq!(resumable_decommission_queue_indices(&meta), vec![1, 2]);
31839    }
31840
31841    #[tokio::test]
31842    async fn test_runtime_recovery_reserves_the_startup_resumable_queue() {
31843        let meta = PoolMeta {
31844            version: super::POOL_META_VERSION,
31845            pools: vec![
31846                decommission_test_pool_status(
31847                    0,
31848                    Some(PoolDecommissionInfo {
31849                        failed: true,
31850                        ..Default::default()
31851                    }),
31852                ),
31853                decommission_test_pool_status(
31854                    1,
31855                    Some(PoolDecommissionInfo {
31856                        canceled: true,
31857                        ..Default::default()
31858                    }),
31859                ),
31860                decommission_test_pool_status(
31861                    2,
31862                    Some(PoolDecommissionInfo {
31863                        start_time: Some(OffsetDateTime::UNIX_EPOCH),
31864                        ..Default::default()
31865                    }),
31866                ),
31867                decommission_test_pool_status(
31868                    3,
31869                    Some(PoolDecommissionInfo {
31870                        queued: true,
31871                        ..Default::default()
31872                    }),
31873                ),
31874            ],
31875            ..Default::default()
31876        };
31877        let startup_ids = meta
31878            .return_resumable_pools()
31879            .into_iter()
31880            .map(|pool| pool.id)
31881            .collect::<Vec<_>>();
31882        let store = decommission_worker_test_store(meta, vec![None, None, None, None]);
31883        let endpoints = EndpointServerPools::from(vec![
31884            decommission_test_pool_endpoint(0, true),
31885            decommission_test_pool_endpoint(1, true),
31886            decommission_test_pool_endpoint(2, true),
31887            decommission_test_pool_endpoint(3, true),
31888        ]);
31889        let reserved = store
31890            .reserve_missing_local_decommission_routines(&CancellationToken::new(), &endpoints)
31891            .await
31892            .expect("runtime recovery reservation should succeed");
31893        let runtime_indices = reserved.iter().map(|(idx, _)| *idx).collect::<Vec<_>>();
31894
31895        assert_eq!(runtime_indices, vec![2, 3]);
31896        assert_eq!(startup_ids, vec![2, 3]);
31897    }
31898
31899    #[tokio::test]
31900    async fn test_runtime_recovery_does_not_reserve_behind_active_predecessor() {
31901        let meta = PoolMeta {
31902            version: super::POOL_META_VERSION,
31903            pools: vec![
31904                decommission_test_pool_status(
31905                    0,
31906                    Some(PoolDecommissionInfo {
31907                        failed: true,
31908                        ..Default::default()
31909                    }),
31910                ),
31911                decommission_test_pool_status(
31912                    1,
31913                    Some(PoolDecommissionInfo {
31914                        canceled: true,
31915                        ..Default::default()
31916                    }),
31917                ),
31918                decommission_test_pool_status(
31919                    2,
31920                    Some(PoolDecommissionInfo {
31921                        start_time: Some(OffsetDateTime::UNIX_EPOCH),
31922                        ..Default::default()
31923                    }),
31924                ),
31925                decommission_test_pool_status(
31926                    3,
31927                    Some(PoolDecommissionInfo {
31928                        queued: true,
31929                        ..Default::default()
31930                    }),
31931                ),
31932            ],
31933            ..Default::default()
31934        };
31935        let active = DecommissionCanceler::new(CancellationToken::new());
31936        let store = decommission_worker_test_store(meta, vec![None, None, Some(active.clone()), None]);
31937        let endpoints = EndpointServerPools::from(vec![
31938            decommission_test_pool_endpoint(0, true),
31939            decommission_test_pool_endpoint(1, true),
31940            decommission_test_pool_endpoint(2, true),
31941            decommission_test_pool_endpoint(3, true),
31942        ]);
31943
31944        let reserved = store
31945            .reserve_missing_local_decommission_routines(&CancellationToken::new(), &endpoints)
31946            .await
31947            .expect("runtime recovery reservation should succeed");
31948
31949        assert!(reserved.is_empty());
31950        assert!(active.is_active());
31951    }
31952
31953    #[test]
31954    fn test_take_decommission_canceler_takes_and_clears_slot() {
31955        let canceler = DecommissionCanceler::new(CancellationToken::new());
31956        let mut cancelers = vec![Some(canceler)];
31957
31958        let taken = take_decommission_canceler(cancelers.as_mut_slice(), 0);
31959        assert!(taken.is_some());
31960        assert!(cancelers[0].is_none());
31961    }
31962
31963    #[test]
31964    fn test_take_decommission_canceler_returns_none_for_missing_slot() {
31965        let mut cancelers: Vec<Option<DecommissionCanceler>> = Vec::new();
31966        assert!(take_decommission_canceler(cancelers.as_mut_slice(), 0).is_none());
31967    }
31968
31969    #[test]
31970    fn test_has_active_decommission_canceler_true_when_any_slot_present() {
31971        let cancelers = vec![None, Some(DecommissionCanceler::new(CancellationToken::new()))];
31972        assert!(has_active_decommission_canceler(cancelers.as_slice()));
31973    }
31974
31975    #[test]
31976    fn test_has_active_decommission_canceler_false_when_all_empty() {
31977        let cancelers = vec![None, None];
31978        assert!(!has_active_decommission_canceler(cancelers.as_slice()));
31979    }
31980
31981    #[test]
31982    fn test_cancel_decommission_canceler_cancels_when_present() {
31983        let canceler = DecommissionCanceler::new(CancellationToken::new());
31984        let canceled = cancel_decommission_canceler(Some(canceler.clone()));
31985
31986        assert!(canceled);
31987        assert!(canceler.is_cancelled());
31988        assert!(!canceler.is_active());
31989    }
31990
31991    #[test]
31992    fn test_cancel_decommission_canceler_returns_false_when_missing() {
31993        assert!(!cancel_decommission_canceler(None));
31994    }
31995
31996    #[test]
31997    fn test_take_and_cancel_decommission_canceler_clears_slot() {
31998        let canceler = DecommissionCanceler::new(CancellationToken::new());
31999        let mut cancelers = vec![Some(canceler.clone())];
32000
32001        assert!(take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), 0));
32002        assert!(cancelers[0].is_none());
32003        assert!(canceler.is_cancelled());
32004        assert!(!canceler.is_active());
32005    }
32006
32007    #[test]
32008    fn test_take_and_cancel_decommission_canceler_missing_slot_is_false() {
32009        let mut cancelers = vec![None];
32010
32011        assert!(!take_and_cancel_decommission_canceler(cancelers.as_mut_slice(), 0));
32012        assert!(cancelers[0].is_none());
32013    }
32014
32015    #[test]
32016    fn test_guarded_decommission_future_releases_without_first_poll() {
32017        let canceler = DecommissionCanceler::new(CancellationToken::new());
32018        let cancelers = vec![Some(canceler.clone())];
32019        let guards = guard_decommission_cancelers(vec![(0, canceler.clone())]);
32020        let unpolled = async move {
32021            let _guards = guards;
32022            std::future::pending::<()>().await;
32023        };
32024
32025        drop(unpolled);
32026
32027        assert!(canceler.is_cancelled());
32028        assert!(!has_active_decommission_canceler(cancelers.as_slice()));
32029    }
32030
32031    #[test]
32032    fn test_partial_decommission_spawn_reservation_releases_bound_slot() {
32033        let parent = CancellationToken::new();
32034        let mut cancelers = vec![None];
32035        let bound = bind_decommission_cancelers(&[0, 1], &parent, cancelers.as_mut_slice());
32036        let guards = guard_decommission_cancelers(bound);
32037
32038        let result = super::ensure_decommission_routines_scheduled(guards.len(), 2);
32039        drop(guards);
32040
32041        assert!(result.is_err());
32042        assert!(!has_active_decommission_canceler(cancelers.as_slice()));
32043    }
32044
32045    #[tokio::test]
32046    async fn test_decommission_supervisor_observes_worker_abort() {
32047        let (started_tx, started_rx) = tokio::sync::oneshot::channel();
32048        let worker = tokio::spawn(async move {
32049            started_tx.send(()).expect("worker start should be observed");
32050            std::future::pending::<()>().await;
32051            #[allow(unreachable_code)]
32052            Ok(())
32053        });
32054
32055        started_rx.await.expect("worker start should be observed");
32056        worker.abort();
32057        let err = await_decommission_worker(3, worker)
32058            .await
32059            .expect_err("supervisor should observe aborted worker");
32060
32061        assert!(err.to_string().contains("decommission worker 3 task join error"));
32062    }
32063
32064    #[tokio::test]
32065    async fn test_decommission_supervisor_observes_worker_panic() {
32066        let worker = tokio::spawn(async move {
32067            panic!("injected decommission worker panic");
32068        });
32069
32070        let err = await_decommission_worker(4, worker)
32071            .await
32072            .expect_err("supervisor should observe panicked worker");
32073
32074        assert!(err.to_string().contains("decommission worker 4 task join error"));
32075    }
32076
32077    #[tokio::test]
32078    async fn test_decommission_worker_metadata_missing_releases_owned_slot() {
32079        let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
32080        let canceler = DecommissionCanceler::new(CancellationToken::new());
32081        *store.pool_meta.write().await = PoolMeta::default();
32082        store.decommission_cancelers.write().await[0] = Some(canceler.clone());
32083
32084        let err = store
32085            .do_decommission_in_routine(canceler.clone(), 0, Arc::new(Semaphore::new(1)))
32086            .await
32087            .expect_err("missing worker metadata should fail the routine");
32088
32089        assert!(err.to_string().contains("target pool was not found"));
32090        assert!(!canceler.is_active());
32091        assert!(store.decommission_cancelers.read().await[0].is_none());
32092    }
32093
32094    #[tokio::test]
32095    async fn test_decommission_supervisor_failure_cancels_queued_successor() {
32096        let first = DecommissionCanceler::new(CancellationToken::new());
32097        let queued = DecommissionCanceler::new(CancellationToken::new());
32098        let store = decommission_worker_test_store(PoolMeta::default(), vec![Some(first.clone()), Some(queued.clone())]);
32099        let guards = guard_decommission_cancelers(vec![(0, first.clone()), (1, queued.clone())]);
32100
32101        spawn_decommission_index_cancelers(store.clone(), CancellationToken::new(), guards, Arc::new(Semaphore::new(1)))
32102            .await
32103            .expect("decommission supervisor should finish after queued cleanup");
32104
32105        assert!(!first.is_active());
32106        assert!(!queued.is_active());
32107        assert!(queued.is_cancelled());
32108        assert!(store.decommission_cancelers.read().await.iter().all(Option::is_none));
32109    }
32110
32111    #[tokio::test]
32112    async fn test_decommission_supervisor_releases_slot_without_terminal_retry_when_pool_meta_is_blocked() {
32113        let generation = OffsetDateTime::UNIX_EPOCH;
32114        let canceler = DecommissionCanceler::new(CancellationToken::new());
32115        let pool_meta = PoolMeta {
32116            version: super::POOL_META_VERSION,
32117            pools: vec![decommission_test_pool_status(
32118                0,
32119                Some(PoolDecommissionInfo {
32120                    start_time: Some(generation),
32121                    ..Default::default()
32122                }),
32123            )],
32124            ..Default::default()
32125        };
32126        let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
32127        store
32128            .pool_meta_save_gate
32129            .lock()
32130            .await
32131            .observe_replicas(super::PoolMetaReplicaState {
32132                needs_repair: true,
32133                repair_write_safe: false,
32134            });
32135        let guards = guard_decommission_cancelers(vec![(0, canceler.clone())]);
32136
32137        tokio::time::timeout(
32138            StdDuration::from_secs(1),
32139            spawn_decommission_index_cancelers(store.clone(), CancellationToken::new(), guards, Arc::new(Semaphore::new(1))),
32140        )
32141        .await
32142        .expect("blocked supervisor must not enter terminal retry")
32143        .expect("blocked supervisor task should not panic");
32144
32145        assert!(store.decommission_cancelers.read().await[0].is_none());
32146        assert!(!canceler.is_active());
32147        let pool_meta = store.pool_meta.read().await;
32148        let info = pool_meta.pools[0]
32149            .decommission
32150            .as_ref()
32151            .expect("blocked decommission metadata should remain active");
32152        assert!(!info.failed);
32153        assert!(!info.canceled);
32154        assert!(!info.complete);
32155        assert_eq!(info.start_time, Some(generation));
32156    }
32157
32158    #[tokio::test]
32159    async fn test_decommission_promotion_rechecks_sticky_gate_after_start_wait() {
32160        let pool_meta = PoolMeta {
32161            pools: vec![decommission_test_pool_status(
32162                0,
32163                Some(PoolDecommissionInfo {
32164                    queued: true,
32165                    ..Default::default()
32166                }),
32167            )],
32168            ..Default::default()
32169        };
32170        let store = decommission_worker_test_store(pool_meta, vec![None]);
32171        let start_guard = store.start_gate.lock().await;
32172        let mut promotion = tokio::spawn({
32173            let store = store.clone();
32174            async move { store.promote_queued_decommission_for_test(0).await }
32175        });
32176        tokio::task::yield_now().await;
32177        assert!(!promotion.is_finished(), "promotion should be waiting for the start gate");
32178
32179        store
32180            .pool_meta_save_gate
32181            .lock()
32182            .await
32183            .observe_replicas(super::PoolMetaReplicaState {
32184                needs_repair: true,
32185                repair_write_safe: false,
32186            });
32187        drop(start_guard);
32188
32189        let err = tokio::time::timeout(StdDuration::from_secs(1), &mut promotion)
32190            .await
32191            .expect("blocked promotion should finish")
32192            .expect("promotion task should not panic")
32193            .expect_err("promotion must recheck the sticky gate after waiting");
32194        assert!(
32195            err.to_string()
32196                .contains("restart after all replicas are readable and consistent")
32197        );
32198
32199        let pool_meta = store.pool_meta.read().await;
32200        let info = pool_meta.pools[0]
32201            .decommission
32202            .as_ref()
32203            .expect("queued metadata should remain present");
32204        assert!(info.queued);
32205        assert!(info.start_time.is_none());
32206    }
32207
32208    #[tokio::test]
32209    #[serial_test::serial(pool_meta_version_env)]
32210    async fn test_decommission_promotion_persists_all_recovered_capacity_sources() {
32211        let (_temp_dirs, store, _other_store) =
32212            crate::services::rebalance::test_three_pool_stores_with_isolated_node_contexts(None).await;
32213        persist_v3_pool_meta_for_test(&store).await;
32214        let now = OffsetDateTime::UNIX_EPOCH + Duration::hours(3);
32215        let layout = DecommissionErasureLayout { data: 1, parity: 0 };
32216        let mut queued_reservation = build_decommission_capacity_reservation(
32217            DecommissionPoolCapacityInfo::for_test(1, layout, 90, 100, 10),
32218            layout,
32219            uuid::Uuid::new_v4(),
32220            7,
32221            now,
32222        )
32223        .expect("queued source reservation should be valid");
32224        queued_reservation.targets.push(DecommissionCapacityTarget {
32225            pool_index: 2,
32226            layout,
32227            physical_total_at_reservation: 200,
32228            physical_free_at_reservation: 100,
32229            reserved_physical_bytes: queued_reservation.peak_physical_bytes,
32230            consumed_physical_bytes: 0,
32231            observed_physical_bytes: 0,
32232            inflight_physical_bytes: 0,
32233            pending_physical_bytes: 0,
32234            pending_mutation_id: None,
32235            temporary_mutations: Vec::new(),
32236        });
32237        {
32238            let mut pool_meta = store.pool_meta.write().await;
32239            pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
32240                start_time: Some(now),
32241                ..Default::default()
32242            });
32243            pool_meta.pools[1].decommission = Some(PoolDecommissionInfo {
32244                queued: true,
32245                start_time: Some(now),
32246                capacity_reservation: Some(queued_reservation),
32247                ..Default::default()
32248            });
32249        }
32250        let mut loaded_v3 = PoolMeta::default();
32251        loaded_v3
32252            .load_no_lock_from_replicas(store.pools.clone())
32253            .await
32254            .expect("the startup fixture must provide a durable V3 pool metadata replica");
32255        assert_eq!(loaded_v3.version, POOL_META_GENERATION_VERSION);
32256        store
32257            .save_current_pool_meta_for_test(&[0, 1, 2])
32258            .await
32259            .expect("active and queued source metadata should be persisted before promotion");
32260        set_decommission_capacity_info_overrides_for_test(
32261            store.id,
32262            vec![vec![
32263                DecommissionPoolCapacityInfo::for_test(0, layout, 90, 100, 10),
32264                DecommissionPoolCapacityInfo::for_test(1, layout, 90, 100, 10),
32265                DecommissionPoolCapacityInfo::for_test(2, layout, 100, 200, 100),
32266            ]],
32267        );
32268
32269        store
32270            .promote_queued_decommission_for_test(1)
32271            .await
32272            .expect("queued promotion should recover all active source reservations");
32273
32274        let mut persisted = PoolMeta::default();
32275        persisted
32276            .load_no_lock_from_replicas(store.pools.clone())
32277            .await
32278            .expect("promoted active and recovered source metadata should reload");
32279        assert_eq!(persisted.version, POOL_META_GENERATION_VERSION);
32280        assert!(
32281            persisted.pools[0]
32282                .decommission
32283                .as_ref()
32284                .and_then(|info| info.capacity_reservation.as_ref())
32285                .is_some_and(DecommissionCapacityReservation::active)
32286        );
32287        let promoted = persisted.pools[1]
32288            .decommission
32289            .as_ref()
32290            .expect("queued source should remain after promotion");
32291        assert!(!promoted.queued, "queued source should be promoted durably");
32292        assert!(
32293            promoted
32294                .capacity_reservation
32295                .as_ref()
32296                .is_some_and(DecommissionCapacityReservation::active)
32297        );
32298    }
32299
32300    #[tokio::test]
32301    async fn test_decommission_bucket_done_rechecks_sticky_gate_before_mutation() {
32302        let bucket = DecomBucketInfo {
32303            name: "bucket-a".to_string(),
32304            prefix: String::new(),
32305        };
32306        let pool_meta = PoolMeta {
32307            pools: vec![decommission_test_pool_status(
32308                0,
32309                Some(PoolDecommissionInfo {
32310                    queued_buckets: vec![bucket.to_string()],
32311                    ..Default::default()
32312                }),
32313            )],
32314            ..Default::default()
32315        };
32316        let store = decommission_worker_test_store(pool_meta, vec![None]);
32317        store
32318            .pool_meta_save_gate
32319            .lock()
32320            .await
32321            .observe_replicas(super::PoolMetaReplicaState {
32322                needs_repair: true,
32323                repair_write_safe: false,
32324            });
32325
32326        let err = store
32327            .mark_decommission_bucket_done_and_save(0, OffsetDateTime::UNIX_EPOCH, &bucket)
32328            .await
32329            .expect_err("bucket completion must stop before mutating sticky pool metadata");
32330        assert!(
32331            err.to_string()
32332                .contains("restart after all replicas are readable and consistent")
32333        );
32334
32335        let pool_meta = store.pool_meta.read().await;
32336        let info = pool_meta.pools[0]
32337            .decommission
32338            .as_ref()
32339            .expect("decommission metadata should remain present");
32340        assert_eq!(info.queued_buckets, vec![bucket.to_string()]);
32341        assert!(info.decommissioned_buckets.is_empty());
32342    }
32343
32344    #[tokio::test]
32345    async fn test_decommission_cancel_save_failure_preserves_generation_and_blocks_retry() {
32346        let generation = OffsetDateTime::UNIX_EPOCH;
32347        let canceler = DecommissionCanceler::new(CancellationToken::new());
32348        let pool_meta = PoolMeta {
32349            version: super::POOL_META_VERSION,
32350            pools: vec![decommission_test_pool_status(
32351                0,
32352                Some(PoolDecommissionInfo {
32353                    start_time: Some(generation),
32354                    ..Default::default()
32355                }),
32356            )],
32357            ..Default::default()
32358        };
32359        let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
32360
32361        let err = store
32362            .decommission_cancel_with_owner_and_save(0, Some(&canceler), |_, _| async { Err(Error::Timeout) })
32363            .await
32364            .expect_err("injected pool metadata timeout should fail cancel");
32365        assert!(matches!(err, Error::Timeout));
32366
32367        {
32368            let pool_meta = store.pool_meta.read().await;
32369            let info = pool_meta.pools[0]
32370                .decommission
32371                .as_ref()
32372                .expect("failed cancel must retain decommission metadata");
32373            assert_eq!(info.start_time, Some(generation));
32374            assert!(!info.canceled);
32375            assert!(!info.complete);
32376            assert!(!info.failed);
32377            assert!(ensure_decommission_generation(&pool_meta, 0, generation).is_ok());
32378        }
32379        {
32380            let cancelers = store.decommission_cancelers.read().await;
32381            let current = cancelers[0].as_ref().expect("failed cancel must retain the worker owner");
32382            assert!(current.owns_same_operation(&canceler));
32383            assert!(current.is_active());
32384        }
32385        assert!(!canceler.is_cancelled());
32386        assert!(store.decommission_terminal_retryable_for_operation(0, &canceler).await);
32387
32388        let retry_save_called = Arc::new(AtomicBool::new(false));
32389        let retry_save_called_by_closure = retry_save_called.clone();
32390        let retry_err = store
32391            .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |_, _| async move {
32392                retry_save_called_by_closure.store(true, Ordering::SeqCst);
32393                Ok(())
32394            })
32395            .await
32396            .expect_err("an ambiguous save failure must block same-process retry");
32397        assert!(
32398            retry_err
32399                .to_string()
32400                .contains("restart after all replicas are readable and consistent")
32401        );
32402        assert!(!retry_save_called.load(Ordering::SeqCst));
32403
32404        {
32405            let pool_meta = store.pool_meta.read().await;
32406            let info = pool_meta.pools[0]
32407                .decommission
32408                .as_ref()
32409                .expect("blocked retry must retain decommission metadata");
32410            assert_eq!(info.start_time, Some(generation));
32411            assert!(!info.canceled);
32412            assert!(!info.complete);
32413            assert!(!info.failed);
32414        }
32415        assert!(store.decommission_cancelers.read().await[0].is_some());
32416        assert!(canceler.is_active());
32417        assert!(!canceler.is_cancelled());
32418    }
32419
32420    #[tokio::test]
32421    async fn test_decommission_cancel_stays_blocked_after_unreadable_pool_meta_replica() {
32422        let generation = OffsetDateTime::UNIX_EPOCH;
32423        let canceler = DecommissionCanceler::new(CancellationToken::new());
32424        let pool_meta = PoolMeta {
32425            version: super::POOL_META_VERSION,
32426            pools: vec![decommission_test_pool_status(
32427                0,
32428                Some(PoolDecommissionInfo {
32429                    start_time: Some(generation),
32430                    ..Default::default()
32431                }),
32432            )],
32433            ..Default::default()
32434        };
32435        let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
32436        store
32437            .pool_meta_save_gate
32438            .lock()
32439            .await
32440            .observe_replicas(super::PoolMetaReplicaState {
32441                needs_repair: true,
32442                repair_write_safe: false,
32443            });
32444        let save_called = Arc::new(AtomicBool::new(false));
32445        let save_called_by_closure = save_called.clone();
32446
32447        let err = store
32448            .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |_, _| async move {
32449                save_called_by_closure.store(true, Ordering::SeqCst);
32450                Ok(())
32451            })
32452            .await
32453            .expect_err("cancel must remain blocked until restart after an unreadable replica");
32454
32455        assert!(
32456            err.to_string()
32457                .contains("restart after all replicas are readable and consistent")
32458        );
32459        assert!(!save_called.load(Ordering::SeqCst));
32460        let pool_meta = store.pool_meta.read().await;
32461        let info = pool_meta.pools[0]
32462            .decommission
32463            .as_ref()
32464            .expect("blocked cancel must preserve decommission metadata");
32465        assert_eq!(info.start_time, Some(generation));
32466        assert!(!info.canceled);
32467        drop(pool_meta);
32468        assert!(canceler.is_active());
32469        assert!(!canceler.is_cancelled());
32470        let cancelers = store.decommission_cancelers.read().await;
32471        assert!(
32472            cancelers[0]
32473                .as_ref()
32474                .is_some_and(|current| current.owns_same_operation(&canceler))
32475        );
32476    }
32477
32478    #[tokio::test]
32479    async fn test_decommission_cancel_serializes_reload_until_local_commit() {
32480        let generation = OffsetDateTime::UNIX_EPOCH;
32481        let canceler = DecommissionCanceler::new(CancellationToken::new());
32482        let pool_meta = PoolMeta {
32483            version: super::POOL_META_VERSION,
32484            pools: vec![decommission_test_pool_status(
32485                0,
32486                Some(PoolDecommissionInfo {
32487                    start_time: Some(generation),
32488                    stage: "migrate_object".to_string(),
32489                    ..Default::default()
32490                }),
32491            )],
32492            ..Default::default()
32493        };
32494        let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
32495        let (persisted_tx, persisted_rx) = tokio::sync::oneshot::channel();
32496        let save_release = Arc::new(tokio::sync::Notify::new());
32497
32498        let cancel = tokio::spawn({
32499            let store = store.clone();
32500            let canceler = canceler.clone();
32501            let save_release = save_release.clone();
32502            async move {
32503                store
32504                    .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move {
32505                        persisted_tx
32506                            .send(snapshot.encode_config_data()?)
32507                            .map_err(|_| Error::other("failed to expose saved cancel snapshot"))?;
32508                        save_release.notified().await;
32509                        Ok(())
32510                    })
32511                    .await
32512            }
32513        });
32514        let persisted = persisted_rx.await.expect("save should expose the canceled snapshot");
32515        let reload_started = Arc::new(tokio::sync::Notify::new());
32516        let reload = tokio::spawn({
32517            let store = store.clone();
32518            let reload_started = reload_started.clone();
32519            async move {
32520                let mut reloaded = PoolMeta::default();
32521                reloaded.load_from_config_data(persisted)?;
32522                reload_started.notify_one();
32523                *store.pool_meta.write().await = reloaded;
32524                Ok::<(), Error>(())
32525            }
32526        });
32527        reload_started.notified().await;
32528        assert!(!reload.is_finished(), "peer reload must wait for cancel publication");
32529
32530        save_release.notify_one();
32531        cancel
32532            .await
32533            .expect("cancel task should not panic")
32534            .expect("cancel should commit before releasing the reload");
32535        reload
32536            .await
32537            .expect("reload task should not panic")
32538            .expect("reload should install the saved cancel after publication");
32539
32540        let pool_meta = store.pool_meta.read().await;
32541        let info = pool_meta.pools[0]
32542            .decommission
32543            .as_ref()
32544            .expect("reloaded cancel metadata should remain present");
32545        assert!(info.canceled);
32546        assert!(!info.complete);
32547        assert!(!info.failed);
32548        assert!(info.start_time.is_none());
32549        assert!(info.stage.is_empty(), "the persisted snapshot should have been decoded before commit");
32550        drop(pool_meta);
32551        assert!(store.decommission_cancelers.read().await[0].is_none());
32552        assert!(!canceler.is_active());
32553        assert!(canceler.is_cancelled());
32554    }
32555
32556    #[tokio::test]
32557    async fn test_decommission_cancel_transaction_survives_caller_abort_after_durable_save() {
32558        let generation = OffsetDateTime::UNIX_EPOCH;
32559        let canceler = DecommissionCanceler::new(CancellationToken::new());
32560        let pool_meta = PoolMeta {
32561            version: super::POOL_META_VERSION,
32562            pools: vec![decommission_test_pool_status(
32563                0,
32564                Some(PoolDecommissionInfo {
32565                    start_time: Some(generation),
32566                    ..Default::default()
32567                }),
32568            )],
32569            ..Default::default()
32570        };
32571        let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
32572        let persisted = Arc::new(std::sync::Mutex::new(None));
32573        let (durable_tx, durable_rx) = tokio::sync::oneshot::channel();
32574        let save_release = Arc::new(tokio::sync::Notify::new());
32575        let cancel = tokio::spawn({
32576            let store = store.clone();
32577            let canceler = canceler.clone();
32578            let persisted = persisted.clone();
32579            let save_release = save_release.clone();
32580            async move {
32581                store
32582                    .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move {
32583                        assert!(
32584                            snapshot.pools[0]
32585                                .decommission
32586                                .as_ref()
32587                                .is_some_and(|info| info.canceled && info.start_time.is_none())
32588                        );
32589                        *persisted.lock().expect("persisted cancel lock should not be poisoned") =
32590                            Some(snapshot.encode_config_data()?);
32591                        durable_tx
32592                            .send(())
32593                            .map_err(|_| Error::other("failed to report durable cancel snapshot"))?;
32594                        save_release.notified().await;
32595                        Ok(())
32596                    })
32597                    .await
32598            }
32599        });
32600        durable_rx.await.expect("save hook should report the durable cancel snapshot");
32601        cancel.abort();
32602        let join_err = cancel.await.expect_err("caller cancel future should be aborted");
32603        assert!(join_err.is_cancelled());
32604        assert!(
32605            store.pool_meta.try_read().is_err(),
32606            "the detached transaction must retain its state guard"
32607        );
32608        assert!(
32609            store.decommission_cancelers.try_read().is_err(),
32610            "the detached transaction must retain its owner guard"
32611        );
32612
32613        save_release.notify_one();
32614        tokio::time::timeout(StdDuration::from_secs(1), canceler.token().cancelled())
32615            .await
32616            .expect("detached cancel transaction should terminate the old token");
32617
32618        let persisted = persisted
32619            .lock()
32620            .expect("persisted cancel lock should not be poisoned")
32621            .take()
32622            .expect("save should capture the durable cancel snapshot");
32623        let mut durable = PoolMeta::default();
32624        durable
32625            .load_from_config_data(persisted)
32626            .expect("durable cancel snapshot should decode");
32627        assert!(
32628            durable.pools[0]
32629                .decommission
32630                .as_ref()
32631                .is_some_and(|info| info.canceled && info.start_time.is_none())
32632        );
32633        assert!(store.decommission_cancelers.read().await[0].is_none());
32634        assert!(!canceler.is_active());
32635        assert!(canceler.is_cancelled());
32636
32637        let side_effect_ran = Arc::new(AtomicBool::new(false));
32638        let operation_gate = store.ctx.data_movement_operation_gate();
32639        let result = run_decommission_side_effect(canceler.token(), &operation_gate, {
32640            let side_effect_ran = side_effect_ran.clone();
32641            move || async move {
32642                side_effect_ran.store(true, Ordering::SeqCst);
32643                Ok(())
32644            }
32645        })
32646        .await;
32647        assert!(matches!(result, Err(Error::OperationCanceled)));
32648        assert!(!side_effect_ran.load(Ordering::SeqCst));
32649        assert!(
32650            store.pool_meta.read().await.pools[0]
32651                .decommission
32652                .as_ref()
32653                .is_some_and(|info| info.canceled && !info.failed)
32654        );
32655    }
32656
32657    #[tokio::test]
32658    async fn test_decommission_cancel_persists_before_signaling_and_quiesces_before_return() {
32659        let generation = OffsetDateTime::UNIX_EPOCH;
32660        let canceler = DecommissionCanceler::new(CancellationToken::new());
32661        let pool_meta = PoolMeta {
32662            pools: vec![decommission_test_pool_status(
32663                0,
32664                Some(PoolDecommissionInfo {
32665                    start_time: Some(generation),
32666                    ..Default::default()
32667                }),
32668            )],
32669            ..Default::default()
32670        };
32671        let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
32672        let operation_gate = store.ctx.data_movement_operation_gate();
32673        let side_effect = operation_gate.read().await;
32674        let save_started = Arc::new(tokio::sync::Notify::new());
32675        let save_release = Arc::new(tokio::sync::Notify::new());
32676        let save_entered = Arc::new(AtomicBool::new(false));
32677        let persisted = Arc::new(std::sync::Mutex::new(None));
32678
32679        let mut cancel = tokio::spawn({
32680            let store = store.clone();
32681            let canceler = canceler.clone();
32682            let save_started = save_started.clone();
32683            let save_release = save_release.clone();
32684            let save_entered = save_entered.clone();
32685            let persisted = persisted.clone();
32686            async move {
32687                store
32688                    .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move {
32689                        *persisted.lock().expect("persisted cancel lock should not be poisoned") =
32690                            Some(snapshot.encode_config_data()?);
32691                        save_entered.store(true, Ordering::SeqCst);
32692                        save_started.notify_one();
32693                        save_release.notified().await;
32694                        Ok(())
32695                    })
32696                    .await
32697            }
32698        });
32699
32700        tokio::time::timeout(StdDuration::from_secs(1), save_started.notified())
32701            .await
32702            .expect("cancel should persist while the side effect is still in flight");
32703        assert!(save_entered.load(Ordering::SeqCst));
32704        assert!(!cancel.is_finished(), "cancel must wait for the injected save");
32705        assert!(
32706            store.pool_meta_save_gate.try_lock().is_err(),
32707            "the cancel save must exclude stale full-document saves until publication"
32708        );
32709        assert!(
32710            store.pool_meta.try_read().is_err(),
32711            "the active generation must stay write-locked through persistence"
32712        );
32713        assert!(!canceler.is_cancelled(), "the token must remain live until persistence commits");
32714
32715        let mut fail = tokio::spawn({
32716            let store = store.clone();
32717            async move { store.decommission_failed(0).await }
32718        });
32719        tokio::task::yield_now().await;
32720        assert!(!fail.is_finished(), "fail must serialize behind the pending cancel");
32721
32722        save_release.notify_one();
32723        tokio::time::timeout(StdDuration::from_secs(1), canceler.token().cancelled())
32724            .await
32725            .expect("the durable cancel must signal the active worker");
32726        assert!(!cancel.is_finished(), "cancel must wait for in-flight movement after the durable signal");
32727        let canceled_at = {
32728            let pool_meta = store.pool_meta.read().await;
32729            let info = pool_meta.pools[0]
32730                .decommission
32731                .as_ref()
32732                .expect("the durable cancel must be published before quiescence");
32733            assert!(info.canceled);
32734            assert!(!info.complete);
32735            assert!(!info.failed);
32736            assert!(info.start_time.is_none());
32737            pool_meta.pools[0].last_update
32738        };
32739        let expected_generation = crate::store::scanner_data_movement_timestamp_generation(canceled_at);
32740        let scanner_status = store.scanner_data_movement_pause_snapshot_for_test().await;
32741        assert_eq!(scanner_status.movement_generation, expected_generation);
32742
32743        drop(side_effect);
32744        tokio::time::timeout(StdDuration::from_secs(1), &mut cancel)
32745            .await
32746            .expect("cancel should finish after in-flight movement quiesces")
32747            .expect("cancel task should not panic")
32748            .expect("cancel should commit");
32749        tokio::time::timeout(StdDuration::from_secs(1), &mut fail)
32750            .await
32751            .expect("fail should finish after cancel commits")
32752            .expect("fail task should not panic")
32753            .expect("stale fail should be a no-op");
32754
32755        let pool_meta = store.pool_meta.read().await;
32756        let info = pool_meta.pools[0]
32757            .decommission
32758            .as_ref()
32759            .expect("cancel metadata should remain present");
32760        assert!(info.canceled);
32761        assert!(!info.complete);
32762        assert!(!info.failed);
32763        assert!(info.start_time.is_none());
32764        drop(pool_meta);
32765        assert!(canceler.is_cancelled());
32766        assert!(!canceler.is_active());
32767        assert!(store.decommission_cancelers.read().await[0].is_none());
32768        assert_eq!(store.scanner_data_movement_generation(), expected_generation);
32769
32770        let persisted = persisted
32771            .lock()
32772            .expect("persisted cancel lock should not be poisoned")
32773            .take()
32774            .expect("cancel should persist a durable snapshot");
32775        let mut durable = PoolMeta::default();
32776        durable
32777            .load_from_config_data(persisted)
32778            .expect("durable cancel snapshot should decode after restart");
32779        let restarted = decommission_worker_test_store(durable.clone(), Vec::new());
32780        let restarted_status = restarted.scanner_data_movement_pause_status().await;
32781        assert_eq!(restarted_status.movement_generation, expected_generation);
32782
32783        assert!(
32784            durable
32785                .clear_decommission_at_for_test(0, canceled_at, None)
32786                .expect("same-tick clear should succeed")
32787        );
32788        assert!(durable.pools[0].last_update > canceled_at);
32789    }
32790
32791    #[test]
32792    fn test_decommission_cancel_commit_rejects_a_replaced_generation() {
32793        let old_generation = OffsetDateTime::UNIX_EPOCH;
32794        let new_generation = old_generation + Duration::seconds(1);
32795        let mut canceled = PoolMeta {
32796            pools: vec![decommission_test_pool_status(
32797                0,
32798                Some(PoolDecommissionInfo {
32799                    start_time: Some(old_generation),
32800                    ..Default::default()
32801                }),
32802            )],
32803            ..Default::default()
32804        };
32805        let previous_last_update = canceled.pools[0].last_update;
32806        assert!(canceled.decommission_cancel(0));
32807        let canceled_pool = canceled.pools.remove(0);
32808        let commit = super::DecommissionCancelCommit {
32809            previous_start_time: Some(old_generation),
32810            previous_queued: false,
32811            previous_last_update,
32812            canceled_pool,
32813        };
32814        let mut current = PoolMeta {
32815            pools: vec![decommission_test_pool_status(
32816                0,
32817                Some(PoolDecommissionInfo {
32818                    start_time: Some(new_generation),
32819                    ..Default::default()
32820                }),
32821            )],
32822            ..Default::default()
32823        };
32824
32825        let err = super::commit_decommission_cancel(&mut current, 0, commit)
32826            .expect_err("an old cancel must not publish over a replacement generation");
32827        assert!(err.to_string().contains("operation generation changed"));
32828        let info = current.pools[0]
32829            .decommission
32830            .as_ref()
32831            .expect("replacement generation should remain present");
32832        assert_eq!(info.start_time, Some(new_generation));
32833        assert!(!info.canceled);
32834    }
32835
32836    #[tokio::test]
32837    async fn test_decommission_cancel_rejects_stale_retry_after_queued_replacement() {
32838        let old_generation = OffsetDateTime::UNIX_EPOCH;
32839        let canceler = DecommissionCanceler::new(CancellationToken::new());
32840        let pool_meta = PoolMeta {
32841            pools: vec![decommission_test_pool_status(
32842                0,
32843                Some(PoolDecommissionInfo {
32844                    start_time: Some(old_generation),
32845                    ..Default::default()
32846                }),
32847            )],
32848            ..Default::default()
32849        };
32850        let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
32851        let queued_replacement = Arc::new(std::sync::Mutex::new(None));
32852        let queued_replacement_for_save = queued_replacement.clone();
32853
32854        store
32855            .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move {
32856                let saved_cancel = snapshot.pools[0]
32857                    .decommission
32858                    .as_ref()
32859                    .expect("saved snapshot should contain decommission metadata");
32860                assert!(saved_cancel.canceled);
32861                assert!(saved_cancel.start_time.is_none());
32862                let mut queued_replacement = snapshot.clone();
32863                let replacement = queued_replacement
32864                    .pools
32865                    .get_mut(0)
32866                    .expect("cancel snapshot should contain the pool");
32867                replacement.last_update += Duration::seconds(1);
32868                let info = replacement
32869                    .decommission
32870                    .as_mut()
32871                    .expect("cancel snapshot should contain decommission metadata");
32872                info.canceled = false;
32873                info.queued = true;
32874                *queued_replacement_for_save
32875                    .lock()
32876                    .expect("queued replacement lock should not be poisoned") = Some(queued_replacement);
32877                Ok(())
32878            })
32879            .await
32880            .expect("cancel should commit before a queued replacement is installed");
32881        assert!(store.decommission_cancelers.read().await[0].is_none());
32882        assert!(!canceler.is_active());
32883        assert!(canceler.is_cancelled());
32884
32885        let queued_replacement = queued_replacement
32886            .lock()
32887            .expect("queued replacement lock should not be poisoned")
32888            .take()
32889            .expect("save should prepare the queued replacement");
32890        *store.pool_meta.write().await = queued_replacement;
32891
32892        let replacement_revision = {
32893            let pool_meta = store.pool_meta.read().await;
32894            let info = pool_meta.pools[0]
32895                .decommission
32896                .as_ref()
32897                .expect("queued replacement should remain present");
32898            assert!(info.queued);
32899            assert!(!info.canceled);
32900            assert!(info.start_time.is_none());
32901            pool_meta.pools[0].last_update
32902        };
32903
32904        store.retry_decommission_cancel_for_operation(0, &canceler).await;
32905
32906        let pool_meta = store.pool_meta.read().await;
32907        let info = pool_meta.pools[0]
32908            .decommission
32909            .as_ref()
32910            .expect("stale retry must preserve the queued replacement");
32911        assert_eq!(pool_meta.pools[0].last_update, replacement_revision);
32912        assert!(info.queued);
32913        assert!(!info.canceled);
32914        assert!(info.start_time.is_none());
32915    }
32916
32917    #[tokio::test]
32918    async fn test_decommission_failed_save_failure_preserves_owner_until_retry_succeeds() {
32919        let canceler = DecommissionCanceler::new(CancellationToken::new());
32920        let pool_meta = PoolMeta {
32921            pools: vec![decommission_test_pool_status(
32922                0,
32923                Some(PoolDecommissionInfo {
32924                    start_time: Some(OffsetDateTime::UNIX_EPOCH),
32925                    ..Default::default()
32926                }),
32927            )],
32928            ..Default::default()
32929        };
32930        let store = decommission_worker_test_store(pool_meta, vec![Some(canceler.clone())]);
32931
32932        store
32933            .decommission_failed_with_owner_and_save(0, Some(&canceler), async { Err(Error::SlowDown) })
32934            .await
32935            .expect_err("injected terminal save failure should be returned");
32936
32937        {
32938            let cancelers = store.decommission_cancelers.read().await;
32939            let current = cancelers[0].as_ref().expect("failed save must retain the exact owner slot");
32940            assert!(current.owns_same_operation(&canceler));
32941            assert!(current.is_active());
32942        }
32943        {
32944            let pool_meta = store.pool_meta.read().await;
32945            let info = pool_meta.pools[0]
32946                .decommission
32947                .as_ref()
32948                .expect("rollback must retain active decommission metadata");
32949            assert!(info.has_decommission_state());
32950            assert!(!info.failed);
32951            assert!(!info.complete);
32952            assert!(!info.canceled);
32953        }
32954        assert!(store.decommission_terminal_retryable_for_operation(0, &canceler).await);
32955
32956        store
32957            .decommission_failed_with_owner_and_save(0, Some(&canceler), async { Ok(()) })
32958            .await
32959            .expect("terminal retry should commit");
32960
32961        let pool_meta = store.pool_meta.read().await;
32962        assert!(
32963            pool_meta.pools[0]
32964                .decommission
32965                .as_ref()
32966                .expect("terminal metadata should remain")
32967                .failed
32968        );
32969        drop(pool_meta);
32970        assert!(store.decommission_cancelers.read().await[0].is_none());
32971        assert!(!canceler.is_active());
32972        assert!(canceler.is_cancelled());
32973        assert_eq!(store.ctx.data_movement_operation_epoch(), 1);
32974    }
32975
32976    #[test]
32977    fn test_stale_decommission_operation_cannot_cancel_replacement() {
32978        let stale = DecommissionCanceler::new(CancellationToken::new());
32979        let replacement = DecommissionCanceler::new(CancellationToken::new());
32980        let cancelers = vec![Some(replacement.clone())];
32981        let mut pool_meta = PoolMeta {
32982            pools: vec![decommission_test_pool_status(
32983                0,
32984                Some(PoolDecommissionInfo {
32985                    start_time: Some(OffsetDateTime::UNIX_EPOCH),
32986                    ..Default::default()
32987                }),
32988            )],
32989            ..Default::default()
32990        };
32991
32992        let changed = update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, 0, Some(&stale), |pool_meta| {
32993            pool_meta.decommission_cancel(0)
32994        });
32995
32996        assert!(changed.is_none());
32997        assert!(
32998            !pool_meta.pools[0]
32999                .decommission
33000                .as_ref()
33001                .expect("replacement metadata should remain")
33002                .canceled
33003        );
33004        assert!(replacement.is_active());
33005        assert!(!replacement.is_cancelled());
33006        assert!(!stale.is_active());
33007        assert!(stale.is_cancelled());
33008    }
33009
33010    #[test]
33011    fn test_ensure_decommission_routines_scheduled_accepts_positive_bound_count() {
33012        assert!(super::ensure_decommission_routines_scheduled(2, 2).is_ok());
33013    }
33014
33015    #[test]
33016    fn test_ensure_decommission_routines_scheduled_rejects_zero_bound_count() {
33017        let err = super::ensure_decommission_routines_scheduled(0, 1).expect_err("zero bound count should be rejected");
33018        assert!(
33019            err.to_string()
33020                .contains("failed to start decommission routines: scheduled 0 of 1 expected workers")
33021        );
33022    }
33023
33024    #[test]
33025    fn test_ensure_decommission_routines_scheduled_rejects_partial_binding() {
33026        let err = super::ensure_decommission_routines_scheduled(1, 2).expect_err("partial binding should be rejected");
33027        assert!(
33028            err.to_string()
33029                .contains("failed to start decommission routines: scheduled 1 of 2 expected workers")
33030        );
33031    }
33032
33033    #[test]
33034    #[cfg(windows)]
33035    fn test_path2_bucket_object_with_base_path_supports_windows_separators() {
33036        let (bucket, object) = super::path2_bucket_object_with_base_path("C:\\data", "C:\\data\\my-bucket\\nested\\object.txt");
33037
33038        assert_eq!(bucket, "my-bucket");
33039        assert_eq!(object, "nested/object.txt");
33040    }
33041}