Skip to main content

rustfs_ecstore/store/
object.rs

1// Copyright 2024 RustFS Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use super::*;
16use crate::bucket::lifecycle::{
17    bucket_lifecycle_ops::{enqueue_committed_free_versions, eval_action_from_lifecycle},
18    get_expiry_configs,
19    tier_delete_journal::{
20        ActiveTierDeleteDispatch, EVENT_LIFECYCLE_TIER_DELETE_JOURNAL, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_LIFECYCLE,
21        TierDeleteChunkParentInspection, complete_tier_delete_chunk_parent, complete_tier_delete_dispatch,
22        inspect_tier_delete_chunk_parent, prepare_tier_delete_chunk_dispatch, prepare_tier_delete_dispatch,
23        record_tier_delete_journal_backend_identity, resume_tier_delete_dispatch, tier_delete_dispatch_batch_limit,
24        tier_delete_journal_object_name, tier_delete_source_matches_dispatch_scope,
25    },
26    tier_sweeper::{
27        Jentry, TierDeleteSourceIdentity, attach_tier_delete_source, transitioned_delete_journal_entry_for_source,
28        transitioned_force_delete_journal_entry,
29    },
30};
31use crate::bucket::metadata_sys::{
32    acquire_bucket_metadata_transaction_read_lock_in, get_bucket_incarnation_id_in, get_cached_bucket_incarnation_id_in,
33    get_object_lock_config_and_incarnation_from_disk_in,
34};
35use crate::bucket::object_lock::objectlock_sys::{
36    check_object_lock_for_deletion_with_state, ensure_recursive_force_delete_allowed_for_state,
37};
38use crate::bucket::replication::{DeleteReplicationConfigSnapshot, ReplicationObjectBridge};
39use crate::bucket::versioning::VersioningApi;
40use crate::core::pools::{DecommissionCapacityAdmission, DecommissionCapacityOwner, ensure_decommission_capacity_mutation_id};
41use crate::diagnostics::object_lock::ObjectLockAttempt;
42use crate::disk::OldCurrentSize;
43use crate::object_api::{
44    NamespaceLockFence, ObjectLockConfigSnapshot, ScannerPublicationCommitScopeGuard, ScannerPublicationCommitState,
45    TierFreeVersionReceiptSink,
46};
47use crate::services::notification_sys::{
48    TierDeleteJournalFleetProofToken, acquire_tier_delete_journal_fleet_proof, tier_delete_journal_fleet_proof_matches,
49    tier_delete_journal_topology_generation,
50};
51use crate::services::tier::tier::{TierConfigMgr, TierDestinationId, TierOperationLease, tier_destination_id_from_metadata};
52use crate::set_disk::{
53    SetDisks, get_lock_acquire_timeout, get_object_lock_diag_slow_acquire_threshold, get_object_lock_diag_slow_hold_threshold,
54    is_lock_optimization_enabled, is_object_lock_diag_enabled,
55};
56use crate::storage_api_contracts::{
57    list::ListOperations as _,
58    namespace::NamespaceLocking as _,
59    object::{DeleteAccounting, ObjectIO as _, ObjectOperations as _},
60};
61use futures::StreamExt as _;
62use parking_lot::Mutex as ParkingMutex;
63use rustfs_filemeta::ObjectPartInfo;
64use rustfs_io_metrics::{
65    record_object_lock_diag_acquire_duration, record_object_lock_diag_hold_duration, record_object_lock_diag_slow_acquire,
66    record_object_lock_diag_slow_hold,
67};
68use std::{
69    fmt,
70    pin::Pin,
71    sync::atomic::{AtomicBool, Ordering},
72    task::{Context, Poll},
73    time::{Duration, Instant},
74};
75use tokio::io::{AsyncRead, ReadBuf};
76
77#[cfg(not(test))]
78const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 1000;
79#[cfg(test)]
80const RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE: i32 = 2;
81const RESTORE_WORKER_LOCK_PREFIX: &str = "ilm/restore-worker-locks";
82const RESTORE_WORKER_LOCK_PROBE_TIMEOUT: Duration = Duration::from_millis(50);
83const TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY: usize = 16;
84
85fn install_tier_free_version_receipt_sink(opts: &mut ObjectOptions) -> Option<TierFreeVersionReceiptSink> {
86    if opts.tier_free_version_receipt_sink.is_some() || opts.skip_free_version || opts.delete_prefix {
87        return None;
88    }
89
90    let sink = TierFreeVersionReceiptSink::new();
91    opts.tier_free_version_receipt_sink = Some(sink.clone());
92    Some(sink)
93}
94
95async fn enqueue_recorded_tier_free_versions(store: &ECStore, sink: Option<TierFreeVersionReceiptSink>) -> usize {
96    let Some(sink) = sink else {
97        return 0;
98    };
99    let Ok(receipts) = sink.drain() else {
100        return 0;
101    };
102    enqueue_committed_free_versions(store, receipts).await
103}
104
105fn build_tier_delete_journal_entry(
106    bucket: &str,
107    object: &str,
108    opts: &ObjectOptions,
109    source: &ObjectInfo,
110) -> Result<Option<Jentry>> {
111    let version_id = opts.version_id.as_deref().map(Uuid::parse_str).transpose()?;
112    let source_object = decode_dir_object(object);
113    let entry = if opts.delete_prefix {
114        transitioned_force_delete_journal_entry(&source.transitioned_object, source.transition_version_state).map(|mut je| {
115            attach_tier_delete_source(&mut je, bucket, source_object.as_str(), source, opts.versioned, opts.version_suspended);
116            je
117        })
118    } else {
119        transitioned_delete_journal_entry_for_source(
120            version_id,
121            opts.versioned,
122            opts.version_suspended,
123            bucket,
124            source_object.as_str(),
125            source,
126        )
127    };
128    let Some(mut je) = entry else {
129        let lifecycle_exact_object_delete_all = opts.delete_prefix_object && opts.lifecycle_delete_all.is_some();
130        if opts.delete_prefix
131            && !lifecycle_exact_object_delete_all
132            && source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE
133        {
134            return Err(Error::other(
135                "recursive prefix delete cannot preserve a transitioned source with legacy Unknown remote-version state",
136            ));
137        }
138        return Ok(None);
139    };
140    record_tier_delete_journal_backend_identity(&mut je, &source.user_defined).map_err(Error::other)?;
141    if opts.delete_prefix && !je.can_replace_tier_free_version() {
142        return Err(Error::other(
143            "recursive prefix delete requires a stable transitioned source and destination identity",
144        ));
145    }
146    Ok(Some(je))
147}
148
149async fn prepare_prefix_tier_delete_journal_entries(
150    api: &Arc<ECStore>,
151    bucket: &str,
152    prefix: &str,
153    opts: &ObjectOptions,
154) -> Result<PreparedPrefixTierDelete> {
155    Box::pin(prepare_prefix_tier_delete_journal_entries_inner(api, bucket, prefix, opts)).await
156}
157
158type TierDeleteLeaseReference = (String, Option<TierDestinationId>);
159
160fn tier_delete_walk_cancellation_is_expected(truncated: bool, limit_cancellation: bool, error: &Error) -> bool {
161    truncated && limit_cancellation && matches!(error, Error::OperationCanceled)
162}
163
164fn combine_tier_delete_walk_results(results: impl IntoIterator<Item = Result<()>>) -> Result<()> {
165    let mut cancelled = false;
166    for result in results {
167        match result {
168            Ok(()) => {}
169            Err(Error::OperationCanceled) => cancelled = true,
170            Err(err) => return Err(err),
171        }
172    }
173    if cancelled { Err(Error::OperationCanceled) } else { Ok(()) }
174}
175
176async fn acquire_prefix_tier_delete_reference_leases(
177    api: &Arc<ECStore>,
178    tier_references: &std::collections::HashSet<TierDeleteLeaseReference>,
179) -> Result<Vec<TierOperationLease>> {
180    let mut tier_references = tier_references.iter().cloned().collect::<Vec<_>>();
181    tier_references.sort_unstable();
182    let mut leases = Vec::with_capacity(tier_references.len());
183    for (tier_name, backend_identity) in tier_references {
184        let lease = match backend_identity {
185            Some(backend_identity) => {
186                TierConfigMgr::acquire_operation_lease_for_backend_identity(&api.tier_config_mgr(), &tier_name, backend_identity)
187                    .await
188            }
189            None => TierConfigMgr::acquire_operation_lease(&api.tier_config_mgr(), &tier_name).await,
190        }
191        .map_err(Error::other)?;
192        leases.push(lease);
193    }
194    Ok(leases)
195}
196
197async fn acquire_prefix_tier_delete_leases(api: &Arc<ECStore>, entries: &[Jentry]) -> Result<Vec<TierOperationLease>> {
198    let tier_references = entries
199        .iter()
200        .map(|entry| (entry.tier_name.clone(), entry.backend_identity))
201        .collect::<std::collections::HashSet<_>>();
202    acquire_prefix_tier_delete_reference_leases(api, &tier_references).await
203}
204
205async fn prepare_prefix_tier_delete_journal_entries_inner(
206    api: &Arc<ECStore>,
207    bucket: &str,
208    prefix: &str,
209    opts: &ObjectOptions,
210) -> Result<PreparedPrefixTierDelete> {
211    let (chunk_parent_active, legacy_manifest_active, chunk_parent_topology_generation) = if is_meta_bucketname(bucket) {
212        (false, false, None)
213    } else {
214        let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
215        let bucket_fence = opts
216            .bucket_lifecycle_lock_fence
217            .as_ref()
218            .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
219        match Box::pin(inspect_tier_delete_chunk_parent(
220            Arc::clone(api),
221            bucket,
222            bucket_incarnation,
223            prefix,
224            bucket_fence,
225        ))
226        .await?
227        {
228            TierDeleteChunkParentInspection::NoParent => (false, false, None),
229            TierDeleteChunkParentInspection::LegacyManifest => (false, true, None),
230            TierDeleteChunkParentInspection::Ready(topology_generation) => (true, false, Some(topology_generation)),
231            TierDeleteChunkParentInspection::Resume(dispatch) => {
232                let leases = acquire_prefix_tier_delete_leases(api, dispatch.entries()?).await?;
233                return Ok(PreparedPrefixTierDelete {
234                    dispatch: Some(*dispatch),
235                    chunk_parent_active: true,
236                    chunk_parent_fleet_proof: None,
237                    _leases: leases,
238                });
239            }
240            TierDeleteChunkParentInspection::RetryRequired => {
241                return Err(Error::other("tier delete chunk parent made durable progress; retry the next batch"));
242            }
243        }
244    };
245    let mut tier_references = std::collections::HashSet::<(String, Option<TierDestinationId>)>::new();
246    let mut entries_by_name = std::collections::BTreeMap::new();
247    let batch_limit = tier_delete_dispatch_batch_limit();
248    let logical_prefix = decode_dir_object(prefix);
249    let exact_object = opts.delete_prefix_object.then(|| logical_prefix.clone());
250    let physical_sets = api
251        .pools
252        .iter()
253        .flat_map(|pool| pool.disk_set.iter().cloned())
254        .collect::<Vec<_>>();
255    let (tx, mut rx) = tokio::sync::mpsc::channel::<ObjectInfoOrErr>(100);
256    let cancellation = tokio_util::sync::CancellationToken::new();
257    let limit_cancellation = Arc::new(AtomicBool::new(false));
258    let walk_cancel = cancellation.clone();
259    let bucket_owned = bucket.to_string();
260    let prefix_owned = prefix.to_string();
261    let walk = async move {
262        let results = futures::stream::iter(physical_sets.into_iter().map(|set| {
263            let tx = tx.clone();
264            let cancellation = walk_cancel.clone();
265            let bucket = bucket_owned.clone();
266            let prefix = prefix_owned.clone();
267            async move {
268                let result = set
269                    .walk(
270                        cancellation.clone(),
271                        &bucket,
272                        &prefix,
273                        tx,
274                        WalkOptions {
275                            include_free_versions: true,
276                            walkdir_timeout: Some(Duration::ZERO),
277                            ..Default::default()
278                        },
279                    )
280                    .await;
281                if result.is_err() {
282                    cancellation.cancel();
283                }
284                result
285            }
286        }))
287        .buffer_unordered(4)
288        .collect::<Vec<_>>()
289        .await;
290        drop(tx);
291        combine_tier_delete_walk_results(results)
292    };
293    let collect_limit_cancellation = limit_cancellation.clone();
294    let collect = async {
295        let mut truncated = false;
296        while let Some(result) = rx.recv().await {
297            if let Some(err) = result.err {
298                // Once limit + 1 has been observed this request can authorize
299                // only the exact retained batch; it cannot infer prefix
300                // absence or run the raw delete. Drain only the explicit
301                // cancellation fallout; a real walker error must still fail
302                // the request even when another set reached the limit first.
303                if tier_delete_walk_cancellation_is_expected(truncated, collect_limit_cancellation.load(Ordering::Acquire), &err)
304                {
305                    continue;
306                }
307                cancellation.cancel();
308                return Err(err);
309            }
310            if truncated {
311                continue;
312            }
313            let Some(source) = result.item else {
314                continue;
315            };
316            let object = decode_dir_object(&source.name);
317            if exact_object.as_ref().is_some_and(|expected| expected != &object) {
318                continue;
319            }
320            if exact_object.is_none() && !object.starts_with(&logical_prefix) {
321                continue;
322            }
323            if source.transitioned_object.free_version {
324                cancellation.cancel();
325                return Err(Error::other(
326                    "recursive prefix delete cannot discard an existing tier free-version cleanup obligation",
327                ));
328            }
329            let tier_reference = if source.transitioned_object.status == rustfs_filemeta::TRANSITION_COMPLETE {
330                let backend_identity = tier_destination_id_from_metadata(&source.user_defined).map_err(Error::other)?;
331                Some((source.transitioned_object.tier.clone(), backend_identity))
332            } else {
333                None
334            };
335            if let Some(entry) = build_tier_delete_journal_entry(bucket, &object, opts, &source)? {
336                let name = tier_delete_journal_object_name(&entry);
337                let at_limit = entries_by_name.len() == batch_limit;
338                match entries_by_name.entry(name) {
339                    std::collections::btree_map::Entry::Occupied(_) => {}
340                    std::collections::btree_map::Entry::Vacant(_) if at_limit => {
341                        truncated = true;
342                        collect_limit_cancellation.store(true, Ordering::Release);
343                        cancellation.cancel();
344                    }
345                    std::collections::btree_map::Entry::Vacant(slot) => {
346                        if let Some(tier_reference) = tier_reference {
347                            tier_references.insert(tier_reference);
348                        }
349                        slot.insert(entry);
350                    }
351                }
352            } else if let Some(tier_reference) = tier_reference {
353                tier_references.insert(tier_reference);
354            }
355        }
356        Ok(truncated)
357    };
358    let (walk_result, collect_result) = tokio::join!(walk, collect);
359    let truncated = collect_result?;
360    // A truncated walk normally reports OperationCanceled from the physical
361    // walkers. That cancellation is expected; any other result is a genuine
362    // scan failure and cannot be hidden by the bounded batch.
363    if let Err(err) = walk_result
364        && !tier_delete_walk_cancellation_is_expected(truncated, limit_cancellation.load(Ordering::Acquire), &err)
365    {
366        return Err(err);
367    }
368    let entries = entries_by_name.into_values().collect::<Vec<_>>();
369    let mut leased_tier_references = tier_references;
370    let mut leases = acquire_prefix_tier_delete_reference_leases(api, &leased_tier_references).await?;
371    if entries.is_empty() {
372        let chunk_parent_fleet_proof = if let Some(expected_topology) = chunk_parent_topology_generation.as_deref() {
373            let fleet_proof = acquire_tier_delete_journal_fleet_proof()
374                .ok_or_else(|| Error::other("tier delete chunk parent fleet capability is unavailable"))?;
375            if tier_delete_journal_topology_generation(&fleet_proof) != expected_topology {
376                return Err(Error::other("tier delete chunk parent topology changed during final source scan"));
377            }
378            Some(fleet_proof)
379        } else {
380            None
381        };
382        return Ok(PreparedPrefixTierDelete {
383            dispatch: None,
384            chunk_parent_active,
385            chunk_parent_fleet_proof,
386            _leases: leases,
387        });
388    }
389
390    let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
391    let bucket_fence = opts
392        .bucket_lifecycle_lock_fence
393        .as_ref()
394        .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
395    let fleet_proof = acquire_tier_delete_journal_fleet_proof()
396        .ok_or_else(|| Error::other("tier delete journal v6 fleet capability is unavailable"))?;
397    if chunk_parent_topology_generation
398        .as_deref()
399        .is_some_and(|expected| tier_delete_journal_topology_generation(&fleet_proof) != expected)
400    {
401        return Err(Error::other("tier delete chunk parent topology changed during source scan"));
402    }
403    let mut dispatch = if !legacy_manifest_active && (chunk_parent_active || truncated) {
404        Box::pin(prepare_tier_delete_chunk_dispatch(
405            Arc::clone(api),
406            bucket,
407            bucket_incarnation,
408            prefix,
409            entries,
410            truncated && !chunk_parent_active,
411            fleet_proof,
412            bucket_fence,
413        ))
414        .await?
415    } else if legacy_manifest_active && truncated {
416        resume_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence)
417            .await?
418    } else {
419        prepare_tier_delete_dispatch(Arc::clone(api), bucket, bucket_incarnation, prefix, entries, fleet_proof, bucket_fence)
420            .await?
421    };
422    // A resumed legacy authorization may own predecessors that are absent
423    // from this bounded scan. Pin every backend generation in the actual
424    // permit before any local mutation, while avoiding duplicate leases for
425    // entries already covered by the scan.
426    let additional_tier_references = dispatch
427        .entries()?
428        .iter()
429        .map(|entry| (entry.tier_name.clone(), entry.backend_identity))
430        .filter(|reference| leased_tier_references.insert(reference.clone()))
431        .collect::<std::collections::HashSet<_>>();
432    leases.extend(acquire_prefix_tier_delete_reference_leases(api, &additional_tier_references).await?);
433    if legacy_manifest_active && truncated {
434        dispatch.require_exact_predecessor_replay();
435    }
436    Ok(PreparedPrefixTierDelete {
437        dispatch: Some(dispatch),
438        chunk_parent_active: chunk_parent_active || truncated,
439        chunk_parent_fleet_proof: None,
440        _leases: leases,
441    })
442}
443
444struct PreparedPrefixTierDelete {
445    dispatch: Option<crate::bucket::lifecycle::tier_delete_journal::PreparedTierDeleteDispatch>,
446    chunk_parent_active: bool,
447    chunk_parent_fleet_proof: Option<TierDeleteJournalFleetProofToken>,
448    _leases: Vec<TierOperationLease>,
449}
450
451#[cfg(all(test, feature = "test-util"))]
452struct PrefixDeleteAfterJournalPrepareBarrierState {
453    bucket: String,
454    prefix: String,
455    arrived: tokio::sync::Notify,
456    release: tokio::sync::Notify,
457}
458
459#[cfg(all(test, feature = "test-util"))]
460pub(crate) struct PrefixDeleteAfterJournalPrepareBarrier {
461    state: Arc<PrefixDeleteAfterJournalPrepareBarrierState>,
462}
463
464#[cfg(all(test, feature = "test-util"))]
465static PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER: std::sync::OnceLock<
466    std::sync::Mutex<Option<Arc<PrefixDeleteAfterJournalPrepareBarrierState>>>,
467> = std::sync::OnceLock::new();
468
469#[cfg(all(test, feature = "test-util"))]
470impl PrefixDeleteAfterJournalPrepareBarrier {
471    pub(crate) fn install(bucket: &str, prefix: &str) -> Self {
472        let state = Arc::new(PrefixDeleteAfterJournalPrepareBarrierState {
473            bucket: bucket.to_string(),
474            prefix: prefix.to_string(),
475            arrived: tokio::sync::Notify::new(),
476            release: tokio::sync::Notify::new(),
477        });
478        let mut slot = PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER
479            .get_or_init(|| std::sync::Mutex::new(None))
480            .lock()
481            .expect("prefix journal barrier mutex should not poison");
482        assert!(slot.is_none(), "prefix journal barrier must not already be installed");
483        *slot = Some(Arc::clone(&state));
484        Self { state }
485    }
486
487    pub(crate) async fn wait_until_paused(&self) {
488        self.state.arrived.notified().await;
489    }
490
491    pub(crate) fn release(&self) {
492        self.state.release.notify_one();
493    }
494}
495
496#[cfg(all(test, feature = "test-util"))]
497impl Drop for PrefixDeleteAfterJournalPrepareBarrier {
498    fn drop(&mut self) {
499        self.state.release.notify_one();
500        if let Some(slot) = PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER.get() {
501            let mut slot = slot.lock().expect("prefix journal barrier mutex should not poison");
502            if slot.as_ref().is_some_and(|installed| Arc::ptr_eq(installed, &self.state)) {
503                *slot = None;
504            }
505        }
506    }
507}
508
509#[cfg(all(test, feature = "test-util"))]
510async fn pause_prefix_delete_after_journal_prepare(bucket: &str, prefix: &str) {
511    let state = PREFIX_DELETE_AFTER_JOURNAL_PREPARE_BARRIER
512        .get_or_init(|| std::sync::Mutex::new(None))
513        .lock()
514        .expect("prefix journal barrier mutex should not poison")
515        .as_ref()
516        .filter(|state| state.bucket == bucket && state.prefix == prefix)
517        .cloned();
518    if let Some(state) = state {
519        state.arrived.notify_one();
520        state.release.notified().await;
521    }
522}
523
524async fn delete_prefix_with_tier_delete_journal(
525    store: &ECStore,
526    bucket: &str,
527    object: &str,
528    opts: &ObjectOptions,
529    tier_journal_api: Option<&Arc<ECStore>>,
530) -> Result<()> {
531    let Some(api) = tier_journal_api else {
532        return store.delete_prefix(bucket, object, opts).await;
533    };
534    let PreparedPrefixTierDelete {
535        dispatch,
536        chunk_parent_active,
537        chunk_parent_fleet_proof,
538        _leases,
539    } = prepare_prefix_tier_delete_journal_entries(api, bucket, object, opts).await?;
540    let Some(dispatch) = dispatch else {
541        // There is no new remote-cleanup candidate. Keep compatibility-path
542        // tier leases and, for a chunked final pass, the matching parent fleet
543        // proof alive until local deletion and parent completion both commit.
544        let _tier_leases = _leases;
545        let mut operation_opts = opts.clone();
546        // `tier_delete_journal_api` means a v6 dispatch authorization must be
547        // present at every destructive phase.  With no v6 candidate, clear
548        // that marker so ordinary objects use the local path and legacy
549        // transitioned metadata retains its FreeVersion fallback.
550        operation_opts.tier_delete_journal_api = None;
551        operation_opts.tier_delete_dispatch_authorization = None;
552        let parent_fleet_proof = if chunk_parent_active {
553            Some(
554                chunk_parent_fleet_proof
555                    .as_ref()
556                    .filter(|proof| tier_delete_journal_fleet_proof_matches(proof))
557                    .ok_or_else(|| Error::other("tier delete chunk parent fleet proof changed before final deletion"))?,
558            )
559        } else {
560            None
561        };
562        store.delete_prefix(bucket, object, &operation_opts).await?;
563        if let Some(parent_fleet_proof) = parent_fleet_proof {
564            #[cfg(all(test, feature = "test-util"))]
565            crate::bucket::lifecycle::tier_delete_journal::tier_delete_chunk_test_pause(
566                crate::bucket::lifecycle::tier_delete_journal::TierDeleteChunkTestStage::FinalLocalDeletionCompleted,
567            )
568            .await;
569            let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
570            let bucket_fence = opts
571                .bucket_lifecycle_lock_fence
572                .as_ref()
573                .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
574            if !Box::pin(complete_tier_delete_chunk_parent(
575                Arc::clone(api),
576                bucket,
577                bucket_incarnation,
578                object,
579                bucket_fence,
580                parent_fleet_proof,
581            ))
582            .await?
583            {
584                return Err(Error::other("tier delete chunk parent disappeared after final local deletion"));
585            }
586        }
587        return Ok(());
588    };
589    let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
590    let bucket_fence = opts
591        .bucket_lifecycle_lock_fence
592        .as_ref()
593        .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
594    let active = dispatch.consume(bucket, bucket_incarnation, object)?;
595    let mut operation_opts = opts.clone();
596    operation_opts.tier_delete_dispatch_authorization = Some(active.authorization());
597
598    #[cfg(all(test, feature = "test-util"))]
599    pause_prefix_delete_after_journal_prepare(bucket, object).await;
600
601    // Keep every backend generation lease until the whole local operation has
602    // either committed its journal set or returned an ambiguous mutation.
603    let _tier_leases = _leases;
604    if active.predecessor_replay_required() || active.is_chunked() {
605        replay_authorized_tier_delete_sources(store, bucket, object, &active, &operation_opts).await?;
606        #[cfg(all(test, feature = "test-util"))]
607        if active.is_chunked() {
608            crate::bucket::lifecycle::tier_delete_journal::tier_delete_chunk_test_pause(
609                crate::bucket::lifecycle::tier_delete_journal::TierDeleteChunkTestStage::LocalReplayCompleted,
610            )
611            .await;
612        }
613        complete_tier_delete_dispatch(Arc::clone(api), &active, bucket_fence).await?;
614        if active.is_chunked() {
615            return Err(Error::other("tier delete chunk completed; retry the next durable batch"));
616        }
617        return Err(Error::other("authorized tier delete predecessor completed; retry the successor dispatch"));
618    }
619    let result = store.delete_prefix(bucket, object, &operation_opts).await;
620    match result {
621        Ok(()) => {
622            if let Err(err) = complete_tier_delete_dispatch(Arc::clone(api), &active, bucket_fence).await {
623                warn!(
624                    event = EVENT_LIFECYCLE_TIER_DELETE_JOURNAL,
625                    component = LOG_COMPONENT_ECSTORE,
626                    subsystem = LOG_SUBSYSTEM_LIFECYCLE,
627                    bucket,
628                    prefix = object,
629                    error = ?err,
630                    "Prefix deletion committed with durable tier cleanup pending"
631                );
632            }
633            Ok(())
634        }
635        // Once DispatchAuthorized exists, every failure is mutation-ambiguous.
636        // Recovery decides from physical proof; never abort the journals here.
637        Err(err) => Err(err),
638    }
639}
640
641async fn replay_authorized_tier_delete_sources(
642    store: &ECStore,
643    bucket: &str,
644    prefix: &str,
645    active: &ActiveTierDeleteDispatch,
646    opts: &ObjectOptions,
647) -> Result<()> {
648    let _publication_scope_guard = opts
649        .scanner_publication_commit_scope
650        .clone()
651        .map(ScannerPublicationCommitScopeGuard::new);
652    let publication_scope = opts.scanner_publication_commit_scope.as_ref();
653    let bucket_incarnation = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
654    let bucket_fence = opts
655        .bucket_lifecycle_lock_fence
656        .as_ref()
657        .ok_or_else(|| Error::other("tier delete dispatch requires a bucket lifecycle write fence"))?;
658    let authorization = active.authorization();
659    authorization.mark_mutation_started(bucket, bucket_incarnation, prefix)?;
660
661    let mut source_objects = std::collections::HashSet::with_capacity(active.entries().len());
662    for entry in active.entries() {
663        let source = entry
664            .source
665            .as_ref()
666            .filter(|source| source.has_stable_identity())
667            .ok_or_else(|| Error::other("authorized tier delete predecessor has no stable source"))?;
668        if !tier_delete_source_matches_replay_scope(source, bucket, prefix, opts.delete_prefix_object) {
669            return Err(Error::other("authorized tier delete predecessor source escaped its prefix scope"));
670        }
671        source_objects.insert(source.object.as_str());
672    }
673
674    if let Some(scope) = publication_scope {
675        if scope.state() == ScannerPublicationCommitState::Admitted {
676            scope
677                .try_begin()
678                .map_err(|_| Error::other("scanner publication predecessor replay scope cannot start"))?;
679        }
680        if !scope.can_commit() {
681            let _ = scope.mark_indeterminate();
682            return Err(StorageError::OperationCanceled);
683        }
684    }
685
686    let stopped = Arc::new(AtomicBool::new(false));
687    // The caller holds bucket lifecycle WRITE. Each bounded future acquires
688    // only one logical object's physical lock set and releases it before
689    // completion; no future nests locks for two object keys.
690    let make_replay = |object: String| {
691        let stopped = stopped.clone();
692        let authorization = authorization.clone();
693        async move {
694            if stopped.load(Ordering::Acquire) {
695                return Ok::<_, Error>(0usize);
696            }
697            let result = async {
698                if bucket_fence.is_lock_lost() {
699                    return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay"));
700                }
701                let encoded_object = encode_dir_object(&object);
702                let guards = if opts.delete_prefix_object {
703                    store
704                        .acquire_remaining_physical_object_write_locks(
705                            "tier_delete_dispatch_predecessor_replay",
706                            bucket,
707                            &encoded_object,
708                        )
709                        .await?
710                } else {
711                    store
712                        .acquire_all_physical_object_write_locks(
713                            "tier_delete_dispatch_predecessor_replay",
714                            bucket,
715                            &encoded_object,
716                        )
717                        .await?
718                };
719                authorization.ensure_current(bucket, bucket_incarnation, prefix)?;
720                if publication_scope.is_some_and(|scope| !scope.can_commit()) {
721                    return Err(StorageError::OperationCanceled);
722                }
723                let mut replay_opts = opts.clone();
724                replay_opts.no_lock = true;
725                replay_opts.delete_prefix = false;
726                replay_opts.delete_prefix_object = false;
727                for guard in &guards {
728                    guard.add_namespace_lock_fence(&mut replay_opts);
729                }
730                let mut deleted = 0usize;
731                for pool in &store.pools {
732                    for set in &pool.disk_set {
733                        authorization.ensure_current(bucket, bucket_incarnation, prefix)?;
734                        deleted = deleted
735                            .checked_add(
736                                set.replay_authorized_tier_delete_sources(bucket, &object, &authorization, &replay_opts)
737                                    .await?,
738                            )
739                            .ok_or_else(|| Error::other("tier delete dispatch replay count overflow"))?;
740                    }
741                }
742                if bucket_fence.is_lock_lost() || guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
743                    return Err(Error::other("tier delete dispatch namespace fence was lost during predecessor replay"));
744                }
745                Ok(deleted)
746            }
747            .await;
748            if result.is_err() {
749                stopped.store(true, Ordering::Release);
750            }
751            result
752        }
753    };
754    let mut objects = source_objects.into_iter();
755    let mut replays = futures::stream::FuturesUnordered::new();
756    for _ in 0..TIER_DELETE_DISPATCH_LOCAL_REPLAY_CONCURRENCY {
757        let Some(object) = objects.next().map(ToOwned::to_owned) else {
758            break;
759        };
760        replays.push(make_replay(object));
761    }
762    let mut deleted = 0usize;
763    let mut first_error = None;
764    while let Some(result) = replays.next().await {
765        match result {
766            Ok(count) => {
767                deleted = deleted
768                    .checked_add(count)
769                    .ok_or_else(|| Error::other("tier delete dispatch replay count overflow"))?;
770            }
771            Err(err) if first_error.is_none() => first_error = Some(err),
772            Err(_) => {}
773        }
774        if first_error.is_none()
775            && !stopped.load(Ordering::Acquire)
776            && let Some(object) = objects.next().map(ToOwned::to_owned)
777        {
778            replays.push(make_replay(object));
779        }
780    }
781    if deleted > 0 {
782        super::list_objects::observe_list_objects_mutation(store, bucket).await;
783    }
784    if let Some(err) = first_error {
785        if publication_scope.is_some_and(|scope| !scope.can_commit())
786            && let Some(scope) = publication_scope
787        {
788            let _ = scope.mark_indeterminate();
789        }
790        return Err(err);
791    }
792    if let Some(scope) = publication_scope {
793        let _ = scope.mark_committed();
794    }
795    Ok(())
796}
797
798fn tier_delete_source_matches_replay_scope(
799    source: &TierDeleteSourceIdentity,
800    bucket: &str,
801    persisted_prefix: &str,
802    exact_object: bool,
803) -> bool {
804    tier_delete_source_matches_dispatch_scope(source, bucket, persisted_prefix)
805        && (!exact_object || source.object == decode_dir_object(persisted_prefix))
806}
807
808async fn delete_recursive_prefix_with_tier_delete_journal(
809    store: &ECStore,
810    bucket: &str,
811    object: &str,
812    opts: &ObjectOptions,
813    tier_journal_api: Option<&Arc<ECStore>>,
814) -> Result<()> {
815    // Prefix deletes cover multiple object keys; an exact lock on the prefix
816    // string would not protect child objects.
817    if !is_meta_bucketname(bucket) {
818        let state = opts
819            .object_lock_config_snapshot
820            .as_deref()
821            .ok_or_else(|| Error::other("recursive delete is missing its Object Lock configuration snapshot"))?
822            .state();
823        ensure_recursive_force_delete_allowed_for_state(bucket, state)?;
824        let bypass_governance = opts
825            .object_lock_delete
826            .as_ref()
827            .is_some_and(|delete_opts| delete_opts.bypass_governance);
828        for pool in &store.pools {
829            for set in &pool.disk_set {
830                let mut marker = None;
831                let mut version_marker = None;
832                loop {
833                    let page = set
834                        .clone()
835                        .inner_list_object_versions_for_recursive_delete(
836                            bucket,
837                            object,
838                            marker.clone(),
839                            version_marker.clone(),
840                            RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE,
841                        )
842                        .await?;
843                    for object_info in &page.objects {
844                        if check_object_lock_for_deletion_with_state(state, object_info, bypass_governance)?.is_some() {
845                            return Err(StorageError::PrefixAccessDenied(bucket.to_string(), object_info.name.clone()));
846                        }
847                    }
848                    if !page.is_truncated {
849                        break;
850                    }
851                    let next_marker = page
852                        .next_marker
853                        .ok_or_else(|| Error::other("recursive delete version scan did not return a continuation marker"))?;
854                    if marker.as_ref() == Some(&next_marker) && version_marker == page.next_version_idmarker {
855                        return Err(Error::other("recursive delete version scan did not advance"));
856                    }
857                    marker = Some(next_marker);
858                    version_marker = page.next_version_idmarker;
859                }
860            }
861        }
862    }
863    // A trailing slash selects a directory, not the object at its parent key.
864    // Raw filesystem recursion would also remove that object's metadata and
865    // data. Preserve it by purging the selected keys individually when they
866    // share this physical directory. The bucket write lock covers both scans.
867    if object.ends_with('/') && !is_meta_bucketname(bucket) {
868        let parent = object.strip_suffix('/').unwrap_or(object);
869        for pool in &store.pools {
870            for set in &pool.disk_set {
871                let page = set
872                    .clone()
873                    .inner_list_object_versions_for_recursive_delete(bucket, parent, None, None, 1)
874                    .await?;
875                if page.objects.iter().any(|info| info.name == parent) {
876                    return delete_directory_keys_with_tier_delete_journal(store, bucket, object, opts, tier_journal_api).await;
877                }
878            }
879        }
880    }
881    delete_prefix_with_tier_delete_journal(store, bucket, object, opts, tier_journal_api).await
882}
883
884async fn delete_directory_keys_with_tier_delete_journal(
885    store: &ECStore,
886    bucket: &str,
887    prefix: &str,
888    opts: &ObjectOptions,
889    tier_journal_api: Option<&Arc<ECStore>>,
890) -> Result<()> {
891    for pool in &store.pools {
892        for set in &pool.disk_set {
893            let mut previous_keys = std::collections::BTreeSet::new();
894            loop {
895                // Restart after each bounded batch: its version markers have
896                // been deleted, and the bucket write lock excludes new keys.
897                let page = set
898                    .clone()
899                    .inner_list_object_versions_for_recursive_delete(
900                        bucket,
901                        prefix,
902                        None,
903                        None,
904                        RECURSIVE_DELETE_VERSION_SCAN_PAGE_SIZE,
905                    )
906                    .await?;
907                let keys = page
908                    .objects
909                    .into_iter()
910                    .map(|info| info.name)
911                    .filter(|key| key.starts_with(prefix))
912                    .collect::<std::collections::BTreeSet<_>>();
913                if keys.is_empty() {
914                    break;
915                }
916                if keys == previous_keys {
917                    return Err(Error::other("directory deletion did not advance"));
918                }
919                for key in &keys {
920                    let encoded_key = encode_dir_object(key);
921                    let mut exact_opts = opts.clone();
922                    exact_opts.delete_prefix_object = true;
923                    let _guard = store
924                        .acquire_object_write_lock_if_needed("delete_object", bucket, &encoded_key, &mut exact_opts)
925                        .await?;
926                    delete_prefix_with_tier_delete_journal(store, bucket, &encoded_key, &exact_opts, tier_journal_api).await?;
927                }
928                previous_keys = keys;
929            }
930        }
931    }
932    Ok(())
933}
934
935/// A GET whose object identity has been resolved while its namespace read lock
936/// remains held, but whose body reader has not been constructed yet.
937///
938/// The application can evaluate request preconditions and cache coordination
939/// against [`Self::object_info`] before consuming this value. Dropping it
940/// releases the read lock without constructing a body reader.
941pub struct PreparedGetObjectReader {
942    pool: Arc<Sets>,
943    bucket: String,
944    object: String,
945    range: Option<HTTPRangeSpec>,
946    headers: HeaderMap,
947    opts: ObjectOptions,
948    metadata: crate::set_disk::PreparedGetObjectMetadata,
949    read_lock_guard: Option<ObjectLockDiagGuard>,
950}
951
952impl PreparedGetObjectReader {
953    /// Returns the fresh metadata snapshot protected by this prepared read.
954    pub fn object_info(&self) -> &ObjectInfo {
955        self.metadata.object_info()
956    }
957
958    /// Finishes a metadata-only decision without constructing a body reader.
959    pub fn into_object_info(mut self) -> ObjectInfo {
960        self.metadata.take_object_info()
961    }
962
963    /// Replaces the headers used when this prepared value constructs its body reader.
964    #[must_use]
965    pub fn with_headers(mut self, headers: HeaderMap) -> Self {
966        self.headers = headers;
967        self
968    }
969
970    /// Constructs the body reader while retaining the metadata snapshot's read
971    /// lock, then transfers that lock to the returned stream as usual. The
972    /// staged caller already performed the authoritative app-layer cache probe,
973    /// so the nested set-disk reader must not probe the same hook again.
974    pub async fn into_reader(self) -> Result<GetObjectReader> {
975        let mut reader =
976            crate::object_api::without_get_object_body_cache_hook(self.pool.get_object_reader_with_prepared_metadata(
977                &self.bucket,
978                &self.object,
979                self.range,
980                self.headers,
981                &self.opts,
982                self.metadata,
983            ))
984            .await?;
985        reader.body_source = crate::object_api::GetObjectBodySource::HookMissed;
986        Ok(ECStore::attach_read_lock_guard(reader, self.read_lock_guard))
987    }
988}
989
990struct LockGuardedReader {
991    inner: Box<dyn AsyncRead + Unpin + Send + Sync>,
992    guard: Option<ObjectLockDiagGuard>,
993}
994
995impl AsyncRead for LockGuardedReader {
996    fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
997        let had_capacity = buf.remaining() > 0;
998        let filled_before = buf.filled().len();
999        let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
1000        if had_capacity && matches!(poll, Poll::Ready(Ok(()))) && buf.filled().len() == filled_before {
1001            self.guard.take();
1002        }
1003        poll
1004    }
1005}
1006
1007#[derive(Clone, Copy, Debug)]
1008enum ObjectLockDiagMode {
1009    Read,
1010    Write,
1011}
1012
1013impl ObjectLockDiagMode {
1014    fn as_str(self) -> &'static str {
1015        match self {
1016            Self::Read => "read",
1017            Self::Write => "write",
1018        }
1019    }
1020}
1021
1022impl fmt::Display for ObjectLockDiagMode {
1023    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
1024        f.write_str(self.as_str())
1025    }
1026}
1027
1028pub(crate) struct ObjectLockDiagGuard {
1029    guard: rustfs_lock::NamespaceLockGuard,
1030    #[cfg(test)]
1031    test_namespace_lock_fence: Option<NamespaceLockFence>,
1032    enabled: bool,
1033    op: &'static str,
1034    bucket: Option<String>,
1035    object: Option<String>,
1036    owner: Option<String>,
1037    mode: ObjectLockDiagMode,
1038    acquired_at: Instant,
1039    _attempt: ObjectLockAttempt,
1040}
1041
1042impl ObjectLockDiagGuard {
1043    fn new(
1044        guard: rustfs_lock::NamespaceLockGuard,
1045        enabled: bool,
1046        op: &'static str,
1047        bucket: Option<String>,
1048        object: Option<String>,
1049        owner: Option<String>,
1050        mode: ObjectLockDiagMode,
1051    ) -> Self {
1052        Self {
1053            guard,
1054            #[cfg(test)]
1055            test_namespace_lock_fence: None,
1056            enabled,
1057            op,
1058            bucket,
1059            object,
1060            owner,
1061            mode,
1062            acquired_at: Instant::now(),
1063            _attempt: ObjectLockAttempt::default(),
1064        }
1065    }
1066
1067    fn with_attempt(mut self, attempt: ObjectLockAttempt) -> Self {
1068        self._attempt = attempt;
1069        self
1070    }
1071
1072    pub(crate) fn lock_lost_signal(&self) -> Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>> {
1073        match &self.guard {
1074            rustfs_lock::NamespaceLockGuard::Standard(guard) => Some(guard.lock_lost()),
1075            rustfs_lock::NamespaceLockGuard::Fast(_) => None,
1076        }
1077    }
1078
1079    pub(crate) fn is_lock_lost(&self) -> bool {
1080        self.guard.is_lock_lost() || {
1081            #[cfg(test)]
1082            {
1083                self.test_namespace_lock_fence
1084                    .as_ref()
1085                    .is_some_and(NamespaceLockFence::is_lock_lost)
1086            }
1087            #[cfg(not(test))]
1088            {
1089                false
1090            }
1091        }
1092    }
1093
1094    pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
1095        opts.ensure_namespace_lock_fence();
1096        if let Some(signal) = self.lock_lost_signal() {
1097            opts.add_namespace_lock_lost_signal(signal);
1098        }
1099        #[cfg(test)]
1100        if let Some(fence) = self.test_namespace_lock_fence.as_ref() {
1101            opts.add_namespace_lock_fence(fence);
1102        }
1103    }
1104}
1105
1106#[cfg(test)]
1107#[derive(Clone, Copy, PartialEq, Eq)]
1108pub(crate) enum DecommissionMutationFenceTestPhase {
1109    Migration,
1110    SourceCleanup,
1111}
1112
1113#[cfg(test)]
1114struct DecommissionMutationFenceLossState {
1115    bucket: String,
1116    object: String,
1117    phase: DecommissionMutationFenceTestPhase,
1118    fence: NamespaceLockFence,
1119    loss_handle: Arc<std::sync::atomic::AtomicBool>,
1120}
1121
1122#[cfg(test)]
1123pub(crate) struct DecommissionMutationFenceLossHook {
1124    state: Arc<DecommissionMutationFenceLossState>,
1125}
1126
1127#[cfg(test)]
1128static DECOMMISSION_MUTATION_FENCE_LOSS_HOOK: std::sync::OnceLock<
1129    std::sync::Mutex<Option<Arc<DecommissionMutationFenceLossState>>>,
1130> = std::sync::OnceLock::new();
1131
1132#[cfg(test)]
1133impl DecommissionMutationFenceLossHook {
1134    pub(crate) fn install(bucket: &str, object: &str, phase: DecommissionMutationFenceTestPhase) -> Self {
1135        let (fence, loss_handle) = NamespaceLockFence::loss_handle_for_test();
1136        let state = Arc::new(DecommissionMutationFenceLossState {
1137            bucket: bucket.to_string(),
1138            object: object.to_string(),
1139            phase,
1140            fence,
1141            loss_handle,
1142        });
1143        let mut slot = DECOMMISSION_MUTATION_FENCE_LOSS_HOOK
1144            .get_or_init(|| std::sync::Mutex::new(None))
1145            .lock()
1146            .expect("decommission mutation fence loss hooks should not poison");
1147        assert!(slot.is_none(), "decommission mutation fence loss hook must be unique");
1148        *slot = Some(Arc::clone(&state));
1149        Self { state }
1150    }
1151
1152    pub(crate) fn mark_lost(&self) {
1153        self.state.loss_handle.store(true, Ordering::Release);
1154    }
1155}
1156
1157#[cfg(test)]
1158impl Drop for DecommissionMutationFenceLossHook {
1159    fn drop(&mut self) {
1160        let mut slot = DECOMMISSION_MUTATION_FENCE_LOSS_HOOK
1161            .get_or_init(|| std::sync::Mutex::new(None))
1162            .lock()
1163            .expect("decommission mutation fence loss hooks should not poison");
1164        if slot.as_ref().is_some_and(|hook| Arc::ptr_eq(hook, &self.state)) {
1165            *slot = None;
1166        }
1167    }
1168}
1169
1170#[cfg(test)]
1171fn decommission_mutation_fence_for_test(
1172    bucket: &str,
1173    object: &str,
1174    phase: DecommissionMutationFenceTestPhase,
1175) -> Option<NamespaceLockFence> {
1176    DECOMMISSION_MUTATION_FENCE_LOSS_HOOK
1177        .get_or_init(|| std::sync::Mutex::new(None))
1178        .lock()
1179        .expect("decommission mutation fence loss hooks should not poison")
1180        .as_ref()
1181        .filter(|hook| hook.bucket == bucket && hook.object == object && hook.phase == phase)
1182        .map(|hook| hook.fence.clone())
1183}
1184
1185#[cfg(all(test, feature = "test-util"))]
1186struct DecommissionFreeVersionSourceRaceState {
1187    bucket: String,
1188    object: String,
1189    arrived: tokio::sync::Notify,
1190    release: tokio::sync::Notify,
1191}
1192
1193#[cfg(all(test, feature = "test-util"))]
1194pub(crate) struct DecommissionFreeVersionSourceRaceBarrier {
1195    state: Arc<DecommissionFreeVersionSourceRaceState>,
1196}
1197
1198#[cfg(all(test, feature = "test-util"))]
1199static DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER: std::sync::OnceLock<
1200    std::sync::Mutex<Option<Arc<DecommissionFreeVersionSourceRaceState>>>,
1201> = std::sync::OnceLock::new();
1202
1203#[cfg(all(test, feature = "test-util"))]
1204impl DecommissionFreeVersionSourceRaceBarrier {
1205    pub(crate) fn install(bucket: &str, object: &str) -> Self {
1206        let state = Arc::new(DecommissionFreeVersionSourceRaceState {
1207            bucket: bucket.to_string(),
1208            object: object.to_string(),
1209            arrived: tokio::sync::Notify::new(),
1210            release: tokio::sync::Notify::new(),
1211        });
1212        let mut slot = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
1213            .get_or_init(|| std::sync::Mutex::new(None))
1214            .lock()
1215            .expect("decommission free-version source race barrier should not poison");
1216        assert!(slot.is_none(), "decommission free-version source race barrier must be unique");
1217        *slot = Some(Arc::clone(&state));
1218        Self { state }
1219    }
1220
1221    pub(crate) async fn wait_until_paused(&self) {
1222        tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
1223            .await
1224            .expect("decommission should pause before acquiring the free-version source lock");
1225    }
1226
1227    pub(crate) fn release(&self) {
1228        self.state.release.notify_one();
1229    }
1230}
1231
1232#[cfg(all(test, feature = "test-util"))]
1233impl Drop for DecommissionFreeVersionSourceRaceBarrier {
1234    fn drop(&mut self) {
1235        self.state.release.notify_one();
1236        let mut slot = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
1237            .get_or_init(|| std::sync::Mutex::new(None))
1238            .lock()
1239            .expect("decommission free-version source race barrier should not poison");
1240        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
1241            *slot = None;
1242        }
1243    }
1244}
1245
1246#[cfg(all(test, feature = "test-util"))]
1247async fn pause_decommission_free_version_before_source_lock(bucket: &str, object: &str) {
1248    let state = DECOMMISSION_FREE_VERSION_SOURCE_RACE_BARRIER
1249        .get_or_init(|| std::sync::Mutex::new(None))
1250        .lock()
1251        .expect("decommission free-version source race barrier should not poison")
1252        .as_ref()
1253        .filter(|state| state.bucket == bucket && state.object == object)
1254        .cloned();
1255    if let Some(state) = state {
1256        state.arrived.notify_one();
1257        state.release.notified().await;
1258    }
1259}
1260
1261pub(crate) struct SourceCleanupMutationFence {
1262    guard: ObjectLockDiagGuard,
1263    source_lock_covered: bool,
1264}
1265
1266impl SourceCleanupMutationFence {
1267    pub(crate) fn source_lock_covered(&self) -> bool {
1268        self.source_lock_covered
1269    }
1270
1271    pub(crate) fn is_lock_lost(&self) -> bool {
1272        self.guard.is_lock_lost()
1273    }
1274
1275    pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
1276        self.guard.add_namespace_lock_fence(opts);
1277    }
1278}
1279
1280/// Opaque write-lock guard for the RestoreObject accept path; see
1281/// [`ECStore::acquire_restore_accept_guard`]. Deliberately not a general lock
1282/// API — it only exists so the accept path's restore-status compare-and-set
1283/// can span the ecstore/API layer boundary.
1284pub struct RestoreAcceptGuard(ObjectLockDiagGuard);
1285
1286impl RestoreAcceptGuard {
1287    /// True when the underlying namespace lock was lost (e.g. heartbeat
1288    /// refresh lost quorum). Callers must check this before committing the
1289    /// restore-status write the guard exists to serialize.
1290    pub fn is_lock_lost(&self) -> bool {
1291        self.0.guard.is_lock_lost()
1292    }
1293
1294    pub fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
1295        self.0.add_namespace_lock_fence(opts);
1296    }
1297}
1298
1299/// Liveness proof for one asynchronous restore generation. Unlike the object
1300/// accept/commit locks, this lock lives in the reserved metadata namespace and
1301/// never blocks reads or unrelated writes of the restored object.
1302pub struct RestoreWorkerGuard(rustfs_lock::NamespaceLockGuard);
1303
1304impl RestoreWorkerGuard {
1305    pub fn is_lock_lost(&self) -> bool {
1306        self.0.is_lock_lost()
1307    }
1308}
1309
1310impl Drop for ObjectLockDiagGuard {
1311    fn drop(&mut self) {
1312        if !self.enabled || self.guard.is_released() {
1313            return;
1314        }
1315
1316        let hold = self.acquired_at.elapsed();
1317        record_object_lock_diag_hold_duration(self.op, self.mode.as_str(), hold);
1318        let threshold = get_object_lock_diag_slow_hold_threshold();
1319        if hold >= threshold {
1320            record_object_lock_diag_slow_hold(self.op, self.mode.as_str());
1321            warn!(
1322                target: "rustfs_ecstore::object_lock_diag",
1323                op = self.op,
1324                bucket = %self.bucket.as_deref().unwrap_or_default(),
1325                object = %self.object.as_deref().unwrap_or_default(),
1326                mode = %self.mode,
1327                owner = %self.owner.as_deref().unwrap_or_default(),
1328                hold_ms = hold.as_millis(),
1329                threshold_ms = threshold.as_millis(),
1330                "object namespace lock held longer than threshold"
1331            );
1332        }
1333    }
1334}
1335
1336/// A failure to preserve one object generation for a SelectObjectContent read.
1337#[derive(Debug, thiserror::Error)]
1338#[non_exhaustive]
1339pub enum SnapshotConsistencyError {
1340    #[error("namespace locking is disabled for SelectObjectContent")]
1341    LockingDisabled,
1342    #[error("SelectObjectContent namespace lock was lost")]
1343    LockLost,
1344    #[error("object read semantics changed while SelectObjectContent was running")]
1345    ObjectChanged,
1346}
1347
1348/// Failure while creating a SelectObjectContent snapshot.
1349#[derive(Debug, thiserror::Error)]
1350#[non_exhaustive]
1351pub enum PrepareSelectObjectSnapshotError {
1352    #[error("storage failed while preparing SelectObjectContent snapshot: {0}")]
1353    Storage(#[source] StorageError),
1354    #[error("SelectObjectContent snapshot consistency failure: {0}")]
1355    Consistency(#[source] SnapshotConsistencyError),
1356    #[error("SelectObjectContent object has invalid logical size {size}")]
1357    InvalidLogicalSize { size: i64 },
1358}
1359
1360impl From<StorageError> for PrepareSelectObjectSnapshotError {
1361    fn from(error: StorageError) -> Self {
1362        Self::Storage(error)
1363    }
1364}
1365
1366impl From<SnapshotConsistencyError> for PrepareSelectObjectSnapshotError {
1367    fn from(error: SnapshotConsistencyError) -> Self {
1368        Self::Consistency(error)
1369    }
1370}
1371
1372/// Failure while opening a reader from a SelectObjectContent snapshot.
1373#[derive(Debug, thiserror::Error)]
1374#[non_exhaustive]
1375pub enum SelectObjectSnapshotReadError {
1376    #[error("storage failed while opening SelectObjectContent snapshot reader: {0}")]
1377    Storage(#[source] StorageError),
1378    #[error("SelectObjectContent snapshot consistency failure: {0}")]
1379    Consistency(#[source] SnapshotConsistencyError),
1380}
1381
1382impl From<StorageError> for SelectObjectSnapshotReadError {
1383    fn from(error: StorageError) -> Self {
1384        Self::Storage(error)
1385    }
1386}
1387
1388impl From<SnapshotConsistencyError> for SelectObjectSnapshotReadError {
1389    fn from(error: SnapshotConsistencyError) -> Self {
1390        Self::Consistency(error)
1391    }
1392}
1393
1394struct SelectObjectSnapshotLease {
1395    guards: Vec<ObjectLockDiagGuard>,
1396    lost: Arc<AtomicBool>,
1397    lock_loss: tokio::sync::watch::Sender<bool>,
1398    _monitor_shutdown: tokio::sync::watch::Sender<()>,
1399}
1400
1401impl SelectObjectSnapshotLease {
1402    fn new(guards: Vec<ObjectLockDiagGuard>) -> Self {
1403        let signals = guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal);
1404        let mut waits = signals
1405            .map(|signal| async move { signal.notified().await })
1406            .collect::<futures::stream::FuturesUnordered<_>>();
1407        let lost = Arc::new(AtomicBool::new(false));
1408        let (lock_loss, _) = tokio::sync::watch::channel(false);
1409        let (monitor_shutdown, mut shutdown_rx) = tokio::sync::watch::channel(());
1410        if !waits.is_empty() {
1411            let task_lost = Arc::clone(&lost);
1412            let task_lock_loss = lock_loss.clone();
1413            tokio::spawn(async move {
1414                tokio::select! {
1415                    lost = futures::StreamExt::next(&mut waits) => {
1416                        if lost.is_some() {
1417                            task_lost.store(true, Ordering::Release);
1418                            task_lock_loss.send_replace(true);
1419                        }
1420                    }
1421                    _ = shutdown_rx.changed() => {}
1422                }
1423            });
1424        }
1425        Self {
1426            guards,
1427            lost,
1428            lock_loss,
1429            _monitor_shutdown: monitor_shutdown,
1430        }
1431    }
1432
1433    fn check(&self) -> std::result::Result<(), SnapshotConsistencyError> {
1434        if self.is_lost() || self.guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
1435            self.lost.store(true, Ordering::Release);
1436            self.lock_loss.send_replace(true);
1437            return Err(SnapshotConsistencyError::LockLost);
1438        }
1439        Ok(())
1440    }
1441
1442    fn is_lost(&self) -> bool {
1443        self.lost.load(Ordering::Acquire)
1444    }
1445
1446    fn subscribe_lock_loss(&self) -> tokio::sync::watch::Receiver<bool> {
1447        self.lock_loss.subscribe()
1448    }
1449}
1450
1451/// Opaque, lock-backed object generation used by SelectObjectContent.
1452pub struct SelectObjectSnapshot {
1453    pool: Arc<Sets>,
1454    bucket: String,
1455    object: String,
1456    headers: HeaderMap,
1457    opts: ObjectOptions,
1458    object_info: ObjectInfo,
1459    logical_size: u64,
1460    read_semantics_identity: [u8; 32],
1461    first_metadata: ParkingMutex<Option<crate::set_disk::PreparedGetObjectMetadata>>,
1462    lease: Arc<SelectObjectSnapshotLease>,
1463}
1464
1465impl fmt::Debug for SelectObjectSnapshot {
1466    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
1467        formatter
1468            .debug_struct("SelectObjectSnapshot")
1469            .field("bucket", &self.bucket)
1470            .field("object", &self.object)
1471            .field("logical_size", &self.logical_size)
1472            .finish_non_exhaustive()
1473    }
1474}
1475
1476impl SelectObjectSnapshot {
1477    pub fn is_for(&self, bucket: &str, object: &str) -> bool {
1478        self.bucket == bucket && self.object == encode_dir_object(object)
1479    }
1480
1481    pub fn object_info(&self) -> &ObjectInfo {
1482        &self.object_info
1483    }
1484
1485    pub fn logical_size(&self) -> u64 {
1486        self.logical_size
1487    }
1488
1489    pub fn matches_version(&self, requested: &str) -> bool {
1490        select_snapshot_version_matches(self.object_info.version_id, requested)
1491    }
1492
1493    pub fn ensure_valid(&self) -> std::result::Result<(), SnapshotConsistencyError> {
1494        self.lease.check()
1495    }
1496
1497    #[cfg(feature = "test-util")]
1498    pub fn mark_lost_for_test(&self) {
1499        self.lease.lost.store(true, Ordering::Release);
1500        self.lease.lock_loss.send_replace(true);
1501    }
1502
1503    pub async fn open_reader(
1504        &self,
1505        range: Option<HTTPRangeSpec>,
1506    ) -> std::result::Result<GetObjectReader, SelectObjectSnapshotReadError> {
1507        self.lease.check()?;
1508        let first_metadata = self.first_metadata.lock().take();
1509        let metadata = match first_metadata {
1510            Some(metadata) => metadata,
1511            None => {
1512                self.pool
1513                    .prepare_get_object_reader_metadata(&self.bucket, &self.object, &self.opts)
1514                    .await?
1515            }
1516        };
1517        self.lease.check()?;
1518        if metadata.read_semantics_identity() != self.read_semantics_identity {
1519            return Err(SnapshotConsistencyError::ObjectChanged.into());
1520        }
1521
1522        let mut reader =
1523            crate::object_api::without_get_object_body_cache_hook(self.pool.get_object_reader_with_prepared_metadata(
1524                &self.bucket,
1525                &self.object,
1526                range,
1527                self.headers.clone(),
1528                &self.opts,
1529                metadata,
1530            ))
1531            .await?;
1532        self.lease.check()?;
1533        reader.body_source = crate::object_api::GetObjectBodySource::HookMissed;
1534        reader.stream = Box::new(SelectObjectSnapshotReader {
1535            inner: reader.stream,
1536            lock_loss_wake: SelectObjectSnapshotLockLossWake::new(self.lease.subscribe_lock_loss()),
1537            lease: Arc::clone(&self.lease),
1538        });
1539        Ok(reader)
1540    }
1541}
1542
1543fn select_snapshot_version_matches(actual: Option<Uuid>, requested: &str) -> bool {
1544    let requested = requested.trim();
1545    let requested = if requested.eq_ignore_ascii_case("null") {
1546        Uuid::nil()
1547    } else if let Ok(requested) = Uuid::parse_str(requested) {
1548        requested
1549    } else {
1550        return false;
1551    };
1552    actual.unwrap_or_else(Uuid::nil) == requested
1553}
1554
1555struct SelectObjectSnapshotReader {
1556    inner: Box<dyn AsyncRead + Unpin + Send + Sync>,
1557    lock_loss_wake: SelectObjectSnapshotLockLossWake,
1558    lease: Arc<SelectObjectSnapshotLease>,
1559}
1560
1561struct SelectObjectSnapshotLockLossWake {
1562    stream: tokio_stream::wrappers::WatchStream<bool>,
1563}
1564
1565impl SelectObjectSnapshotLockLossWake {
1566    fn new(receiver: tokio::sync::watch::Receiver<bool>) -> Self {
1567        Self {
1568            stream: tokio_stream::wrappers::WatchStream::new(receiver),
1569        }
1570    }
1571
1572    fn poll_lost(&mut self, cx: &mut Context<'_>) -> bool {
1573        loop {
1574            match futures::Stream::poll_next(Pin::new(&mut self.stream), cx) {
1575                Poll::Ready(Some(true)) => return true,
1576                Poll::Ready(Some(false)) => {}
1577                Poll::Ready(None) | Poll::Pending => return false,
1578            }
1579        }
1580    }
1581}
1582
1583impl AsyncRead for SelectObjectSnapshotReader {
1584    fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
1585        if self.lock_loss_wake.poll_lost(cx) || self.lease.is_lost() {
1586            return Poll::Ready(Err(std::io::Error::other(SnapshotConsistencyError::LockLost)));
1587        }
1588        let filled_before = buf.filled().len();
1589        let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
1590        if self.lease.check().is_err() {
1591            buf.set_filled(filled_before);
1592            return Poll::Ready(Err(std::io::Error::other(SnapshotConsistencyError::LockLost)));
1593        }
1594        poll
1595    }
1596}
1597
1598fn log_object_lock_acquire_if_slow(
1599    op: &'static str,
1600    bucket: &str,
1601    object: &str,
1602    owner: Option<&str>,
1603    mode: ObjectLockDiagMode,
1604    elapsed: Duration,
1605    diag_enabled: bool,
1606) {
1607    if !diag_enabled {
1608        return;
1609    }
1610
1611    let threshold = get_object_lock_diag_slow_acquire_threshold();
1612    record_object_lock_diag_acquire_duration(op, mode.as_str(), elapsed);
1613    if elapsed >= threshold {
1614        record_object_lock_diag_slow_acquire(op, mode.as_str());
1615        warn!(
1616            target: "rustfs_ecstore::object_lock_diag",
1617            op,
1618            bucket,
1619            object,
1620            mode = %mode,
1621            owner = owner.unwrap_or_default(),
1622            acquire_ms = elapsed.as_millis(),
1623            threshold_ms = threshold.as_millis(),
1624            "object namespace lock acquisition exceeded threshold"
1625        );
1626    }
1627}
1628
1629fn select_data_movement_target_pool(
1630    existing_pool_idx: Result<usize>,
1631    src_pool_idx: usize,
1632    delete_marker: bool,
1633) -> Result<Option<usize>> {
1634    match existing_pool_idx {
1635        Ok(pool_idx) => {
1636            if delete_marker && pool_idx == src_pool_idx {
1637                Ok(None)
1638            } else {
1639                Ok(Some(pool_idx))
1640            }
1641        }
1642        Err(err) => {
1643            if is_err_read_quorum(&err) {
1644                return Err(StorageError::ErasureWriteQuorum);
1645            }
1646            if delete_marker && (is_err_object_not_found(&err) || is_err_version_not_found(&err)) {
1647                Ok(None)
1648            } else {
1649                Err(err)
1650            }
1651        }
1652    }
1653}
1654
1655fn latest_object_access_delete_marker_error(
1656    bucket: &str,
1657    object: &str,
1658    info: &ObjectInfo,
1659    opts: &ObjectOptions,
1660) -> Option<Error> {
1661    if !info.delete_marker {
1662        return None;
1663    }
1664
1665    Some(if opts.version_id.is_none() || opts.delete_marker {
1666        to_object_err(StorageError::FileNotFound, vec![bucket, object])
1667    } else {
1668        to_object_err(StorageError::MethodNotAllowed, vec![bucket, object])
1669    })
1670}
1671
1672fn resolve_latest_object_access(
1673    bucket: &str,
1674    object: &str,
1675    info: ObjectInfo,
1676    idx: usize,
1677    opts: &ObjectOptions,
1678) -> Result<(ObjectInfo, usize)> {
1679    if let Some(err) = latest_object_access_delete_marker_error(bucket, object, &info, opts) {
1680        return Err(err);
1681    }
1682
1683    Ok((info, idx))
1684}
1685
1686fn should_create_delete_marker_for_missing_object(opts: &ObjectOptions) -> bool {
1687    (opts.versioned || opts.version_suspended) && opts.version_id.is_none() && !opts.delete_marker && !opts.data_movement
1688}
1689
1690fn latest_versioned_delete_creates_distinct_marker(opts: &ObjectOptions) -> bool {
1691    opts.versioned
1692        && !opts.version_suspended
1693        && opts.version_id.is_none()
1694        && !opts.delete_marker
1695        && !opts.data_movement
1696        && !opts.replication_request
1697}
1698
1699#[cfg(any(test, feature = "test-util"))]
1700struct DeleteAfterObjectLockSnapshotBarrierState {
1701    bucket: String,
1702    arrived: tokio::sync::Notify,
1703    release: tokio::sync::Notify,
1704    namespace_pending: tokio::sync::Notify,
1705    namespace_acquired: AtomicBool,
1706}
1707
1708#[cfg(any(test, feature = "test-util"))]
1709pub struct DeleteAfterObjectLockSnapshotBarrier {
1710    state: Arc<DeleteAfterObjectLockSnapshotBarrierState>,
1711}
1712
1713#[cfg(any(test, feature = "test-util"))]
1714static DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER: std::sync::OnceLock<
1715    std::sync::Mutex<Option<Arc<DeleteAfterObjectLockSnapshotBarrierState>>>,
1716> = std::sync::OnceLock::new();
1717
1718#[cfg(any(test, feature = "test-util"))]
1719impl DeleteAfterObjectLockSnapshotBarrier {
1720    pub fn install(bucket: &str) -> Self {
1721        let state = Arc::new(DeleteAfterObjectLockSnapshotBarrierState {
1722            bucket: bucket.to_string(),
1723            arrived: tokio::sync::Notify::new(),
1724            release: tokio::sync::Notify::new(),
1725            namespace_pending: tokio::sync::Notify::new(),
1726            namespace_acquired: AtomicBool::new(false),
1727        });
1728        let mut slot = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
1729            .get_or_init(|| std::sync::Mutex::new(None))
1730            .lock()
1731            .expect("delete snapshot barrier mutex should not poison");
1732        assert!(slot.is_none(), "delete snapshot barrier must not already be installed");
1733        *slot = Some(Arc::clone(&state));
1734        Self { state }
1735    }
1736
1737    pub async fn wait_until_paused(&self) {
1738        self.state.arrived.notified().await;
1739    }
1740
1741    pub fn release(&self) {
1742        self.state.release.notify_one();
1743    }
1744
1745    pub async fn release_and_wait_until_namespace_pending(&self) {
1746        let namespace_pending = self.state.namespace_pending.notified();
1747        self.release();
1748        tokio::time::timeout(Duration::from_secs(5), namespace_pending)
1749            .await
1750            .expect("delete should proceed to its namespace lock after leaving the snapshot barrier");
1751    }
1752
1753    pub fn namespace_acquired(&self) -> bool {
1754        self.state.namespace_acquired.load(Ordering::Acquire)
1755    }
1756}
1757
1758#[cfg(any(test, feature = "test-util"))]
1759impl Drop for DeleteAfterObjectLockSnapshotBarrier {
1760    fn drop(&mut self) {
1761        self.state.release.notify_one();
1762        if let Some(slot) = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER.get() {
1763            let mut slot = slot.lock().expect("delete snapshot barrier mutex should not poison");
1764            if slot.as_ref().is_some_and(|installed| Arc::ptr_eq(installed, &self.state)) {
1765                *slot = None;
1766            }
1767        }
1768    }
1769}
1770
1771#[cfg(any(test, feature = "test-util"))]
1772async fn pause_delete_after_object_lock_snapshot(bucket: &str) {
1773    let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
1774        .get_or_init(|| std::sync::Mutex::new(None))
1775        .lock()
1776        .expect("delete snapshot barrier mutex should not poison")
1777        .as_ref()
1778        .filter(|state| state.bucket == bucket)
1779        .cloned();
1780    if let Some(state) = state {
1781        state.arrived.notify_one();
1782        state.release.notified().await;
1783    }
1784}
1785
1786#[cfg(any(test, feature = "test-util"))]
1787fn notify_delete_namespace_pending(bucket: &str) {
1788    let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
1789        .get_or_init(|| std::sync::Mutex::new(None))
1790        .lock()
1791        .expect("delete snapshot barrier mutex should not poison")
1792        .as_ref()
1793        .filter(|state| state.bucket == bucket)
1794        .cloned();
1795    if let Some(state) = state {
1796        state.namespace_pending.notify_one();
1797    }
1798}
1799
1800#[cfg(any(test, feature = "test-util"))]
1801fn notify_delete_namespace_acquired(bucket: &str) {
1802    let state = DELETE_AFTER_OBJECT_LOCK_SNAPSHOT_BARRIER
1803        .get_or_init(|| std::sync::Mutex::new(None))
1804        .lock()
1805        .expect("delete snapshot barrier mutex should not poison")
1806        .as_ref()
1807        .filter(|state| state.bucket == bucket)
1808        .cloned();
1809    if let Some(state) = state {
1810        state.namespace_acquired.store(true, Ordering::Release);
1811    }
1812}
1813
1814#[cfg(test)]
1815struct VersionedDeleteMarkerCommitBarrierState {
1816    bucket: String,
1817    object: String,
1818    arrived: tokio::sync::Notify,
1819    release: tokio::sync::Notify,
1820}
1821
1822#[cfg(test)]
1823pub(crate) struct VersionedDeleteMarkerCommitBarrier {
1824    state: Arc<VersionedDeleteMarkerCommitBarrierState>,
1825}
1826
1827#[cfg(test)]
1828static VERSIONED_DELETE_MARKER_COMMIT_BARRIER: std::sync::OnceLock<
1829    std::sync::Mutex<Option<Arc<VersionedDeleteMarkerCommitBarrierState>>>,
1830> = std::sync::OnceLock::new();
1831
1832#[cfg(test)]
1833impl VersionedDeleteMarkerCommitBarrier {
1834    pub(crate) fn install(bucket: &str, object: &str) -> Self {
1835        let state = Arc::new(VersionedDeleteMarkerCommitBarrierState {
1836            bucket: bucket.to_string(),
1837            object: object.to_string(),
1838            arrived: tokio::sync::Notify::new(),
1839            release: tokio::sync::Notify::new(),
1840        });
1841        let mut slot = VERSIONED_DELETE_MARKER_COMMIT_BARRIER
1842            .get_or_init(|| std::sync::Mutex::new(None))
1843            .lock()
1844            .expect("versioned delete-marker commit barrier mutex should not poison");
1845        assert!(slot.is_none(), "versioned delete-marker commit barrier must be unique");
1846        *slot = Some(Arc::clone(&state));
1847        Self { state }
1848    }
1849
1850    pub(crate) async fn wait_until_paused(&self) {
1851        tokio::time::timeout(Duration::from_secs(30), self.state.arrived.notified())
1852            .await
1853            .expect("versioned DELETE should reach the post-marker-commit barrier");
1854    }
1855
1856    pub(crate) fn release(&self) {
1857        self.state.release.notify_one();
1858    }
1859}
1860
1861#[cfg(test)]
1862impl Drop for VersionedDeleteMarkerCommitBarrier {
1863    fn drop(&mut self) {
1864        self.state.release.notify_one();
1865        let mut slot = VERSIONED_DELETE_MARKER_COMMIT_BARRIER
1866            .get_or_init(|| std::sync::Mutex::new(None))
1867            .lock()
1868            .expect("versioned delete-marker commit barrier mutex should not poison");
1869        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
1870            *slot = None;
1871        }
1872    }
1873}
1874
1875#[cfg(test)]
1876async fn pause_versioned_delete_marker_after_commit(bucket: &str, object: &str) {
1877    let state = VERSIONED_DELETE_MARKER_COMMIT_BARRIER
1878        .get_or_init(|| std::sync::Mutex::new(None))
1879        .lock()
1880        .expect("versioned delete-marker commit barrier mutex should not poison")
1881        .as_ref()
1882        .filter(|state| state.bucket == bucket && state.object == object)
1883        .cloned();
1884    if let Some(state) = state {
1885        state.arrived.notify_one();
1886        state.release.notified().await;
1887    }
1888}
1889
1890/// Whether a delete-time lookup miss on a directory key should trigger an orphan
1891/// empty-directory tree purge (issue #4189).
1892///
1893/// The lookup surfaces *version*-not-found here, not object-not-found: `del_opts`
1894/// pins `version_id = Uuid::nil()` for directory keys, so a missing dir object fails
1895/// the specific-version lookup. Both misses must be accepted, otherwise the real
1896/// HTTP delete path (which always sets the nil version) never reaches the purge and
1897/// the ghost folder survives with a fake 204 — the exact #4189 symptom.
1898fn should_purge_orphan_dir_on_missing(err: &Error, object: &str) -> bool {
1899    (is_err_object_not_found(err) || is_err_version_not_found(err)) && rustfs_utils::path::is_dir_object(object)
1900}
1901
1902fn version_aware_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
1903    let mut lookup_opts = opts.clone();
1904    lookup_opts.no_lock = no_lock;
1905    if lookup_opts.version_id.is_some() {
1906        lookup_opts.metadata_chg = true;
1907    }
1908
1909    lookup_opts
1910}
1911
1912fn data_movement_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
1913    writer_pool_lookup_opts(opts, no_lock)
1914}
1915
1916fn uses_data_movement_pool_selection(opts: &ObjectOptions) -> bool {
1917    opts.data_movement && (opts.version_id.is_some() || DecommissionCapacityOwner::from_options(opts).is_some())
1918}
1919
1920fn writer_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
1921    let mut lookup_opts = version_aware_lookup_opts(opts, no_lock);
1922    lookup_opts.skip_decommissioned = true;
1923    lookup_opts.skip_rebalancing = true;
1924
1925    lookup_opts
1926}
1927
1928fn delete_pool_lookup_opts(opts: &ObjectOptions, no_lock: bool) -> ObjectOptions {
1929    let mut lookup_opts = writer_pool_lookup_opts(opts, no_lock);
1930    lookup_opts.skip_decommissioned = opts.data_movement;
1931    lookup_opts
1932}
1933
1934fn should_delete_from_all_pools(opts: &ObjectOptions, pool_count: usize) -> bool {
1935    pool_count > 0 && delete_only_releases_capacity(opts)
1936}
1937
1938fn delete_only_releases_capacity(opts: &ObjectOptions) -> bool {
1939    !opts.versioned && !opts.version_suspended || opts.version_id.is_some()
1940}
1941
1942fn batch_delete_creates_latest_marker(object: &ObjectToDelete, delete_config_snapshot: &DeleteReplicationConfigSnapshot) -> bool {
1943    if object.version_id.is_some() {
1944        return false;
1945    }
1946
1947    let object_name = decode_dir_object(&object.object_name);
1948    let (versioned, version_suspended) = delete_config_snapshot.versioning_config().delete_state(&object_name);
1949    versioned || version_suspended
1950}
1951
1952fn batch_delete_targets_pool(creates_latest_marker: bool, marker_target_pool_idx: Option<usize>, pool_idx: usize) -> bool {
1953    !creates_latest_marker || marker_target_pool_idx == Some(pool_idx)
1954}
1955
1956#[cfg(test)]
1957struct BatchDeletePoolErrorInjectionState {
1958    bucket: String,
1959    pool_idx: usize,
1960    errors: std::collections::HashMap<String, Error>,
1961    observed: std::sync::atomic::AtomicUsize,
1962}
1963
1964#[cfg(test)]
1965pub(crate) struct BatchDeletePoolErrorInjection {
1966    state: Arc<BatchDeletePoolErrorInjectionState>,
1967}
1968
1969#[cfg(test)]
1970static BATCH_DELETE_POOL_ERROR_INJECTION: std::sync::OnceLock<std::sync::Mutex<Option<Arc<BatchDeletePoolErrorInjectionState>>>> =
1971    std::sync::OnceLock::new();
1972
1973#[cfg(test)]
1974impl BatchDeletePoolErrorInjection {
1975    pub(crate) fn install(bucket: &str, pool_idx: usize, errors: Vec<(String, Error)>) -> Self {
1976        let state = Arc::new(BatchDeletePoolErrorInjectionState {
1977            bucket: bucket.to_string(),
1978            pool_idx,
1979            errors: errors.into_iter().collect(),
1980            observed: std::sync::atomic::AtomicUsize::new(0),
1981        });
1982        let mut slot = BATCH_DELETE_POOL_ERROR_INJECTION
1983            .get_or_init(|| std::sync::Mutex::new(None))
1984            .lock()
1985            .expect("batch delete pool error injection mutex should not poison");
1986        assert!(slot.is_none(), "batch delete pool error injection must be unique");
1987        *slot = Some(Arc::clone(&state));
1988        Self { state }
1989    }
1990
1991    pub(crate) fn observed(&self) -> usize {
1992        self.state.observed.load(Ordering::Acquire)
1993    }
1994}
1995
1996#[cfg(test)]
1997impl Drop for BatchDeletePoolErrorInjection {
1998    fn drop(&mut self) {
1999        let mut slot = BATCH_DELETE_POOL_ERROR_INJECTION
2000            .get_or_init(|| std::sync::Mutex::new(None))
2001            .lock()
2002            .expect("batch delete pool error injection mutex should not poison");
2003        if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
2004            *slot = None;
2005        }
2006    }
2007}
2008
2009#[cfg(test)]
2010fn inject_batch_delete_pool_errors(
2011    bucket: &str,
2012    pool_idx: usize,
2013    object_names: &[String],
2014    deleted: &[DeletedObject],
2015    errors: &mut [Option<Error>],
2016) {
2017    let state = BATCH_DELETE_POOL_ERROR_INJECTION
2018        .get_or_init(|| std::sync::Mutex::new(None))
2019        .lock()
2020        .expect("batch delete pool error injection mutex should not poison")
2021        .as_ref()
2022        .filter(|state| state.bucket == bucket && state.pool_idx == pool_idx)
2023        .cloned();
2024    let Some(state) = state else {
2025        return;
2026    };
2027
2028    for (idx, object_name) in object_names.iter().enumerate() {
2029        let Some(error) = state.errors.get(object_name) else {
2030            continue;
2031        };
2032        if errors[idx].is_none() && deleted[idx].found {
2033            errors[idx] = Some(error.clone());
2034            state.observed.fetch_add(1, Ordering::AcqRel);
2035        }
2036    }
2037}
2038
2039fn resolve_batch_delete_pool_results<'a>(
2040    initial_error: Option<Error>,
2041    pool_results: impl IntoIterator<Item = (&'a DeletedObject, &'a Option<Error>)>,
2042) -> (Option<DeletedObject>, Option<Error>, bool) {
2043    let mut failure = initial_error.map(|err| (None, err));
2044    let mut deleted = None;
2045    let mut fallback: Option<(DeletedObject, Option<Error>)> = None;
2046    let mut attempted = false;
2047
2048    for (pool_delete, pool_error) in pool_results {
2049        attempted = true;
2050        match pool_error {
2051            Some(err) if is_err_object_not_found(err) || is_err_version_not_found(err) => {
2052                if fallback.as_ref().is_none_or(|(_, error)| error.is_none()) {
2053                    fallback = Some(((*pool_delete).clone(), Some(err.clone())));
2054                }
2055            }
2056            Some(err) => {
2057                if failure.is_none() {
2058                    failure = Some((Some((*pool_delete).clone()), err.clone()));
2059                }
2060            }
2061            None if pool_delete.found => {
2062                if deleted.is_none() {
2063                    deleted = Some((*pool_delete).clone());
2064                }
2065            }
2066            None => {
2067                if fallback.is_none() {
2068                    fallback = Some(((*pool_delete).clone(), None));
2069                }
2070            }
2071        }
2072    }
2073
2074    if let Some((failed_delete, err)) = failure {
2075        return (failed_delete, Some(err), attempted);
2076    }
2077    if let Some(deleted) = deleted {
2078        return (Some(deleted), None, attempted);
2079    }
2080    if let Some((deleted, err)) = fallback {
2081        return (Some(deleted), err, attempted);
2082    }
2083
2084    (None, None, attempted)
2085}
2086
2087fn transition_restore_pool_opts(opts: &ObjectOptions) -> ObjectOptions {
2088    let mut lookup_opts = opts.clone();
2089    // `no_lock` is an internal implementation detail, not transferable
2090    // authority. Restore deliberately does not hold an outer object write lock,
2091    // so every final local commit must acquire its own commit-late lock.
2092    lookup_opts.no_lock = false;
2093    lookup_opts.skip_decommissioned = true;
2094    lookup_opts.skip_rebalancing = true;
2095    lookup_opts
2096}
2097
2098fn effective_object_actual_size(info: &ObjectInfo) -> Option<i64> {
2099    info.get_actual_size().ok()
2100}
2101
2102pub(super) fn is_equivalent_data_movement_delete_marker(source: &ObjectInfo, target: &ObjectInfo) -> bool {
2103    is_data_movement_delete_marker(source)
2104        && is_data_movement_delete_marker(target)
2105        && source.version_id == target.version_id
2106        && source.mod_time == target.mod_time
2107        && is_equivalent_data_movement_delete_marker_metadata(&source.user_defined, &target.user_defined)
2108        && source.user_tags == target.user_tags
2109        && source.replication_status_internal == target.replication_status_internal
2110        && source.replication_status == target.replication_status
2111        && source.version_purge_status_internal == target.version_purge_status_internal
2112        && source.version_purge_status == target.version_purge_status
2113}
2114
2115fn is_equivalent_data_movement_delete_marker_metadata(
2116    source: &HashMap<String, String>,
2117    target: &HashMap<String, String>,
2118) -> bool {
2119    matches!(
2120        (
2121            data_movement_delete_marker_metadata_identity(source),
2122            data_movement_delete_marker_metadata_identity(target)
2123        ),
2124        (Some(source), Some(target)) if source == target
2125    )
2126}
2127
2128fn data_movement_delete_marker_metadata_identity(metadata: &HashMap<String, String>) -> Option<HashMap<String, String>> {
2129    let mut identity = HashMap::with_capacity(metadata.len());
2130    let mut local_tier_free_version_id = None;
2131    for (key, value) in metadata {
2132        let Some(suffix) = rustfs_utils::http::strip_internal_prefix_preserving_case(key) else {
2133            identity.insert(key.clone(), value.clone());
2134            continue;
2135        };
2136
2137        if suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_TIER_FV_ID) {
2138            let version_id = Uuid::parse_str(value).ok().filter(|version_id| !version_id.is_nil())?;
2139            if local_tier_free_version_id.is_some_and(|expected| expected != version_id) {
2140                return None;
2141            }
2142            local_tier_free_version_id = Some(version_id);
2143            continue;
2144        }
2145        if suffix.eq_ignore_ascii_case(rustfs_utils::http::metadata_compat::SUFFIX_BUCKET_INCARNATION_ID) {
2146            continue;
2147        }
2148
2149        let canonical_suffix = [
2150            rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP,
2151            rustfs_utils::http::SUFFIX_REPLICA_STATUS,
2152            rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
2153            rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
2154            rustfs_utils::http::SUFFIX_PURGESTATUS,
2155        ]
2156        .into_iter()
2157        .find(|candidate| suffix.eq_ignore_ascii_case(candidate))
2158        .map(str::to_string)
2159        .or_else(|| {
2160            [
2161                rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX,
2162                rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
2163            ]
2164            .into_iter()
2165            .find_map(|prefix| {
2166                suffix
2167                    .get(..prefix.len())
2168                    .is_some_and(|candidate| candidate.eq_ignore_ascii_case(prefix))
2169                    .then(|| format!("{prefix}{}", &suffix[prefix.len()..]))
2170            })
2171        })
2172        .unwrap_or_else(|| suffix.to_string());
2173        let canonical_value = if canonical_suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP)
2174            || canonical_suffix.eq_ignore_ascii_case(rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP)
2175        {
2176            rustfs_filemeta::parse_replication_timestamp(value)?
2177                .unix_timestamp_nanos()
2178                .to_string()
2179        } else {
2180            value.clone()
2181        };
2182        let canonical_key = format!("{}{canonical_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
2183        if identity
2184            .insert(canonical_key, canonical_value.clone())
2185            .is_some_and(|existing| existing != canonical_value)
2186        {
2187            return None;
2188        }
2189    }
2190    for (status_suffix, timestamp_suffix) in [
2191        (rustfs_utils::http::SUFFIX_REPLICA_STATUS, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP),
2192        (
2193            rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
2194            rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
2195        ),
2196    ] {
2197        let status_key = format!("{}{status_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
2198        let timestamp_key = format!("{}{timestamp_suffix}", rustfs_utils::http::RUSTFS_INTERNAL_PREFIX);
2199        match (identity.contains_key(&status_key), identity.contains_key(&timestamp_key)) {
2200            (true, false) => {
2201                identity.insert(timestamp_key, OffsetDateTime::UNIX_EPOCH.unix_timestamp_nanos().to_string());
2202            }
2203            (false, true) => return None,
2204            _ => {}
2205        }
2206    }
2207    Some(identity)
2208}
2209
2210fn is_data_movement_delete_marker(info: &ObjectInfo) -> bool {
2211    info.delete_marker
2212}
2213
2214fn is_expected_data_movement_delete_marker_source(source: &ObjectInfo, expected_mod_time: Option<OffsetDateTime>) -> bool {
2215    is_data_movement_delete_marker(source)
2216        && source.mod_time.is_some()
2217        && source.mod_time == expected_mod_time
2218        && data_movement_delete_marker_metadata_identity(&source.user_defined).is_some()
2219}
2220
2221fn current_data_movement_delete_marker_opts(source: &ObjectInfo, opts: &ObjectOptions) -> Option<ObjectOptions> {
2222    let replica_status = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_STATUS);
2223    let replica_timestamp = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP);
2224    let (replica_status, replica_timestamp) = match (replica_status, replica_timestamp) {
2225        (None, None) => Default::default(),
2226        (Some(status), timestamp) => {
2227            let status = crate::bucket::replication::ReplicationStatusType::from(status.as_str());
2228            if status.is_empty() {
2229                return None;
2230            }
2231            let timestamp = match timestamp {
2232                Some(timestamp) => rustfs_filemeta::parse_replication_timestamp(&timestamp)?,
2233                None => OffsetDateTime::UNIX_EPOCH,
2234            };
2235            (status, Some(timestamp))
2236        }
2237        (None, Some(_)) => return None,
2238    };
2239    let replication_status = rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_STATUS);
2240    let replication_timestamp =
2241        rustfs_utils::http::get_str(&source.user_defined, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP);
2242    let (replication_status, replication_timestamp, replication_targets) = match (replication_status, replication_timestamp) {
2243        (None, None) => Default::default(),
2244        (Some(status), timestamp) => {
2245            let direct_status = crate::bucket::replication::ReplicationStatusType::from(status.as_str());
2246            let targets = crate::bucket::replication::replication_statuses_map(status.as_str());
2247            if direct_status.is_empty() && targets.is_empty() {
2248                return None;
2249            }
2250            let timestamp = match timestamp {
2251                Some(timestamp) => rustfs_filemeta::parse_replication_timestamp(&timestamp)?,
2252                None => OffsetDateTime::UNIX_EPOCH,
2253            };
2254            (Some(status), Some(timestamp), targets)
2255        }
2256        (None, Some(_)) => return None,
2257    };
2258    let mut state = source.replication_state();
2259    if state.target_delete_marker_version_ids_corrupt {
2260        return None;
2261    }
2262    state.replica_status = replica_status;
2263    state.replica_timestamp = replica_timestamp;
2264    state.replication_status_internal = replication_status;
2265    state.replication_timestamp = replication_timestamp;
2266    state.targets = replication_targets;
2267    state.replicate_decision_str = source.replication_decision.clone();
2268    state.delete_marker = true;
2269
2270    let mut target_opts = opts.clone();
2271    target_opts.mod_time = source.mod_time;
2272    target_opts.delete_replication = Some(state);
2273    Some(target_opts)
2274}
2275
2276fn expected_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo) -> ObjectInfo {
2277    ObjectInfo::from_file_info(source, "", &source.name, source.version_id.is_some())
2278}
2279
2280fn is_equivalent_data_movement_tiered_object(source: &rustfs_filemeta::FileInfo, target: &ObjectInfo) -> bool {
2281    let expected = expected_data_movement_tiered_object(source);
2282    let Some(source_actual_size) = effective_object_actual_size(&expected) else {
2283        return false;
2284    };
2285    let Some(target_actual_size) = effective_object_actual_size(target) else {
2286        return false;
2287    };
2288
2289    source.version_id == target.version_id
2290        && !target.delete_marker
2291        && source.size == target.size
2292        && source.get_etag() == target.etag
2293        && source.checksum == target.checksum
2294        && crate::data_movement::are_equivalent_data_movement_parts(&source.parts, &target.parts)
2295        && source.mod_time == target.mod_time
2296        && crate::data_movement::is_equivalent_data_movement_metadata(&expected, target, source_actual_size, target_actual_size)
2297        && expected.user_tags == target.user_tags
2298        && expected.expires == target.expires
2299        && expected.storage_class == target.storage_class
2300        && expected.replication_status_internal == target.replication_status_internal
2301        && expected.replication_status == target.replication_status
2302        && expected.version_purge_status_internal == target.version_purge_status_internal
2303        && expected.version_purge_status == target.version_purge_status
2304        && expected.transitioned_object.status == target.transitioned_object.status
2305        && expected.transition_version_state == target.transition_version_state
2306        && expected.transitioned_object.name == target.transitioned_object.name
2307        && expected.transitioned_object.tier == target.transitioned_object.tier
2308        && expected.transitioned_object.version_id == target.transitioned_object.version_id
2309        && expected.transitioned_object.free_version == target.transitioned_object.free_version
2310        && source_actual_size == target_actual_size
2311}
2312
2313pub(crate) fn tiered_data_movement_source_matches(
2314    expected: &rustfs_filemeta::FileInfo,
2315    current: &rustfs_filemeta::FileInfo,
2316) -> Result<bool> {
2317    let expected_backend = crate::services::tier::tier::tier_destination_id_from_metadata(&expected.metadata)?;
2318    let current_backend = crate::services::tier::tier::tier_destination_id_from_metadata(&current.metadata)?;
2319    Ok(expected.version_id == current.version_id
2320        && expected.deleted == current.deleted
2321        && expected.tier_free_version() == current.tier_free_version()
2322        && expected.data_dir == current.data_dir
2323        && expected.mod_time == current.mod_time
2324        && expected.size == current.size
2325        && expected.get_etag() == current.get_etag()
2326        && expected.transition_status == current.transition_status
2327        && expected.transitioned_objname == current.transitioned_objname
2328        && expected.transition_tier == current.transition_tier
2329        && expected.transition_version_id == current.transition_version_id
2330        && expected.transition_version == current.transition_version
2331        && expected.transition_version_state == current.transition_version_state
2332        && expected_backend == current_backend)
2333}
2334
2335fn decommission_free_version_overwrite_error(bucket: &str, object: &str, version_id: Option<Uuid>) -> Error {
2336    StorageError::DataMovementOverwriteErr(
2337        bucket.to_owned(),
2338        object.to_owned(),
2339        version_id.map(|id| id.to_string()).unwrap_or_default(),
2340    )
2341}
2342
2343fn should_check_data_movement_resume_target(src_pool_idx: usize, target_pool_idx: usize) -> bool {
2344    target_pool_idx != src_pool_idx
2345}
2346
2347fn resolve_data_movement_resume_target_pool(
2348    selected_target_pool_idx: usize,
2349    resume_target_pool_idx: Option<usize>,
2350    src_pool_idx: usize,
2351) -> usize {
2352    if should_check_data_movement_resume_target(src_pool_idx, selected_target_pool_idx) {
2353        selected_target_pool_idx
2354    } else {
2355        resume_target_pool_idx.unwrap_or(selected_target_pool_idx)
2356    }
2357}
2358
2359fn resolve_data_movement_delete_marker_resume_result(
2360    target_result: Result<Option<ObjectInfo>>,
2361    source: &ObjectInfo,
2362    src_pool_idx: usize,
2363    target_pool_idx: usize,
2364) -> Result<bool> {
2365    if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
2366        return Ok(false);
2367    }
2368
2369    let Some(target) = target_result? else {
2370        return Ok(false);
2371    };
2372
2373    Ok(is_equivalent_data_movement_delete_marker(source, &target))
2374}
2375
2376fn resolve_data_movement_tiered_resume_result(
2377    target_result: Result<Option<ObjectInfo>>,
2378    source: &rustfs_filemeta::FileInfo,
2379    src_pool_idx: usize,
2380    target_pool_idx: usize,
2381) -> Result<bool> {
2382    if !should_check_data_movement_resume_target(src_pool_idx, target_pool_idx) {
2383        return Ok(false);
2384    }
2385
2386    let Some(target) = target_result? else {
2387        return Ok(false);
2388    };
2389
2390    Ok(is_equivalent_data_movement_tiered_object(source, &target))
2391}
2392
2393fn return_batch_delete_lock_error(objects: &[ObjectToDelete], err: Error) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
2394    let del_objects = objects
2395        .iter()
2396        .map(|object| DeletedObject {
2397            object_name: decode_dir_object(&object.object_name),
2398            version_id: object.version_id,
2399            ..Default::default()
2400        })
2401        .collect();
2402    let del_errs = objects.iter().map(|_| Some(err.clone())).collect();
2403
2404    (del_objects, del_errs)
2405}
2406
2407fn return_batch_delete_lock_error_with_accounting(
2408    objects: &[ObjectToDelete],
2409    err: Error,
2410) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
2411    let (deleted, errors) = return_batch_delete_lock_error(objects, err);
2412    (deleted, errors, vec![None; objects.len()])
2413}
2414
2415fn sorted_unique_delete_object_names(objects: &[ObjectToDelete]) -> Vec<&str> {
2416    let mut object_names: Vec<&str> = objects.iter().map(|object| object.object_name.as_str()).collect();
2417    object_names.sort_unstable();
2418    object_names.dedup();
2419    object_names
2420}
2421
2422/// A CopyObject branch that writes a fresh local data stream owns new bytes and
2423/// must not inherit the source version's remote-tier ownership record.  FileMeta
2424/// exposes internal transition fields through `ObjectInfo::user_defined` for
2425/// compatibility, so cloning that map verbatim would publish a second owner of
2426/// the same remote tuple even though the destination has local data.
2427///
2428/// Keep this normalization in the store layer so every full-copy branch in
2429/// `handle_copy_object` shares the same fail-safe boundary. Metadata-only and
2430/// version-only same-key copies do not use this helper and continue to preserve
2431/// the existing version's protected state.
2432fn materialized_copy_user_defined(source: &HashMap<String, String>) -> HashMap<String, String> {
2433    let mut metadata = source.clone();
2434    for suffix in [
2435        rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
2436        rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
2437        rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
2438        rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
2439        rustfs_utils::http::SUFFIX_TRANSITION_TIER,
2440        rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
2441        rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID,
2442        rustfs_utils::http::SUFFIX_FREE_VERSION,
2443        rustfs_utils::http::SUFFIX_TIER_FV_ID,
2444        rustfs_utils::http::SUFFIX_TIER_FV_MARKER,
2445    ] {
2446        rustfs_utils::http::metadata_compat::remove_str(&mut metadata, suffix);
2447    }
2448    metadata
2449}
2450
2451enum RemoteTuplePublicationSource {
2452    Object(ObjectInfo),
2453    Tiered(rustfs_filemeta::FileInfo),
2454}
2455
2456/// A decommission migration may retain the fixed store domain as a read
2457/// anchor while it streams bytes. This type cannot authorize publication: it
2458/// must be consumed and dropped before the commit path acquires the fixed
2459/// domain as a write lock.
2460#[must_use = "the fixed read anchor must remain live until publication or migration abort"]
2461pub(crate) struct DecommissionFixedReadAnchor {
2462    guard: ObjectLockDiagGuard,
2463}
2464
2465impl DecommissionFixedReadAnchor {
2466    pub(crate) fn guard(&self) -> &ObjectLockDiagGuard {
2467        &self.guard
2468    }
2469
2470    fn is_lock_lost(&self) -> bool {
2471        self.guard.is_lock_lost()
2472    }
2473}
2474
2475/// Opaque, non-cloneable capability retained by every background path that can
2476/// publish an existing object identity into a new physical metadata owner.
2477///
2478/// The exact tier destination is captured before the copy, but its operation
2479/// lease and namespace write locks are deliberately deferred until the final
2480/// metadata commit. The source is then re-read under the same lock set. This
2481/// keeps large migrations compatible with the commit-late locking contract and
2482/// avoids pinning a tier generation throughout multipart staging.
2483#[must_use = "a data-movement publication fence must be consumed by the final target commit"]
2484pub(crate) struct RemoteTuplePublicationFence {
2485    store: Arc<ECStore>,
2486    bucket: String,
2487    object: String,
2488    source_pool_idx: usize,
2489    source: RemoteTuplePublicationSource,
2490    include_fixed_domain: bool,
2491    fixed_read_anchor: Option<DecommissionFixedReadAnchor>,
2492    backend_target: Option<(String, Option<TierDestinationId>)>,
2493}
2494
2495/// Locks and tier generation retained from source revalidation through the
2496/// target rename quorum (and any rename-tail guard handoff).
2497#[must_use = "the publication commit guard must live through the target metadata commit"]
2498pub(crate) struct RemoteTuplePublicationCommitGuard {
2499    guards: Vec<ObjectLockDiagGuard>,
2500    backend_lease: Option<TierOperationLease>,
2501    revalidated_tiered_source: Option<rustfs_filemeta::FileInfo>,
2502}
2503
2504impl RemoteTuplePublicationCommitGuard {
2505    pub(crate) fn is_lock_lost(&self) -> bool {
2506        self.guards.iter().any(ObjectLockDiagGuard::is_lock_lost)
2507            || self
2508                .backend_lease
2509                .as_ref()
2510                .is_some_and(|lease| !lease.is_current_generation())
2511    }
2512
2513    pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
2514        for guard in &self.guards {
2515            guard.add_namespace_lock_fence(opts);
2516        }
2517    }
2518
2519    pub(crate) fn revalidated_tiered_source(&self) -> Option<&rustfs_filemeta::FileInfo> {
2520        self.revalidated_tiered_source.as_ref()
2521    }
2522}
2523
2524fn remote_tuple_publication_target(source: &ObjectInfo) -> std::io::Result<Option<(String, Option<TierDestinationId>)>> {
2525    if source.transitioned_object.status != rustfs_filemeta::TRANSITION_COMPLETE && !source.transitioned_object.free_version {
2526        return Ok(None);
2527    }
2528    if source.transitioned_object.tier.is_empty() || source.transitioned_object.name.is_empty() {
2529        return Err(std::io::Error::other(
2530            "remote tuple publication source is missing its tier or remote object identity",
2531        ));
2532    }
2533    let backend_identity = tier_destination_id_from_metadata(source.user_defined.as_ref())?;
2534    Ok(Some((source.transitioned_object.tier.clone(), backend_identity)))
2535}
2536
2537fn remote_tuple_publication_object_source_matches(expected: &ObjectInfo, current: &ObjectInfo) -> bool {
2538    let (Ok(expected_actual_size), Ok(current_actual_size)) = (expected.get_actual_size(), current.get_actual_size()) else {
2539        return false;
2540    };
2541
2542    expected.data_dir.is_some_and(|data_dir| !data_dir.is_nil())
2543        && expected.data_dir == current.data_dir
2544        && expected.version_id == current.version_id
2545        && expected.delete_marker == current.delete_marker
2546        && expected.size == current.size
2547        && expected_actual_size == current_actual_size
2548        && expected.etag == current.etag
2549        && expected.checksum == current.checksum
2550        && expected.mod_time == current.mod_time
2551        && expected.storage_class == current.storage_class
2552        && crate::data_movement::is_equivalent_data_movement_metadata(
2553            expected,
2554            current,
2555            expected_actual_size,
2556            current_actual_size,
2557        )
2558        && expected.user_tags == current.user_tags
2559        && expected.expires == current.expires
2560        && expected.replication_status_internal == current.replication_status_internal
2561        && expected.replication_status == current.replication_status
2562        && expected.version_purge_status_internal == current.version_purge_status_internal
2563        && expected.version_purge_status == current.version_purge_status
2564        && expected.transitioned_object.name == current.transitioned_object.name
2565        && expected.transitioned_object.version_id == current.transitioned_object.version_id
2566        && expected.transitioned_object.tier == current.transitioned_object.tier
2567        && expected.transitioned_object.free_version == current.transitioned_object.free_version
2568        && expected.transitioned_object.status == current.transitioned_object.status
2569        && expected.transition_version_state == current.transition_version_state
2570        && remote_tuple_publication_parts_match(&expected.parts, &current.parts)
2571}
2572
2573fn remote_tuple_publication_parts_match(expected: &[ObjectPartInfo], current: &[ObjectPartInfo]) -> bool {
2574    if expected.len() != current.len() {
2575        return false;
2576    }
2577    let Some(mut current_parts) = crate::data_movement::data_movement_parts_by_number(current) else {
2578        return false;
2579    };
2580
2581    expected.iter().all(|expected_part| {
2582        current_parts.remove(&expected_part.number).is_some_and(|current_part| {
2583            current_part.size == expected_part.size
2584                && current_part.actual_size == expected_part.actual_size
2585                && current_part.etag == expected_part.etag
2586        })
2587    })
2588}
2589
2590impl RemoteTuplePublicationFence {
2591    pub(crate) fn under_fixed_read_anchor(mut self, anchor: DecommissionFixedReadAnchor) -> Result<Self> {
2592        if self.include_fixed_domain {
2593            return Err(Error::other(
2594                "data movement publication capability cannot attach a fixed read anchor after requesting fixed write",
2595            ));
2596        }
2597        if anchor.is_lock_lost() {
2598            return Err(Error::other(
2599                "data movement publication capability received an already-lost fixed read anchor",
2600            ));
2601        }
2602        self.include_fixed_domain = self.backend_target.is_some();
2603        self.fixed_read_anchor = Some(anchor);
2604        Ok(self)
2605    }
2606
2607    pub(crate) fn fixed_read_anchor_guard(&self) -> Option<&ObjectLockDiagGuard> {
2608        self.fixed_read_anchor.as_ref().map(DecommissionFixedReadAnchor::guard)
2609    }
2610
2611    pub(crate) async fn into_commit_guard(
2612        self,
2613        target_pool_idx: usize,
2614        bucket: &str,
2615        object: &str,
2616    ) -> Result<RemoteTuplePublicationCommitGuard> {
2617        let Self {
2618            store,
2619            bucket: expected_bucket,
2620            object: expected_object,
2621            source_pool_idx,
2622            source,
2623            include_fixed_domain,
2624            fixed_read_anchor,
2625            backend_target,
2626        } = self;
2627        if bucket != expected_bucket || object != expected_object {
2628            return Err(Error::other_with_context(
2629                "data movement publication capability target mismatch",
2630                format!("expected {expected_bucket}/{expected_object}, got {bucket}/{object}"),
2631            ));
2632        }
2633        if fixed_read_anchor
2634            .as_ref()
2635            .is_some_and(DecommissionFixedReadAnchor::is_lock_lost)
2636        {
2637            return Err(Error::other("data movement publication fixed read anchor was lost before commit"));
2638        }
2639        // Never upgrade a held read lock in place. Releasing here keeps the
2640        // long transfer on a read anchor while the commit obtains a short
2641        // fixed-domain write lock in the global tier -> namespace order.
2642        drop(fixed_read_anchor);
2643        // Tier generation is acquired at the commit boundary, before any
2644        // namespace lock, preserving bucket -> tier -> namespace lock order.
2645        let backend_lease = if let Some((tier_name, backend_identity)) = backend_target {
2646            let manager = store.tier_config_mgr();
2647            Some(
2648                match backend_identity {
2649                    Some(identity) => {
2650                        TierConfigMgr::acquire_operation_lease_for_backend_identity(&manager, &tier_name, identity).await
2651                    }
2652                    None => TierConfigMgr::acquire_operation_lease(&manager, &tier_name).await,
2653                }
2654                .map_err(Error::other)?,
2655            )
2656        } else {
2657            None
2658        };
2659
2660        let guards = store
2661            .acquire_data_movement_publication_write_locks(bucket, object, source_pool_idx, target_pool_idx, include_fixed_domain)
2662            .await?;
2663        if guards.iter().any(ObjectLockDiagGuard::is_lock_lost)
2664            || backend_lease.as_ref().is_some_and(|lease| !lease.is_current_generation())
2665        {
2666            return Err(Error::other("data movement publication fence was lost before source revalidation"));
2667        }
2668
2669        let source_pool = store.pools.get(source_pool_idx).ok_or_else(|| {
2670            Error::other_with_context("invalid data movement source pool", format!("pool index {source_pool_idx}"))
2671        })?;
2672        let mut revalidated_tiered_source = None;
2673        let source_matches = match source {
2674            RemoteTuplePublicationSource::Object(expected) => {
2675                let lookup_opts = ObjectOptions {
2676                    versioned: expected.version_id.is_some(),
2677                    version_id: expected.version_id.map(|version_id| version_id.to_string()),
2678                    incl_free_versions: expected.transitioned_object.free_version,
2679                    include_part_checksums: true,
2680                    no_lock: true,
2681                    ..Default::default()
2682                };
2683                match source_pool.get_object_info(bucket, object, &lookup_opts).await {
2684                    Ok(current) => remote_tuple_publication_object_source_matches(&expected, &current),
2685                    Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => false,
2686                    Err(err) => return Err(err),
2687                }
2688            }
2689            RemoteTuplePublicationSource::Tiered(expected) => {
2690                let logical_object = decode_dir_object(object);
2691                let current_versions = source_pool
2692                    .get_disks_by_key(object)
2693                    .load_file_info_versions_exact(bucket, &logical_object)
2694                    .await?;
2695                let is_free_version = expected.tier_free_version();
2696                let Some(current) = current_versions.as_ref().and_then(|versions| {
2697                    versions.versions.iter().find(|current| {
2698                        current.version_id == expected.version_id && current.tier_free_version() == is_free_version
2699                    })
2700                }) else {
2701                    return Err(to_object_err(StorageError::FileNotFound, vec![bucket, object]));
2702                };
2703                if tiered_data_movement_source_matches(&expected, current)? {
2704                    revalidated_tiered_source = Some(current.clone());
2705                    true
2706                } else {
2707                    false
2708                }
2709            }
2710        };
2711        if !source_matches {
2712            return Err(StorageError::DataMovementOverwriteErr(
2713                bucket.to_string(),
2714                object.to_string(),
2715                String::new(),
2716            ));
2717        }
2718
2719        let commit_guard = RemoteTuplePublicationCommitGuard {
2720            guards,
2721            backend_lease,
2722            revalidated_tiered_source,
2723        };
2724        if commit_guard.is_lock_lost() {
2725            return Err(Error::other("data movement publication fence was lost after source revalidation"));
2726        }
2727        Ok(commit_guard)
2728    }
2729}
2730
2731impl ECStore {
2732    /// Acquire the pre-copy half of a data-movement publication fence. Legacy
2733    /// remote objects without a destination identity pin the named tier; an
2734    /// ordinary local source takes the zero-tier-manager-work `None` lease
2735    /// branch but still receives source-at-commit validation.
2736    pub(crate) async fn acquire_remote_tuple_publication_fence(
2737        self: &Arc<Self>,
2738        bucket: &str,
2739        source_pool_idx: usize,
2740        source: &ObjectInfo,
2741        include_fixed_domain: bool,
2742    ) -> Result<RemoteTuplePublicationFence> {
2743        if self.pools.get(source_pool_idx).is_none() {
2744            return Err(Error::other_with_context(
2745                "data movement source pool is out of range",
2746                format!("pool index {source_pool_idx} for {bucket}/{}", source.name),
2747            ));
2748        }
2749        let backend_target = remote_tuple_publication_target(source).map_err(Error::other)?;
2750        let include_fixed_domain = include_fixed_domain && backend_target.is_some();
2751        Ok(RemoteTuplePublicationFence {
2752            store: Arc::clone(self),
2753            bucket: bucket.to_string(),
2754            object: encode_dir_object(&source.name),
2755            source_pool_idx,
2756            source: RemoteTuplePublicationSource::Object(source.clone()),
2757            include_fixed_domain,
2758            fixed_read_anchor: None,
2759            backend_target,
2760        })
2761    }
2762
2763    async fn acquire_tiered_remote_tuple_publication_fence(
2764        self: &Arc<Self>,
2765        bucket: &str,
2766        object: &str,
2767        source_pool_idx: usize,
2768        source: &rustfs_filemeta::FileInfo,
2769    ) -> Result<RemoteTuplePublicationFence> {
2770        let object_info = ObjectInfo::from_file_info(source, bucket, object, true);
2771        let Some((tier_name, backend_identity)) = remote_tuple_publication_target(&object_info).map_err(Error::other)? else {
2772            return Err(Error::other("tiered data movement source has no publishable remote tuple"));
2773        };
2774        Ok(RemoteTuplePublicationFence {
2775            store: Arc::clone(self),
2776            bucket: bucket.to_string(),
2777            object: encode_dir_object(object),
2778            source_pool_idx,
2779            source: RemoteTuplePublicationSource::Tiered(source.clone()),
2780            include_fixed_domain: true,
2781            fixed_read_anchor: None,
2782            backend_target: Some((tier_name, backend_identity)),
2783        })
2784    }
2785
2786    pub(crate) fn is_equivalent_decommission_capacity_target(source: &ObjectInfo, target: &ObjectInfo) -> bool {
2787        source.bucket == target.bucket
2788            && source.name == decode_dir_object(&target.name)
2789            && if source.delete_marker {
2790                is_equivalent_data_movement_delete_marker(source, target)
2791            } else {
2792                crate::data_movement::is_equivalent_data_movement_object_identity(source, target, true, false)
2793            }
2794    }
2795
2796    /// Captures Object Lock state once for a batch of PUTs to the same bucket.
2797    /// `handle_put_object` only reuses the token for the same store, bucket,
2798    /// bucket incarnation, and Object Lock configuration revision.
2799    pub async fn object_lock_config_snapshot(&self, bucket: &str) -> Result<Arc<ObjectLockConfigSnapshot>> {
2800        check_valid_bucket_name(bucket)?;
2801        let lifecycle_guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
2802        let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
2803        let (state, bucket_incarnation_id, config_revision) =
2804            get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
2805        if lifecycle_guard.is_lock_lost() || metadata_guard.is_lock_lost() {
2806            return Err(Error::other("bucket lifecycle lock was lost while loading the Object Lock snapshot"));
2807        }
2808        Ok(Arc::new(ObjectLockConfigSnapshot::for_guarded_store_bucket(
2809            self.id,
2810            bucket,
2811            bucket_incarnation_id,
2812            config_revision,
2813            state,
2814            lifecycle_guard,
2815            metadata_guard,
2816        )))
2817    }
2818
2819    pub(super) async fn object_lock_config_snapshot_under_lifecycle_fence(
2820        &self,
2821        bucket: &str,
2822        lifecycle_fence: &NamespaceLockFence,
2823    ) -> Result<Arc<ObjectLockConfigSnapshot>> {
2824        if lifecycle_fence.is_lock_lost() {
2825            return Err(Error::other("bucket lifecycle lock was lost before loading the Object Lock snapshot"));
2826        }
2827        let metadata_guard = acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?;
2828        let (state, bucket_incarnation_id, config_revision) =
2829            get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
2830        if lifecycle_fence.is_lock_lost() || metadata_guard.is_lock_lost() {
2831            return Err(Error::other("bucket lock was lost while loading the Object Lock snapshot"));
2832        }
2833        Ok(Arc::new(ObjectLockConfigSnapshot::for_store_bucket_under_lifecycle_fence(
2834            self.id,
2835            bucket,
2836            bucket_incarnation_id,
2837            config_revision,
2838            state,
2839            lifecycle_fence.clone(),
2840            metadata_guard,
2841        )))
2842    }
2843
2844    /// Resolves a GET's object identity without constructing its body reader.
2845    ///
2846    /// This is an additive two-stage counterpart to `get_object_reader`. The
2847    /// existing method remains the compatibility path for callers that do not
2848    /// need a pre-reader decision point.
2849    pub async fn prepare_select_object_snapshot(
2850        &self,
2851        bucket: &str,
2852        object: &str,
2853        headers: &HeaderMap,
2854        opts: &ObjectOptions,
2855    ) -> std::result::Result<SelectObjectSnapshot, PrepareSelectObjectSnapshotError> {
2856        check_get_obj_args(bucket, object)?;
2857
2858        let object = encode_dir_object(object);
2859        let mut opts = opts.clone();
2860        opts.no_lock = false;
2861        opts.metadata_cache_safe = false;
2862        let read_lock_guards = self
2863            .acquire_all_object_read_locks("select_object", bucket, &object, &mut opts)
2864            .await?;
2865        if self.ctx.lock_manager().is_disabled() {
2866            return Err(SnapshotConsistencyError::LockingDisabled.into());
2867        }
2868        if read_lock_guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
2869            return Err(SnapshotConsistencyError::LockLost.into());
2870        }
2871
2872        let (mut metadata, pool) = if self.single_pool() {
2873            let pool = Arc::clone(&self.pools[0]);
2874            let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?;
2875            (metadata, pool)
2876        } else {
2877            // Keep the large multi-pool selection future off the caller stack.
2878            // Debug builds otherwise exceed the common 2 MiB worker stack.
2879            Box::pin(async {
2880                let (metadata, pool_idx) = self.prepare_latest_object_metadata_with_idx(bucket, &object, &opts).await?;
2881                let pool = self.pools.get(pool_idx).cloned().ok_or_else(|| {
2882                    StorageError::other(format!("resolved SelectObjectContent pool index {pool_idx} is out of bounds"))
2883                })?;
2884                Ok::<_, StorageError>((metadata, pool))
2885            })
2886            .await?
2887        };
2888        if read_lock_guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
2889            return Err(SnapshotConsistencyError::LockLost.into());
2890        }
2891        if let Some(error) = latest_object_access_delete_marker_error(bucket, &object, metadata.object_info(), &opts) {
2892            return Err(error.into());
2893        }
2894
2895        let logical_size_i64 = metadata.object_info().get_actual_size().map_err(StorageError::from)?;
2896        let logical_size = u64::try_from(logical_size_i64)
2897            .map_err(|_| PrepareSelectObjectSnapshotError::InvalidLogicalSize { size: logical_size_i64 })?;
2898        let read_semantics_identity = metadata.read_semantics_identity();
2899        let object_info = metadata.take_object_info();
2900        if read_lock_guards.iter().any(ObjectLockDiagGuard::is_lock_lost) {
2901            return Err(SnapshotConsistencyError::LockLost.into());
2902        }
2903
2904        Ok(SelectObjectSnapshot {
2905            pool,
2906            bucket: bucket.to_owned(),
2907            object,
2908            headers: rustfs_utils::http::project_ssec_transport_headers(headers),
2909            opts,
2910            object_info,
2911            logical_size,
2912            read_semantics_identity,
2913            first_metadata: ParkingMutex::new(Some(metadata)),
2914            lease: Arc::new(SelectObjectSnapshotLease::new(read_lock_guards)),
2915        })
2916    }
2917
2918    pub async fn prepare_get_object_reader(
2919        &self,
2920        bucket: &str,
2921        object: &str,
2922        range: Option<HTTPRangeSpec>,
2923        headers: HeaderMap,
2924        opts: &ObjectOptions,
2925    ) -> Result<PreparedGetObjectReader> {
2926        check_get_obj_args(bucket, object)?;
2927
2928        let object = encode_dir_object(object);
2929        let mut opts = opts.clone();
2930        let read_lock_guard = self
2931            .acquire_object_read_lock_if_needed("prepare_get_object", bucket, &object, &mut opts)
2932            .await?;
2933
2934        let (metadata, pool) = if self.single_pool() {
2935            let pool = Arc::clone(&self.pools[0]);
2936            let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?;
2937            (metadata, pool)
2938        } else {
2939            // Keep the large multi-pool selection future off the caller stack.
2940            // Debug builds otherwise exceed the common 2 MiB worker stack.
2941            Box::pin(async {
2942                let (metadata, pool_idx) = self.prepare_latest_object_metadata_with_idx(bucket, &object, &opts).await?;
2943                if let Some(error) = latest_object_access_delete_marker_error(bucket, &object, metadata.object_info(), &opts) {
2944                    return Err(error);
2945                }
2946                let pool = self
2947                    .pools
2948                    .get(pool_idx)
2949                    .cloned()
2950                    .ok_or_else(|| Error::other(format!("resolved GET pool index {pool_idx} is out of bounds")))?;
2951                Ok((metadata, pool))
2952            })
2953            .await?
2954        };
2955
2956        Ok(PreparedGetObjectReader {
2957            pool,
2958            bucket: bucket.to_owned(),
2959            object,
2960            range,
2961            headers,
2962            opts,
2963            metadata,
2964            read_lock_guard,
2965        })
2966    }
2967
2968    fn map_namespace_lock_error(bucket: &str, object: &str, mode: &'static str, err: rustfs_lock::LockError) -> StorageError {
2969        match err {
2970            rustfs_lock::LockError::QuorumNotReached { required, achieved } => StorageError::NamespaceLockQuorumUnavailable {
2971                mode,
2972                bucket: bucket.to_string(),
2973                object: object.to_string(),
2974                required,
2975                achieved,
2976            },
2977            other => StorageError::Lock(other),
2978        }
2979    }
2980
2981    pub(super) async fn acquire_object_write_lock(
2982        &self,
2983        op: &'static str,
2984        bucket: &str,
2985        object: &str,
2986    ) -> Result<ObjectLockDiagGuard> {
2987        let diag_enabled = is_object_lock_diag_enabled();
2988        let ns_lock = self.handle_new_ns_lock(bucket, object).await?;
2989        let acquire_start = Instant::now();
2990        #[cfg(any(test, feature = "test-util"))]
2991        if matches!(op, "delete_object" | "delete_objects") {
2992            notify_delete_namespace_pending(bucket);
2993        }
2994        let timeout = get_lock_acquire_timeout();
2995        let mut attempt = ObjectLockAttempt::start(op, bucket, object, None, &ns_lock, "write", timeout);
2996        let result = ns_lock.get_write_lock(timeout).await;
2997        attempt.observe(&result);
2998        let guard = result.map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?;
2999        let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3000        log_object_lock_acquire_if_slow(
3001            op,
3002            bucket,
3003            object,
3004            owner.as_deref(),
3005            ObjectLockDiagMode::Write,
3006            acquire_start.elapsed(),
3007            diag_enabled,
3008        );
3009
3010        Ok(ObjectLockDiagGuard::new(
3011            guard,
3012            diag_enabled,
3013            op,
3014            diag_enabled.then(|| bucket.to_string()),
3015            diag_enabled.then(|| object.to_string()),
3016            owner,
3017            ObjectLockDiagMode::Write,
3018        )
3019        .with_attempt(attempt))
3020    }
3021
3022    async fn acquire_object_write_lock_if_needed(
3023        &self,
3024        op: &'static str,
3025        bucket: &str,
3026        object: &str,
3027        opts: &mut ObjectOptions,
3028    ) -> Result<Option<ObjectLockDiagGuard>> {
3029        if opts.no_lock {
3030            return Ok(None);
3031        }
3032
3033        let guard = self.acquire_object_write_lock(op, bucket, object).await?;
3034        if let Some(signal) = guard.lock_lost_signal() {
3035            opts.add_namespace_lock_lost_signal(signal);
3036        }
3037        opts.ensure_namespace_lock_fence();
3038        opts.no_lock = true;
3039
3040        Ok(Some(guard))
3041    }
3042
3043    /// Serializes the RestoreObject accept path — the read of the current
3044    /// `x-amz-restore` status, the ongoing/already-restored decision, and the
3045    /// metadata write that flips `ongoing-request="true"` — against concurrent
3046    /// accepts of the same object, making that read-check-write an atomic
3047    /// compare-and-set (backlog#1304). While the guard is held the caller must
3048    /// pass `no_lock: true` on its reads/writes of this object, check
3049    /// [`RestoreAcceptGuard::is_lock_lost`] before the status write, and drop
3050    /// the guard before starting the tier copy-back so concurrent
3051    /// HEAD/`get_object_info` stay non-blocking during the restore.
3052    pub async fn acquire_restore_accept_guard(&self, bucket: &str, object: &str) -> Result<RestoreAcceptGuard> {
3053        let object = encode_dir_object(object);
3054        let guard = self
3055            .acquire_object_write_lock("restore_object_accept", bucket, &object)
3056            .await?;
3057        Ok(RestoreAcceptGuard(guard))
3058    }
3059
3060    /// Acquire the liveness lock for a newly generated restore UUID using the
3061    /// normal namespace-lock timeout. New generations cannot legitimately
3062    /// contend, but a distributed quorum can take longer than the short orphan
3063    /// probe window under load.
3064    pub async fn acquire_restore_worker_guard(&self, operation_id: Uuid) -> Result<RestoreWorkerGuard> {
3065        let object = format!("{RESTORE_WORKER_LOCK_PREFIX}/{operation_id}");
3066        let lock = self.handle_new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
3067        lock.get_write_lock_quiet(get_lock_acquire_timeout())
3068            .await
3069            .map(RestoreWorkerGuard)
3070            .map_err(|err| Self::map_namespace_lock_error(RUSTFS_META_BUCKET, &object, "restore_worker", err))
3071    }
3072
3073    /// Probe the liveness lock for an already-persisted restore UUID.
3074    /// Contention is returned as `None`; infrastructure/quorum failures remain
3075    /// typed errors so callers fail closed instead of reaping an active worker.
3076    pub async fn try_acquire_restore_worker_guard(&self, operation_id: Uuid) -> Result<Option<RestoreWorkerGuard>> {
3077        let object = format!("{RESTORE_WORKER_LOCK_PREFIX}/{operation_id}");
3078        let lock = self.handle_new_ns_lock(RUSTFS_META_BUCKET, &object).await?;
3079        match lock.get_write_lock_quiet(RESTORE_WORKER_LOCK_PROBE_TIMEOUT).await {
3080            Ok(guard) => Ok(Some(RestoreWorkerGuard(guard))),
3081            Err(rustfs_lock::LockError::Timeout { .. } | rustfs_lock::LockError::AlreadyLocked { .. }) => Ok(None),
3082            Err(err) => Err(Self::map_namespace_lock_error(RUSTFS_META_BUCKET, &object, "restore_worker", err)),
3083        }
3084    }
3085
3086    async fn acquire_delete_objects_write_locks(
3087        &self,
3088        bucket: &str,
3089        objects: &[ObjectToDelete],
3090        opts: &mut ObjectOptions,
3091    ) -> Result<Vec<ObjectLockDiagGuard>> {
3092        if opts.no_lock || objects.is_empty() {
3093            return Ok(Vec::new());
3094        }
3095
3096        let object_names = sorted_unique_delete_object_names(objects);
3097        // Lock order: encoded object names are acquired in ascending order, then
3098        // the set-layer calls receive no_lock so they do not reacquire them.
3099        let mut guards = Vec::with_capacity(object_names.len());
3100        for object in object_names {
3101            guards.push(self.acquire_object_write_lock("delete_objects", bucket, object).await?);
3102        }
3103        opts.no_lock = true;
3104        for signal in guards.iter().filter_map(ObjectLockDiagGuard::lock_lost_signal) {
3105            opts.add_namespace_lock_lost_signal(signal);
3106        }
3107        opts.ensure_namespace_lock_fence();
3108
3109        Ok(guards)
3110    }
3111
3112    async fn acquire_object_read_lock_if_needed(
3113        &self,
3114        op: &'static str,
3115        bucket: &str,
3116        object: &str,
3117        opts: &mut ObjectOptions,
3118    ) -> Result<Option<ObjectLockDiagGuard>> {
3119        if opts.no_lock {
3120            return Ok(None);
3121        }
3122
3123        let diag_enabled = is_object_lock_diag_enabled();
3124        let ns_lock = self.handle_new_ns_lock(bucket, object).await?;
3125        let acquire_start = Instant::now();
3126        let timeout = get_lock_acquire_timeout();
3127        let mut attempt = ObjectLockAttempt::start(op, bucket, object, opts.version_id.as_deref(), &ns_lock, "read", timeout);
3128        let result = ns_lock.get_read_lock(timeout).await;
3129        attempt.observe(&result);
3130        let guard = result.map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?;
3131        let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3132        log_object_lock_acquire_if_slow(
3133            op,
3134            bucket,
3135            object,
3136            owner.as_deref(),
3137            ObjectLockDiagMode::Read,
3138            acquire_start.elapsed(),
3139            diag_enabled,
3140        );
3141        opts.no_lock = true;
3142        opts.metadata_cache_safe = true;
3143
3144        Ok(Some(
3145            ObjectLockDiagGuard::new(
3146                guard,
3147                diag_enabled,
3148                op,
3149                diag_enabled.then(|| bucket.to_string()),
3150                diag_enabled.then(|| object.to_string()),
3151                owner,
3152                ObjectLockDiagMode::Read,
3153            )
3154            .with_attempt(attempt),
3155        ))
3156    }
3157
3158    /// Publish a native scanner replica without allowing stale pool selection
3159    /// to race retirement. Failed/canceled membership repair remains permitted.
3160    /// A canceled waiter cannot release publication fences from an in-flight write.
3161    pub async fn save_scanner_pause_backlog_replica(
3162        self: &Arc<Self>,
3163        pool_index: usize,
3164        set_index: usize,
3165        data: Vec<u8>,
3166        preconditions: crate::storage_api_contracts::object::HTTPPreconditions,
3167    ) -> Result<ObjectInfo> {
3168        let set = self
3169            .pools
3170            .get(pool_index)
3171            .and_then(|pool| pool.disk_set.get(set_index))
3172            .ok_or_else(|| Error::InvalidArgument("scanner-backlog".into(), "replica".into(), "unknown pool or set".into()))?;
3173        let set = Arc::clone(set);
3174        let store = Arc::clone(self);
3175        let write = async move {
3176            let object = "buckets/.scanner-pause-backlog.json";
3177            let mut opts = ObjectOptions::default();
3178            // Match migration: fixed object namespace -> durable pool metadata ->
3179            // actual replica namespace. The replica need not be the hash-routed set.
3180            let object_guard = if store.single_pool() {
3181                None
3182            } else {
3183                Some(
3184                    store
3185                        .acquire_object_write_lock("scanner_backlog_replica", RUSTFS_META_BUCKET, object)
3186                        .await?,
3187                )
3188            };
3189            let capacity_guard = if let Some(guard) = object_guard.as_ref() {
3190                guard.add_namespace_lock_fence(&mut opts);
3191                opts.no_lock = match store.pools.first().and_then(|pool| pool.disk_set.first()) {
3192                    Some(fixed) => fixed.shares_namespace_lock_domain(&set).await,
3193                    None => false,
3194                };
3195                let capacity_guard = store
3196                    .acquire_external_decommission_capacity_fence(&[pool_index], DecommissionCapacityAdmission::ScannerBacklog)
3197                    .await?;
3198                opts.add_namespace_lock_guard(&capacity_guard);
3199                Some(capacity_guard)
3200            } else {
3201                None
3202            };
3203            let result = crate::data_movement::scanner_backlog::persist_native_scanner_pause_backlog_replica(
3204                set,
3205                data,
3206                preconditions,
3207                opts,
3208                "publish",
3209            )
3210            .await;
3211            drop(capacity_guard);
3212            drop(object_guard);
3213            result
3214        };
3215        // The set layer may detach its rename owner, even for full-tail writes.
3216        // Keep these outer guards alive until that owner finishes if scanner exits.
3217        tokio::spawn(write).await.map_err(Error::from)?
3218    }
3219
3220    pub(super) async fn run_external_decommission_capacity_object_mutation<T, F, Fut>(
3221        &self,
3222        target_pool_idx: usize,
3223        bucket: &str,
3224        lock_object: &str,
3225        target_object: &str,
3226        opts: ObjectOptions,
3227        operation: F,
3228    ) -> Result<T>
3229    where
3230        F: FnOnce(ObjectOptions) -> Fut,
3231        Fut: std::future::Future<Output = Result<T>>,
3232    {
3233        self.run_external_decommission_capacity_object_operation(
3234            target_pool_idx,
3235            bucket,
3236            (lock_object, target_object),
3237            opts,
3238            false,
3239            operation,
3240        )
3241        .await
3242    }
3243
3244    pub(super) async fn run_external_decommission_capacity_object_delete<T, F, Fut>(
3245        &self,
3246        target_pool_idx: usize,
3247        bucket: &str,
3248        lock_object: &str,
3249        target_object: &str,
3250        opts: ObjectOptions,
3251        operation: F,
3252    ) -> Result<T>
3253    where
3254        F: FnOnce(ObjectOptions) -> Fut,
3255        Fut: std::future::Future<Output = Result<T>>,
3256    {
3257        let capacity_releasing = delete_only_releases_capacity(&opts);
3258        self.run_external_decommission_capacity_object_operation(
3259            target_pool_idx,
3260            bucket,
3261            (lock_object, target_object),
3262            opts,
3263            capacity_releasing,
3264            operation,
3265        )
3266        .await
3267    }
3268
3269    async fn run_external_decommission_capacity_object_operation<T, F, Fut>(
3270        &self,
3271        target_pool_idx: usize,
3272        bucket: &str,
3273        objects: (&str, &str),
3274        mut opts: ObjectOptions,
3275        capacity_releasing: bool,
3276        operation: F,
3277    ) -> Result<T>
3278    where
3279        F: FnOnce(ObjectOptions) -> Fut,
3280        Fut: std::future::Future<Output = Result<T>>,
3281    {
3282        let (lock_object, target_object) = objects;
3283        if self.single_pool() {
3284            return operation(opts).await;
3285        }
3286        let (capacity_guard, has_active_decommission) = if capacity_releasing {
3287            self.acquire_decommission_capacity_release_fence_with_active_source().await?
3288        } else {
3289            self.acquire_external_decommission_capacity_fence_with_active_source(
3290                &[target_pool_idx],
3291                DecommissionCapacityAdmission::Mutation,
3292            )
3293            .await?
3294        };
3295        let (capacity_guard, object_guard) = if has_active_decommission && !opts.no_lock {
3296            // Active migration acquires the object namespace before its capacity
3297            // write. Match that order, then recheck capacity admission.
3298            drop(capacity_guard);
3299            #[cfg(test)]
3300            crate::core::pools::notify_decommission_external_object_capacity_released(self.id);
3301            let guard = self
3302                .acquire_object_write_lock("external_capacity_order", bucket, lock_object)
3303                .await?;
3304            self.apply_decommission_target_mutation_fence(target_pool_idx, target_object, &mut opts, Some(&guard))
3305                .await;
3306            let capacity_guard = if capacity_releasing {
3307                self.acquire_decommission_capacity_release_fence_with_active_source().await?.0
3308            } else {
3309                self.acquire_external_decommission_capacity_fence(&[target_pool_idx], DecommissionCapacityAdmission::Mutation)
3310                    .await?
3311            };
3312            (capacity_guard, Some(guard))
3313        } else {
3314            (capacity_guard, None)
3315        };
3316
3317        let result = operation(opts).await;
3318        drop(capacity_guard);
3319        drop(object_guard);
3320        result
3321    }
3322
3323    /// Finish an external object mutation's staged phase by acquiring the
3324    /// fixed namespace before the target-side commit lock. The caller must
3325    /// recheck capacity only after all applicable namespaces are held.
3326    ///
3327    /// Callers must invoke this only after consuming and staging their input
3328    /// stream. The returned namespace guard remains owned by the caller until
3329    /// its local commit completes; `target_lock_covered` tells the set layer
3330    /// whether that guard also covers its target namespace. When no
3331    /// decommission is active, the returned capacity guard is the admission
3332    /// probe and must likewise remain held until the commit completes.
3333    pub(crate) async fn acquire_external_decommission_commit_guards(
3334        &self,
3335        target_pool_idx: usize,
3336        bucket: &str,
3337        object: &str,
3338        no_lock: bool,
3339        admission: DecommissionCapacityAdmission,
3340    ) -> Result<(Option<ObjectLockDiagGuard>, bool, Option<rustfs_lock::NamespaceLockGuard>)> {
3341        let (capacity_guard, has_active_decommission) = self
3342            .acquire_external_decommission_capacity_fence_with_active_source(&[target_pool_idx], admission)
3343            .await?;
3344        if !has_active_decommission {
3345            // Keep the read probe through the staged commit. This closes the
3346            // activation gap between admission and publication.
3347            #[cfg(test)]
3348            {
3349                crate::core::pools::notify_decommission_external_object_capacity_probe_acquired(self.id);
3350                crate::core::pools::wait_for_decommission_external_object_capacity_probe_release(self.id).await;
3351            }
3352            return Ok((None, false, Some(capacity_guard)));
3353        }
3354        drop(capacity_guard);
3355        if no_lock {
3356            return Ok((None, false, None));
3357        }
3358
3359        // Active migration holds the fixed object namespace before taking its
3360        // capacity write fence. Drop the probe and acquire that namespace
3361        // before rechecking capacity, so the staged external commit cannot
3362        // form capacity-read -> object-write against the migration path.
3363        let object_guard = self
3364            .acquire_object_write_lock("external_capacity_order", bucket, object)
3365            .await?;
3366        let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first());
3367        let target_set = self.pools.get(target_pool_idx).map(|pool| pool.get_disks_by_key(object));
3368        let target_lock_covered = match (fixed_set, target_set) {
3369            (Some(fixed), Some(target)) => fixed.shares_namespace_lock_domain(&target).await,
3370            _ => false,
3371        };
3372        Ok((Some(object_guard), target_lock_covered, None))
3373    }
3374
3375    pub(super) async fn run_external_decommission_capacity_heal<T, F, Fut>(
3376        &self,
3377        target_pool_idx: usize,
3378        bucket: &str,
3379        lock_object: &str,
3380        mut opts: HealOpts,
3381        operation: F,
3382    ) -> Result<T>
3383    where
3384        F: FnOnce(HealOpts) -> Fut,
3385        Fut: std::future::Future<Output = Result<T>>,
3386    {
3387        if self.single_pool() {
3388            return operation(opts).await;
3389        }
3390        let (capacity_guard, has_active_decommission) = self
3391            .acquire_external_decommission_capacity_fence_with_active_source(
3392                &[target_pool_idx],
3393                DecommissionCapacityAdmission::Heal,
3394            )
3395            .await?;
3396        let (capacity_guard, object_guard) = if has_active_decommission && !opts.no_lock {
3397            // Active migration acquires the object namespace before its capacity
3398            // write. Match that order, then recheck capacity admission.
3399            drop(capacity_guard);
3400            #[cfg(test)]
3401            crate::core::pools::notify_decommission_external_object_capacity_released(self.id);
3402            let guard = self
3403                .acquire_object_write_lock("external_capacity_order", bucket, lock_object)
3404                .await?;
3405            let target_set = self
3406                .pools
3407                .get(target_pool_idx)
3408                .ok_or_else(|| Error::other("heal target pool is unavailable"))?
3409                .get_disks_for_heal_object(lock_object, &opts)?;
3410            let target_lock_covered = match self.pools.first().and_then(|pool| pool.disk_set.first()) {
3411                Some(fixed_set) => fixed_set.shares_namespace_lock_domain(&target_set).await,
3412                None => false,
3413            };
3414            let capacity_guard = self
3415                .acquire_external_decommission_capacity_fence(&[target_pool_idx], DecommissionCapacityAdmission::Heal)
3416                .await?;
3417            opts.no_lock = target_lock_covered;
3418            (capacity_guard, Some(guard))
3419        } else {
3420            (capacity_guard, None)
3421        };
3422
3423        #[cfg(test)]
3424        if !opts.no_lock {
3425            crate::core::pools::notify_decommission_external_heal_target_lock_attempted();
3426        }
3427        let result = operation(opts).await;
3428        drop(capacity_guard);
3429        drop(object_guard);
3430        result
3431    }
3432
3433    pub(crate) async fn acquire_decommission_object_mutation_fence(
3434        &self,
3435        bucket: &str,
3436        object: &str,
3437    ) -> Result<DecommissionFixedReadAnchor> {
3438        if self.ctx.lock_manager().is_disabled() {
3439            return Err(Error::other("decommission object migration requires namespace locking"));
3440        }
3441
3442        #[cfg(test)]
3443        let test_namespace_lock_fence =
3444            decommission_mutation_fence_for_test(bucket, object, DecommissionMutationFenceTestPhase::Migration);
3445        let object = encode_dir_object(object);
3446        let mut opts = ObjectOptions::default();
3447        let guard = self
3448            .acquire_object_read_lock_if_needed("decommission_object", bucket, &object, &mut opts)
3449            .await?
3450            .ok_or_else(|| Error::other("decommission object migration failed to acquire its namespace fence"))?;
3451        #[cfg(test)]
3452        let guard = {
3453            let mut guard = guard;
3454            guard.test_namespace_lock_fence = test_namespace_lock_fence;
3455            guard
3456        };
3457        Ok(DecommissionFixedReadAnchor { guard })
3458    }
3459
3460    pub(super) async fn apply_decommission_target_mutation_fence(
3461        &self,
3462        target_pool_idx: usize,
3463        object: &str,
3464        opts: &mut ObjectOptions,
3465        mutation_fence: Option<&ObjectLockDiagGuard>,
3466    ) {
3467        let Some(mutation_fence) = mutation_fence else {
3468            return;
3469        };
3470
3471        mutation_fence.add_namespace_lock_fence(opts);
3472        let fixed_set = self.pools.first().and_then(|pool| pool.disk_set.first());
3473        let target_set = self.pools.get(target_pool_idx).map(|pool| pool.get_disks_by_key(object));
3474        opts.no_lock = match (fixed_set, target_set) {
3475            (Some(fixed), Some(target)) => fixed.shares_namespace_lock_domain(&target).await,
3476            _ => false,
3477        };
3478    }
3479
3480    pub(crate) async fn acquire_decommission_source_cleanup_fence(
3481        &self,
3482        bucket: &str,
3483        object: &str,
3484        source_set: &SetDisks,
3485    ) -> Result<SourceCleanupMutationFence> {
3486        if self.ctx.lock_manager().is_disabled() {
3487            return Err(Error::other("decommission source cleanup requires namespace locking"));
3488        }
3489
3490        #[cfg(test)]
3491        crate::data_movement::notify_source_cleanup_mutation_fence_pending(bucket, object);
3492        #[cfg(test)]
3493        let test_namespace_lock_fence =
3494            decommission_mutation_fence_for_test(bucket, object, DecommissionMutationFenceTestPhase::SourceCleanup);
3495        let object = encode_dir_object(object);
3496        let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3497        let source_lock_covered = fixed_set.shares_namespace_lock_domain(source_set).await;
3498        // Lock order: fixed store mutation domain first; source cleanup takes its
3499        // hashed source-domain lock second only when this guard does not cover it.
3500        let guard = self
3501            .acquire_object_write_lock("decommission_source_cleanup", bucket, &object)
3502            .await?;
3503        #[cfg(test)]
3504        let guard = {
3505            let mut guard = guard;
3506            guard.test_namespace_lock_fence = test_namespace_lock_fence;
3507            guard
3508        };
3509
3510        Ok(SourceCleanupMutationFence {
3511            guard,
3512            source_lock_covered,
3513        })
3514    }
3515
3516    pub(crate) async fn acquire_all_object_read_locks(
3517        &self,
3518        op: &'static str,
3519        bucket: &str,
3520        object: &str,
3521        opts: &mut ObjectOptions,
3522    ) -> Result<Vec<ObjectLockDiagGuard>> {
3523        let diag_enabled = is_object_lock_diag_enabled();
3524        let mut guards = Vec::with_capacity(self.pools.len() + 1);
3525
3526        // Lock order is the store fixed domain first, then pool index ascending
3527        // for each object's hashed set. DELETE and same-key CopyObject use the
3528        // fixed domain, while PUT commits and data movement use the hashed set.
3529        let distributed = self.ctx.is_dist_erasure().await;
3530        if let Some(guard) = self.acquire_object_read_lock_if_needed(op, bucket, object, opts).await? {
3531            guards.push(guard);
3532        }
3533        let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3534        let mut locked_sets = vec![fixed_set];
3535
3536        for pool in &self.pools {
3537            let hashed_set = pool.get_disks_by_key(object);
3538            let mut lock_domain_already_held = !distributed;
3539            if !lock_domain_already_held {
3540                for locked_set in &locked_sets {
3541                    if locked_set.shares_namespace_lock_domain(&hashed_set).await {
3542                        lock_domain_already_held = true;
3543                        break;
3544                    }
3545                }
3546            }
3547            if lock_domain_already_held {
3548                continue;
3549            }
3550            let ns_lock = hashed_set.new_ns_lock(bucket, object).await?;
3551            let acquire_start = Instant::now();
3552            let guard = ns_lock
3553                .get_read_lock(get_lock_acquire_timeout())
3554                .await
3555                .map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?;
3556            let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3557            log_object_lock_acquire_if_slow(
3558                op,
3559                bucket,
3560                object,
3561                owner.as_deref(),
3562                ObjectLockDiagMode::Read,
3563                acquire_start.elapsed(),
3564                diag_enabled,
3565            );
3566            guards.push(ObjectLockDiagGuard::new(
3567                guard,
3568                diag_enabled,
3569                op,
3570                diag_enabled.then(|| bucket.to_string()),
3571                diag_enabled.then(|| object.to_string()),
3572                owner,
3573                ObjectLockDiagMode::Read,
3574            ));
3575            locked_sets.push(hashed_set);
3576        }
3577        Ok(guards)
3578    }
3579
3580    /// Acquire the fixed store domain followed by every physical set domain in
3581    /// stable `(pool_idx, set_idx)` order. This stronger form is reserved for
3582    /// remote-tuple destruction proofs: old topology generations and failed
3583    /// movements may leave a valid source outside the current hash-selected
3584    /// set, so a hashed-only lock/read is not authoritative.
3585    pub(crate) async fn acquire_all_physical_object_read_locks(
3586        &self,
3587        op: &'static str,
3588        bucket: &str,
3589        object: &str,
3590        opts: &mut ObjectOptions,
3591    ) -> Result<Vec<ObjectLockDiagGuard>> {
3592        let diag_enabled = is_object_lock_diag_enabled();
3593        let distributed = self.ctx.is_dist_erasure().await;
3594        let mut guards = Vec::new();
3595        if let Some(guard) = self.acquire_object_read_lock_if_needed(op, bucket, object, opts).await? {
3596            guards.push(guard);
3597        }
3598
3599        let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3600        let mut locked_sets = vec![fixed_set];
3601        for pool in &self.pools {
3602            for set in &pool.disk_set {
3603                let mut lock_domain_already_held = !distributed;
3604                if !lock_domain_already_held {
3605                    for locked_set in &locked_sets {
3606                        if locked_set.shares_namespace_lock_domain(set).await {
3607                            lock_domain_already_held = true;
3608                            break;
3609                        }
3610                    }
3611                }
3612                if lock_domain_already_held {
3613                    continue;
3614                }
3615                let ns_lock = set.new_ns_lock(bucket, object).await?;
3616                let acquire_start = Instant::now();
3617                let guard = ns_lock
3618                    .get_read_lock(get_lock_acquire_timeout())
3619                    .await
3620                    .map_err(|err| Self::map_namespace_lock_error(bucket, object, "read", err))?;
3621                let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3622                log_object_lock_acquire_if_slow(
3623                    op,
3624                    bucket,
3625                    object,
3626                    owner.as_deref(),
3627                    ObjectLockDiagMode::Read,
3628                    acquire_start.elapsed(),
3629                    diag_enabled,
3630                );
3631                guards.push(ObjectLockDiagGuard::new(
3632                    guard,
3633                    diag_enabled,
3634                    op,
3635                    diag_enabled.then(|| bucket.to_string()),
3636                    diag_enabled.then(|| object.to_string()),
3637                    owner,
3638                    ObjectLockDiagMode::Read,
3639                ));
3640                locked_sets.push(Arc::clone(set));
3641            }
3642        }
3643        Ok(guards)
3644    }
3645
3646    pub(crate) async fn acquire_all_physical_object_write_locks(
3647        &self,
3648        op: &'static str,
3649        bucket: &str,
3650        object: &str,
3651    ) -> Result<Vec<ObjectLockDiagGuard>> {
3652        self.acquire_physical_object_write_locks(op, bucket, object, false).await
3653    }
3654
3655    async fn acquire_remaining_physical_object_write_locks(
3656        &self,
3657        op: &'static str,
3658        bucket: &str,
3659        object: &str,
3660    ) -> Result<Vec<ObjectLockDiagGuard>> {
3661        self.acquire_physical_object_write_locks(op, bucket, object, true).await
3662    }
3663
3664    async fn acquire_physical_object_write_locks(
3665        &self,
3666        op: &'static str,
3667        bucket: &str,
3668        object: &str,
3669        store_lock_already_held: bool,
3670    ) -> Result<Vec<ObjectLockDiagGuard>> {
3671        // Caller-held bucket lifecycle and tier-generation guards precede the
3672        // fixed store domain, followed by every distinct physical set domain
3673        // in stable `(pool_idx, set_idx)` order. Exact-object predecessor
3674        // replay sets `store_lock_already_held` only while retaining that same
3675        // fixed-domain write lock.
3676        if self.ctx.lock_manager().is_disabled() {
3677            return Err(Error::other("physical object mutation requires namespace locking"));
3678        }
3679        let diag_enabled = is_object_lock_diag_enabled();
3680        let distributed = self.ctx.is_dist_erasure().await;
3681        let mut guards = Vec::new();
3682        if !store_lock_already_held {
3683            guards.push(self.acquire_object_write_lock(op, bucket, object).await?);
3684        }
3685        let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3686        let mut locked_sets = vec![fixed_set];
3687        for pool in &self.pools {
3688            for set in &pool.disk_set {
3689                let mut lock_domain_already_held = !distributed;
3690                if !lock_domain_already_held {
3691                    for locked_set in &locked_sets {
3692                        if locked_set.shares_namespace_lock_domain(set).await {
3693                            lock_domain_already_held = true;
3694                            break;
3695                        }
3696                    }
3697                }
3698                if lock_domain_already_held {
3699                    continue;
3700                }
3701                let ns_lock = set.new_ns_lock(bucket, object).await?;
3702                let acquire_start = Instant::now();
3703                let guard = ns_lock
3704                    .get_write_lock(get_lock_acquire_timeout())
3705                    .await
3706                    .map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?;
3707                let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3708                log_object_lock_acquire_if_slow(
3709                    op,
3710                    bucket,
3711                    object,
3712                    owner.as_deref(),
3713                    ObjectLockDiagMode::Write,
3714                    acquire_start.elapsed(),
3715                    diag_enabled,
3716                );
3717                guards.push(ObjectLockDiagGuard::new(
3718                    guard,
3719                    diag_enabled,
3720                    op,
3721                    diag_enabled.then(|| bucket.to_string()),
3722                    diag_enabled.then(|| object.to_string()),
3723                    owner,
3724                    ObjectLockDiagMode::Write,
3725                ));
3726                locked_sets.push(Arc::clone(set));
3727            }
3728        }
3729        Ok(guards)
3730    }
3731
3732    pub(crate) async fn acquire_data_movement_publication_write_locks(
3733        &self,
3734        bucket: &str,
3735        object: &str,
3736        source_pool_idx: usize,
3737        target_pool_idx: usize,
3738        include_fixed_domain: bool,
3739    ) -> Result<Vec<ObjectLockDiagGuard>> {
3740        if self.ctx.lock_manager().is_disabled() {
3741            return Err(Error::other("data movement publication requires namespace locking"));
3742        }
3743        let distributed = self.ctx.is_dist_erasure().await;
3744        let diag_enabled = is_object_lock_diag_enabled();
3745        let mut pool_indices = [source_pool_idx, target_pool_idx];
3746        pool_indices.sort_unstable();
3747        let fixed_set = Arc::clone(&self.pools[0].disk_set[0]);
3748        let mut locked_sets = Vec::with_capacity(3);
3749        let mut guards = Vec::with_capacity(3);
3750        #[cfg(test)]
3751        let test_namespace_lock_fence = decommission_mutation_fence_for_test(
3752            bucket,
3753            &decode_dir_object(object),
3754            DecommissionMutationFenceTestPhase::Migration,
3755        );
3756
3757        // Decommission publishers take the fixed store domain first to match
3758        // tier-delete recovery. Rebalance publishers do not need that global
3759        // serialization, but both paths take source/target domains in stable
3760        // pool order and revalidate the source before target publication.
3761        if include_fixed_domain {
3762            let guard = self
3763                .acquire_object_write_lock("data_movement_publication", bucket, object)
3764                .await?;
3765            #[cfg(test)]
3766            let guard = {
3767                let mut guard = guard;
3768                guard.test_namespace_lock_fence = test_namespace_lock_fence.clone();
3769                guard
3770            };
3771            guards.push(guard);
3772            locked_sets.push(fixed_set);
3773        }
3774        for pool_idx in pool_indices {
3775            let pool = self
3776                .pools
3777                .get(pool_idx)
3778                .ok_or_else(|| Error::other(format!("invalid data movement publication pool {pool_idx}")))?;
3779            let set = pool.get_disks_by_key(object);
3780            let mut lock_domain_already_held = !locked_sets.is_empty() && !distributed;
3781            if !lock_domain_already_held {
3782                for locked_set in &locked_sets {
3783                    if locked_set.shares_namespace_lock_domain(&set).await {
3784                        lock_domain_already_held = true;
3785                        break;
3786                    }
3787                }
3788            }
3789            if lock_domain_already_held {
3790                continue;
3791            }
3792            let ns_lock = set.new_ns_lock(bucket, object).await?;
3793            let acquire_start = Instant::now();
3794            let guard = ns_lock
3795                .get_write_lock(get_lock_acquire_timeout())
3796                .await
3797                .map_err(|err| Self::map_namespace_lock_error(bucket, object, "write", err))?;
3798            let owner = diag_enabled.then(|| ns_lock.owner().to_string());
3799            log_object_lock_acquire_if_slow(
3800                "data_movement_publication",
3801                bucket,
3802                object,
3803                owner.as_deref(),
3804                ObjectLockDiagMode::Write,
3805                acquire_start.elapsed(),
3806                diag_enabled,
3807            );
3808            let guard = ObjectLockDiagGuard::new(
3809                guard,
3810                diag_enabled,
3811                "data_movement_publication",
3812                diag_enabled.then(|| bucket.to_string()),
3813                diag_enabled.then(|| object.to_string()),
3814                owner,
3815                ObjectLockDiagMode::Write,
3816            );
3817            #[cfg(test)]
3818            let guard = {
3819                let mut guard = guard;
3820                if guards.is_empty() {
3821                    guard.test_namespace_lock_fence = test_namespace_lock_fence.clone();
3822                }
3823                guard
3824            };
3825            guards.push(guard);
3826            locked_sets.push(set);
3827        }
3828        Ok(guards)
3829    }
3830
3831    fn attach_read_lock_guard(mut reader: GetObjectReader, guard: Option<ObjectLockDiagGuard>) -> GetObjectReader {
3832        if is_lock_optimization_enabled() || reader.buffered_body.is_some() {
3833            return reader;
3834        }
3835
3836        if let Some(guard) = guard {
3837            reader.stream = Box::new(LockGuardedReader {
3838                inner: reader.stream,
3839                guard: Some(guard),
3840            });
3841        }
3842
3843        reader
3844    }
3845
3846    async fn get_latest_accessible_object_info_with_idx(
3847        &self,
3848        bucket: &str,
3849        object: &str,
3850        opts: &ObjectOptions,
3851    ) -> Result<(ObjectInfo, usize)> {
3852        let (info, idx) = self.get_latest_object_info_with_idx(bucket, object, opts).await?;
3853        resolve_latest_object_access(bucket, object, info, idx, opts)
3854    }
3855
3856    pub(crate) async fn select_data_movement_pool_idx(
3857        &self,
3858        bucket: &str,
3859        object: &str,
3860        size: i64,
3861        opts: &ObjectOptions,
3862        no_lock: bool,
3863    ) -> Result<usize> {
3864        let capacity_owner = DecommissionCapacityOwner::from_options(opts);
3865        match self
3866            .get_pool_info_existing_with_opts(bucket, object, &data_movement_pool_lookup_opts(opts, no_lock))
3867            .await
3868        {
3869            Ok((pinfo, _)) => {
3870                if let Some(owner) = capacity_owner {
3871                    if self.is_decommission_capacity_target_reserved(owner, pinfo.index).await? {
3872                        return Ok(pinfo.index);
3873                    }
3874                } else {
3875                    return Ok(pinfo.index);
3876                }
3877            }
3878            Err(err) => {
3879                if !is_err_object_not_found(&err) && !is_err_version_not_found(&err) {
3880                    return Err(err);
3881                }
3882            }
3883        }
3884        if let Some(owner) = capacity_owner {
3885            let expected_data_bytes = opts
3886                .capacity_expected_data_bytes()
3887                .or_else(|| usize::try_from(size).ok())
3888                .unwrap_or_default();
3889            return self
3890                .select_decommission_capacity_target_pool(owner, expected_data_bytes)
3891                .await;
3892        }
3893
3894        self.get_available_pool_idx(bucket, object, size).await.ok_or(Error::DiskFull)
3895    }
3896
3897    async fn find_data_movement_target_info(
3898        &self,
3899        bucket: &str,
3900        object: &str,
3901        target_pool_idx: usize,
3902        opts: &ObjectOptions,
3903    ) -> Result<Option<ObjectInfo>> {
3904        let mut lookup_opts = version_aware_lookup_opts(opts, true);
3905        lookup_opts.include_part_checksums = true;
3906
3907        let Some(pool) = self.pools.get(target_pool_idx) else {
3908            return Err(Error::other(format!(
3909                "data movement resume target pool {target_pool_idx} is out of range for {bucket}/{object}"
3910            )));
3911        };
3912
3913        match pool.get_object_info(bucket, object, &lookup_opts).await {
3914            Ok(info) => Ok(Some(info)),
3915            Err(err) if is_err_object_not_found(&err) || is_err_version_not_found(&err) => Ok(None),
3916            Err(err) => Err(err),
3917        }
3918    }
3919
3920    async fn has_equivalent_data_movement_delete_marker(
3921        &self,
3922        bucket: &str,
3923        object: &str,
3924        source: &ObjectInfo,
3925        opts: &ObjectOptions,
3926        target_pool_idx: usize,
3927    ) -> Result<bool> {
3928        let equivalent = resolve_data_movement_delete_marker_resume_result(
3929            self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
3930                .await,
3931            source,
3932            opts.src_pool_idx,
3933            target_pool_idx,
3934        )?;
3935        if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
3936            self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0)
3937                .await?;
3938        }
3939        Ok(equivalent)
3940    }
3941
3942    async fn has_equivalent_data_movement_tiered_object(
3943        &self,
3944        bucket: &str,
3945        object: &str,
3946        source: &rustfs_filemeta::FileInfo,
3947        opts: &ObjectOptions,
3948        target_pool_idx: usize,
3949    ) -> Result<bool> {
3950        let equivalent = resolve_data_movement_tiered_resume_result(
3951            self.find_data_movement_target_info(bucket, object, target_pool_idx, opts)
3952                .await,
3953            source,
3954            opts.src_pool_idx,
3955            target_pool_idx,
3956        )?;
3957        if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
3958            let expected_data_bytes = usize::try_from(source.size).unwrap_or_default();
3959            self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, expected_data_bytes)
3960                .await?;
3961        }
3962        Ok(equivalent)
3963    }
3964
3965    async fn has_equivalent_data_movement_tier_free_version(
3966        &self,
3967        bucket: &str,
3968        object: &str,
3969        source: &rustfs_filemeta::FileInfo,
3970        opts: &ObjectOptions,
3971        target_pool_idx: usize,
3972    ) -> Result<bool> {
3973        let pool = self
3974            .pools
3975            .get(target_pool_idx)
3976            .ok_or_else(|| Error::other(format!("invalid tiered data movement target pool {target_pool_idx}")))?;
3977        let equivalent = pool
3978            .get_disks_by_key(object)
3979            .has_decommission_tier_free_version_write_quorum(bucket, object, source, opts)
3980            .await?;
3981        if equivalent && let Some(owner) = DecommissionCapacityOwner::from_options(opts) {
3982            self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0)
3983                .await?;
3984        }
3985        Ok(equivalent)
3986    }
3987
3988    fn resolve_decommission_target_pool_idx_result(result: Result<usize>, bucket: &str, object: &str) -> Result<usize> {
3989        result.map_err(|err| Error::other(format!("failed to select decommission target pool for {bucket}/{object}: {err}")))
3990    }
3991
3992    fn resolve_decommission_tiered_object_result(result: Result<()>, bucket: &str, object: &str) -> Result<()> {
3993        result.map_err(|err| {
3994            crate::data_movement::data_movement_context_error(
3995                format!("failed to decommission tiered object for {bucket}/{object}: {err}"),
3996                err,
3997            )
3998        })
3999    }
4000
4001    #[instrument(skip(self, fi, opts))]
4002    pub(crate) async fn decommission_tiered_object(
4003        self: &Arc<Self>,
4004        bucket: &str,
4005        object: &str,
4006        fi: &rustfs_filemeta::FileInfo,
4007        opts: &ObjectOptions,
4008    ) -> Result<()> {
4009        check_put_object_args(bucket, object)?;
4010
4011        let mut opts = opts.clone();
4012        let is_free_version = fi.tier_free_version();
4013        if is_free_version {
4014            opts.incl_free_versions = true;
4015        }
4016        let bucket_incarnation_fence = if is_meta_bucketname(bucket) {
4017            None
4018        } else {
4019            let expected = opts
4020                .expected_bucket_incarnation_id
4021                .ok_or_else(|| Error::other("tiered data movement is missing its bucket incarnation snapshot"))?;
4022            let guard = self.acquire_bucket_incarnation_fence(bucket, expected).await?;
4023            if let Some(namespace_guard) = guard.namespace_lock_guard() {
4024                opts.add_bucket_lifecycle_lock_guard(namespace_guard);
4025            }
4026            Some(guard)
4027        };
4028
4029        // Capture exact source/tier identity now; the tier lease and namespace
4030        // writes are acquired only at the final metadata publication boundary.
4031        let publication_fence = self
4032            .acquire_tiered_remote_tuple_publication_fence(bucket, object, opts.src_pool_idx, fi)
4033            .await?;
4034
4035        let logical_object = object;
4036        let object = encode_dir_object(logical_object);
4037        ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts);
4038        if self.single_pool() {
4039            return Self::resolve_decommission_tiered_object_result(
4040                Err(Error::other("single pool deployments cannot decommission tiered objects")),
4041                bucket,
4042                &object,
4043            );
4044        }
4045
4046        let idx = if uses_data_movement_pool_selection(&opts) {
4047            Self::resolve_decommission_target_pool_idx_result(
4048                self.select_data_movement_pool_idx(bucket, &object, fi.size, &opts, true)
4049                    .await,
4050                bucket,
4051                &object,
4052            )?
4053        } else {
4054            Self::resolve_decommission_target_pool_idx_result(
4055                self.get_pool_idx_no_lock(bucket, &object, fi.size).await,
4056                bucket,
4057                &object,
4058            )?
4059        };
4060        #[cfg(all(test, feature = "test-util"))]
4061        if is_free_version {
4062            pause_decommission_free_version_before_source_lock(bucket, logical_object).await;
4063        }
4064        let publication_guard = publication_fence.into_commit_guard(idx, bucket, &object).await?;
4065        publication_guard.add_namespace_lock_fence(&mut opts);
4066        opts.no_lock = true;
4067        let mut fi = publication_guard
4068            .revalidated_tiered_source()
4069            .ok_or_else(|| Error::other("tiered data movement publication guard is missing its revalidated source"))?
4070            .clone();
4071        if opts.data_movement {
4072            crate::data_movement::prepare_tiered_data_movement_file_info(&mut fi)?;
4073        }
4074        if opts.data_movement && idx == opts.src_pool_idx {
4075            let resume_target_pool_idx = if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) {
4076                Some(
4077                    self.select_decommission_capacity_target_pool(owner, usize::try_from(fi.size).unwrap_or_default())
4078                        .await?,
4079                )
4080            } else {
4081                self.get_available_pool_idx_excluding(bucket, &object, fi.size, opts.src_pool_idx)
4082                    .await
4083            };
4084            let target_pool_idx = resolve_data_movement_resume_target_pool(idx, resume_target_pool_idx, opts.src_pool_idx);
4085            if is_free_version && target_pool_idx == opts.src_pool_idx {
4086                return Err(Error::DiskFull);
4087            }
4088            let equivalent = if is_free_version {
4089                self.has_equivalent_data_movement_tier_free_version(bucket, &object, &fi, &opts, target_pool_idx)
4090                    .await?
4091            } else {
4092                self.has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, target_pool_idx)
4093                    .await?
4094            };
4095            if equivalent {
4096                return Ok(());
4097            }
4098
4099            return Err(decommission_free_version_overwrite_error(bucket, &object, fi.version_id));
4100        }
4101
4102        let expected_data_bytes = usize::try_from(fi.size).ok();
4103        let result = self
4104            .run_decommission_capacity_admitted_mutation(
4105                idx,
4106                DecommissionCapacityOwner::from_options(&opts),
4107                expected_data_bytes,
4108                || async {
4109                    if is_free_version {
4110                        self.pools[idx]
4111                            .get_disks_by_key(&object)
4112                            .decommission_tier_free_version(bucket, &object, &fi, &opts)
4113                            .await
4114                    } else {
4115                        self.pools[idx]
4116                            .get_disks_by_key(&object)
4117                            .decommission_tiered_object(bucket, &object, &fi, &opts)
4118                            .await
4119                    }
4120                },
4121            )
4122            .await;
4123        if matches!(result, Err(Error::PreconditionFailed)) {
4124            if self
4125                .has_equivalent_data_movement_tiered_object(bucket, &object, &fi, &opts, idx)
4126                .await?
4127            {
4128                return Ok(());
4129            }
4130            return Err(StorageError::DataMovementOverwriteErr(
4131                bucket.to_owned(),
4132                object,
4133                opts.version_id.clone().unwrap_or_default(),
4134            ));
4135        }
4136        if bucket_incarnation_fence.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
4137            return Err(Error::other("tiered data movement bucket incarnation fence was lost during target write"));
4138        }
4139        Self::resolve_decommission_tiered_object_result(result, bucket, &object)
4140    }
4141
4142    /// Open a source reader for a server-side copy.
4143    ///
4144    /// Copy consumers hold the source reader while a destination write can
4145    /// apply backpressure. Keep that read contract explicit at the storage
4146    /// boundary so the lower-level legacy multipart pipeline can suppress its
4147    /// speculative next-part setup without changing the public `ObjectIO`
4148    /// trait or `ObjectOptions` layout.
4149    pub async fn get_object_reader_for_copy(
4150        &self,
4151        bucket: &str,
4152        object: &str,
4153        range: Option<HTTPRangeSpec>,
4154        h: HeaderMap,
4155        opts: &ObjectOptions,
4156    ) -> Result<(GetObjectReader, tokio_util::sync::CancellationToken)> {
4157        let cancellation = tokio_util::sync::CancellationToken::new();
4158        let reader = crate::set_disk::with_get_object_read_cancellation(
4159            cancellation.clone(),
4160            crate::set_disk::with_get_object_read_policy(
4161                crate::set_disk::GetObjectReadPolicy::CopySource,
4162                self.handle_get_object_reader(bucket, object, range, h, opts),
4163            ),
4164        )
4165        .await?;
4166        Ok((reader, cancellation))
4167    }
4168
4169    #[instrument(level = "debug", skip(self, h))]
4170    #[hotpath::measure(impl_type = "ECStore")]
4171    pub(super) async fn handle_get_object_reader(
4172        &self,
4173        bucket: &str,
4174        object: &str,
4175        range: Option<HTTPRangeSpec>,
4176        h: HeaderMap,
4177        opts: &ObjectOptions,
4178    ) -> Result<GetObjectReader> {
4179        check_get_obj_args(bucket, object)?;
4180
4181        let object = rustfs_utils::path::encode_dir_object_ref(object);
4182        let mut opts = opts.clone();
4183        let read_lock_guard = self
4184            .acquire_object_read_lock_if_needed("get_object", bucket, &object, &mut opts)
4185            .await?;
4186
4187        let reader = if self.single_pool() {
4188            self.pools[0]
4189                .get_object_reader(bucket, object.as_ref(), range, h, &opts)
4190                .await?
4191        } else {
4192            // Keep selection plus prepared-open state off the caller stack.
4193            // Debug builds otherwise exceed the common 2 MiB worker stack.
4194            Box::pin(async {
4195                let (metadata, idx) = self.prepare_latest_object_metadata_with_idx(bucket, &object, &opts).await?;
4196                if let Some(error) = latest_object_access_delete_marker_error(bucket, &object, metadata.object_info(), &opts) {
4197                    return Err(error);
4198                }
4199                self.pools[idx]
4200                    .get_object_reader_with_prepared_metadata(bucket, object.as_ref(), range, h, &opts, metadata)
4201                    .await
4202            })
4203            .await?
4204        };
4205
4206        Ok(Self::attach_read_lock_guard(reader, read_lock_guard))
4207    }
4208
4209    async fn prepare_put_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<(String, ObjectOptions)> {
4210        check_put_object_args(bucket, object)?;
4211
4212        let object = encode_dir_object(object);
4213        let mut opts = opts.clone();
4214        if !is_meta_bucketname(bucket) && opts.expected_bucket_incarnation_id.is_none() {
4215            opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(bucket).await?);
4216        }
4217        if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
4218            let expected_incarnation_id = opts
4219                .expected_bucket_incarnation_id
4220                .ok_or_else(|| Error::other("destructive PUT is missing its bucket incarnation"))?;
4221            if opts.object_lock_config_snapshot.is_none() {
4222                opts.object_lock_config_snapshot = Some(self.object_lock_config_snapshot(bucket).await?);
4223            }
4224            let snapshot = match opts.object_lock_config_snapshot.as_ref() {
4225                Some(snapshot) if snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) => {
4226                    Arc::clone(snapshot)
4227                }
4228                _ => {
4229                    return Err(Error::other(
4230                        "Object Lock snapshot does not hold valid target bucket generation and configuration fences",
4231                    ));
4232                }
4233            };
4234            snapshot.add_lock_fences(&mut opts);
4235        }
4236        Ok((object, opts))
4237    }
4238
4239    async fn select_put_object_pool_idx(&self, bucket: &str, object: &str, size: i64, opts: &ObjectOptions) -> Result<usize> {
4240        if self.single_pool() {
4241            return Ok(0);
4242        }
4243
4244        let idx = if uses_data_movement_pool_selection(opts) {
4245            self.select_data_movement_pool_idx(bucket, object, size, opts, false).await?
4246        } else if opts.no_lock {
4247            self.get_pool_idx_no_lock(bucket, object, size).await?
4248        } else {
4249            self.get_pool_idx(bucket, object, size).await?
4250        };
4251
4252        if opts.data_movement && idx == opts.src_pool_idx {
4253            return Err(StorageError::DataMovementOverwriteErr(
4254                bucket.to_owned(),
4255                object.to_owned(),
4256                opts.version_id.clone().unwrap_or_default(),
4257            ));
4258        }
4259        Ok(idx)
4260    }
4261
4262    #[cfg(test)]
4263    pub(crate) async fn put_object_for_data_movement(
4264        self: &Arc<Self>,
4265        bucket: &str,
4266        object: &str,
4267        data: &mut PutObjReader,
4268        opts: &ObjectOptions,
4269        mutation_fence: Option<&ObjectLockDiagGuard>,
4270    ) -> Result<(usize, Result<ObjectInfo>)> {
4271        self.put_object_for_data_movement_inner(bucket, object, data, opts, mutation_fence, None)
4272            .await
4273    }
4274
4275    pub(crate) async fn put_object_for_data_movement_with_publication_fence(
4276        self: &Arc<Self>,
4277        bucket: &str,
4278        object: &str,
4279        data: &mut PutObjReader,
4280        opts: &ObjectOptions,
4281        publication_fence: RemoteTuplePublicationFence,
4282    ) -> Result<(usize, Result<ObjectInfo>)> {
4283        self.put_object_for_data_movement_inner(bucket, object, data, opts, None, Some(publication_fence))
4284            .await
4285    }
4286
4287    async fn put_object_for_data_movement_inner(
4288        self: &Arc<Self>,
4289        bucket: &str,
4290        object: &str,
4291        data: &mut PutObjReader,
4292        opts: &ObjectOptions,
4293        mutation_fence: Option<&ObjectLockDiagGuard>,
4294        publication_fence: Option<RemoteTuplePublicationFence>,
4295    ) -> Result<(usize, Result<ObjectInfo>)> {
4296        if !opts.data_movement {
4297            return Err(Error::other("data movement PUT requires data_movement options"));
4298        }
4299        let (object, mut opts) = self.prepare_put_object(bucket, object, opts).await?;
4300        ensure_decommission_capacity_mutation_id(bucket, &object, &mut opts);
4301        let idx = self
4302            .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
4303            .await?;
4304        let fixed_read_anchor = publication_fence
4305            .as_ref()
4306            .and_then(RemoteTuplePublicationFence::fixed_read_anchor_guard);
4307        self.apply_decommission_target_mutation_fence(idx, object.as_str(), &mut opts, mutation_fence.or(fixed_read_anchor))
4308            .await;
4309        let expected_data_bytes = usize::try_from(data.size()).ok();
4310        let result = self
4311            .run_decommission_capacity_admitted_mutation_with_capacity_lease(
4312                idx,
4313                DecommissionCapacityOwner::from_options(&opts),
4314                expected_data_bytes,
4315                |capacity_lease| async move {
4316                    if let Some(capacity_lease) = capacity_lease {
4317                        opts.add_namespace_lock_lost_signal(capacity_lease);
4318                    }
4319                    let result = match publication_fence {
4320                        Some(publication_fence) => {
4321                            self.pools[idx]
4322                                .put_object_with_old_current_size_for_data_movement(
4323                                    bucket,
4324                                    &object,
4325                                    data,
4326                                    &opts,
4327                                    publication_fence,
4328                                )
4329                                .await
4330                        }
4331                        None => {
4332                            self.pools[idx]
4333                                .put_object_with_old_current_size(bucket, &object, data, &opts)
4334                                .await
4335                        }
4336                    };
4337                    result.map(|(object_info, _)| object_info)
4338                },
4339            )
4340            .await;
4341        let result = enqueue_transition_after_write(result, LcEventSrc::S3PutObject).await;
4342        if result.is_ok() {
4343            list_objects::observe_list_objects_mutation(self, bucket).await;
4344        }
4345        Ok((idx, result))
4346    }
4347
4348    #[instrument(level = "debug", skip(self, data))]
4349    #[hotpath::measure(impl_type = "ECStore")]
4350    pub(super) async fn handle_put_object(
4351        &self,
4352        bucket: &str,
4353        object: &str,
4354        data: &mut PutObjReader,
4355        opts: &ObjectOptions,
4356    ) -> Result<(ObjectInfo, Option<OldCurrentSize>)> {
4357        let (object, opts) = self.prepare_put_object(bucket, object, opts).await?;
4358        let idx = self
4359            .select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
4360            .await?;
4361        let mut opts = opts;
4362        if !self.single_pool() {
4363            opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
4364        }
4365        let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
4366        let result = self.pools[idx]
4367            .put_object_with_old_current_size(bucket, object.as_str(), data, &opts)
4368            .await;
4369        enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4370        result
4371    }
4372
4373    #[instrument(level = "trace", skip(self))]
4374    pub(super) async fn handle_get_object_info(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
4375        self.get_object_info_snapshot(bucket, object, opts, false).await
4376    }
4377
4378    /// Return metadata for DELETE preflight, including an explicitly addressed
4379    /// delete marker. GET/HEAD may also use this metadata-only lookup to enrich
4380    /// an already failed read with marker headers, never to serve marker data.
4381    /// Normal reads must keep using `get_object_info`; authorization and Object
4382    /// Lock enforcement still belong to the caller and locked delete.
4383    #[instrument(level = "trace", skip_all)]
4384    pub async fn get_object_info_for_delete(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
4385        self.get_object_info_snapshot(bucket, object, opts, true).await
4386    }
4387
4388    async fn get_object_info_snapshot(
4389        &self,
4390        bucket: &str,
4391        object: &str,
4392        opts: &ObjectOptions,
4393        allow_delete_marker: bool,
4394    ) -> Result<ObjectInfo> {
4395        check_object_args(bucket, object)?;
4396
4397        let object = encode_dir_object(object);
4398        let mut opts = opts.clone();
4399        let _object_lock_guard = self
4400            .acquire_object_read_lock_if_needed("get_object_info", bucket, &object, &mut opts)
4401            .await?;
4402
4403        let info = if self.single_pool() {
4404            self.pools[0].get_object_info(bucket, object.as_str(), &opts).await?
4405        } else if allow_delete_marker {
4406            self.get_latest_object_info_with_idx(bucket, object.as_str(), &opts).await?.0
4407        } else {
4408            self.get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &opts)
4409                .await?
4410                .0
4411        };
4412        opts.precondition_check(&info)?;
4413        Ok(info)
4414    }
4415
4416    #[instrument(skip(self))]
4417    #[allow(clippy::too_many_arguments)]
4418    pub(super) async fn handle_copy_object(
4419        &self,
4420        src_bucket: &str,
4421        src_object: &str,
4422        dst_bucket: &str,
4423        dst_object: &str,
4424        src_info: &mut ObjectInfo,
4425        src_opts: &ObjectOptions,
4426        dst_opts: &ObjectOptions,
4427    ) -> Result<ObjectInfo> {
4428        check_copy_obj_args(src_bucket, src_object)?;
4429        check_copy_obj_args(dst_bucket, dst_object)?;
4430
4431        let src_object = encode_dir_object(src_object);
4432        let dst_object = encode_dir_object(dst_object);
4433
4434        let cp_src_dst_same = path_join_buf(&[src_bucket, &src_object]) == path_join_buf(&[dst_bucket, &dst_object]);
4435
4436        let mut dst_opts = dst_opts.clone();
4437        dst_opts.inherit_shard_integrity(src_info);
4438        if !is_meta_bucketname(dst_bucket) && dst_opts.expected_bucket_incarnation_id.is_none() {
4439            dst_opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(dst_bucket).await?);
4440        }
4441        let _bucket_lifecycle_guard = if is_meta_bucketname(dst_bucket) || dst_opts.bucket_lifecycle_lock_fence.is_some() {
4442            None
4443        } else {
4444            Some(self.acquire_bucket_lifecycle_read_lock(dst_bucket).await?)
4445        };
4446        let current_bucket_incarnation_id = if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
4447            dst_opts.add_bucket_lifecycle_lock_guard(guard);
4448            let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, dst_bucket).await?;
4449            if dst_opts
4450                .expected_bucket_incarnation_id
4451                .is_some_and(|expected| expected != current_incarnation_id)
4452            {
4453                return Err(StorageError::BucketNotFound(dst_bucket.to_string()));
4454            }
4455            Some(current_incarnation_id)
4456        } else {
4457            dst_opts.expected_bucket_incarnation_id
4458        };
4459        if dst_opts
4460            .bucket_lifecycle_lock_fence
4461            .as_ref()
4462            .is_some_and(NamespaceLockFence::is_lock_lost)
4463        {
4464            return Err(StorageError::NamespaceLockQuorumUnavailable {
4465                mode: "copy_object_bucket_generation",
4466                bucket: dst_bucket.to_string(),
4467                object: dst_object.clone(),
4468                required: 1,
4469                achieved: 0,
4470            });
4471        }
4472        if dst_opts.overwrites_existing_version() && !is_meta_bucketname(dst_bucket) {
4473            let incarnation_id =
4474                current_bucket_incarnation_id.ok_or_else(|| Error::other("copy is missing its bucket incarnation snapshot"))?;
4475            let lifecycle_fence = dst_opts
4476                .bucket_lifecycle_lock_fence
4477                .as_ref()
4478                .ok_or_else(|| Error::other("copy is missing its bucket lifecycle fence"))?;
4479            let snapshot = match dst_opts.object_lock_config_snapshot.as_ref() {
4480                Some(snapshot) => Arc::clone(snapshot),
4481                None => {
4482                    self.object_lock_config_snapshot_under_lifecycle_fence(dst_bucket, lifecycle_fence)
4483                        .await?
4484                }
4485            };
4486            if !snapshot.is_valid_for_destructive_put(self.id, dst_bucket, incarnation_id) {
4487                return Err(Error::other("copy Object Lock snapshot does not match the target bucket generation"));
4488            }
4489            snapshot.add_lock_fences(&mut dst_opts);
4490            dst_opts.object_lock_config_snapshot = Some(snapshot);
4491        }
4492        let _dst_lock_guard = if cp_src_dst_same && dst_opts.expected_current_version_id.is_none() {
4493            self.acquire_object_write_lock_if_needed("copy_object", dst_bucket, &dst_object, &mut dst_opts)
4494                .await?
4495        } else {
4496            None
4497        };
4498
4499        if cp_src_dst_same {
4500            let (_, pool_idx) = self
4501                .get_latest_accessible_object_info_with_idx(src_bucket, &src_object, &version_aware_lookup_opts(src_opts, true))
4502                .await?;
4503            let source_pool_writable = !self.is_suspended(pool_idx).await && !self.is_pool_rebalancing(pool_idx).await;
4504
4505            if source_pool_writable
4506                && let (Some(src_vid), Some(dst_vid)) = (&src_opts.version_id, &dst_opts.version_id)
4507                && src_vid == dst_vid
4508            {
4509                let capacity_object = dst_object.clone();
4510                return self
4511                    .run_external_decommission_capacity_object_mutation(
4512                        pool_idx,
4513                        dst_bucket,
4514                        &capacity_object,
4515                        &capacity_object,
4516                        dst_opts.clone(),
4517                        |opts| async move {
4518                            self.pools[pool_idx]
4519                                .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts)
4520                                .await
4521                        },
4522                    )
4523                    .await;
4524            }
4525
4526            if source_pool_writable && !dst_opts.versioned && src_opts.version_id.is_none() {
4527                if src_info.metadata_only {
4528                    // Zero-copy update: only xl.meta is rewritten, the data blocks stay as they
4529                    // are. The caller must therefore guarantee that the destination metadata
4530                    // still describes the stored bytes. In particular a copy that re-derives
4531                    // encryption material may NOT set metadata_only — that would leave a fresh
4532                    // DEK beside ciphertext sealed under the old one, permanently destroying the
4533                    // object. The S3 handler enforces this before calling in (see the
4534                    // metadata_only decision in rustfs/src/app/object_usecase.rs); the sibling
4535                    // versioned branch below resolves the same risk by rewriting through
4536                    // put_object (issue #4238).
4537                    let capacity_object = dst_object.clone();
4538                    return self
4539                        .run_external_decommission_capacity_object_mutation(
4540                            pool_idx,
4541                            dst_bucket,
4542                            &capacity_object,
4543                            &capacity_object,
4544                            dst_opts.clone(),
4545                            |opts| async move {
4546                                self.pools[pool_idx]
4547                                    .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts)
4548                                    .await
4549                            },
4550                        )
4551                        .await;
4552                }
4553                // Transitioned object self-copy: restore from tier into the same pool.
4554                let mut put_opts = ObjectOptions {
4555                    user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
4556                    versioned: dst_opts.versioned,
4557                    shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
4558                    version_id: dst_opts.version_id.clone(),
4559                    no_lock: dst_opts.no_lock,
4560                    mod_time: dst_opts.mod_time,
4561                    http_preconditions: dst_opts.http_preconditions.clone(),
4562                    expected_current_version_id: dst_opts.expected_current_version_id.clone(),
4563                    expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
4564                    namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
4565                    bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
4566                    object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
4567                    ..Default::default()
4568                };
4569                if !self.single_pool() {
4570                    put_opts.decommission_capacity_admission =
4571                        crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
4572                }
4573                return if let Some(reader) = src_info.put_object_reader.as_mut() {
4574                    let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
4575                    let result = self.pools[pool_idx]
4576                        .put_object(dst_bucket, &dst_object, reader, &put_opts)
4577                        .await;
4578                    enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4579                    result
4580                } else {
4581                    Err(StorageError::InvalidArgument(
4582                        src_bucket.to_owned(),
4583                        src_object.to_owned(),
4584                        "put_object_reader is none".to_owned(),
4585                    ))
4586                };
4587            }
4588
4589            if source_pool_writable && dst_opts.versioned && src_opts.version_id != dst_opts.version_id {
4590                // Restoring a specific historical version onto the current key creates a NEW
4591                // version. When the caller supplies a reader (S3 CopyObject), write the fetched
4592                // bytes through put_object so any re-encryption/compression applied to the reader
4593                // stays consistent with the new version's metadata. Sharing the source data_dir via
4594                // a metadata-only version copy would corrupt SSE/compressed objects (issue #4238).
4595                if let Some(reader) = src_info.put_object_reader.as_mut() {
4596                    let mut put_opts = ObjectOptions {
4597                        user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
4598                        versioned: dst_opts.versioned,
4599                        shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
4600                        version_id: dst_opts.version_id.clone(),
4601                        no_lock: dst_opts.no_lock,
4602                        mod_time: dst_opts.mod_time,
4603                        http_preconditions: dst_opts.http_preconditions.clone(),
4604                        expected_current_version_id: dst_opts.expected_current_version_id.clone(),
4605                        expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
4606                        namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
4607                        bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
4608                        object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
4609                        ..Default::default()
4610                    };
4611                    if !self.single_pool() {
4612                        put_opts.decommission_capacity_admission =
4613                            crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
4614                    }
4615                    let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
4616                    let result = self.pools[pool_idx]
4617                        .put_object(dst_bucket, &dst_object, reader, &put_opts)
4618                        .await;
4619                    enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4620                    return result;
4621                }
4622                src_info.version_only = true;
4623                let capacity_object = dst_object.clone();
4624                return self
4625                    .run_external_decommission_capacity_object_mutation(
4626                        pool_idx,
4627                        dst_bucket,
4628                        &capacity_object,
4629                        &capacity_object,
4630                        dst_opts.clone(),
4631                        |opts| async move {
4632                            self.pools[pool_idx]
4633                                .copy_object(src_bucket, &src_object, dst_bucket, &dst_object, src_info, src_opts, &opts)
4634                                .await
4635                        },
4636                    )
4637                    .await;
4638            }
4639        }
4640
4641        let pool_idx = if dst_opts.no_lock {
4642            self.get_pool_idx_no_lock(dst_bucket, &dst_object, src_info.size).await?
4643        } else {
4644            self.get_pool_idx(dst_bucket, &dst_object, src_info.size).await?
4645        };
4646        let dst_object_name = dst_object.as_str();
4647
4648        let mut put_opts = ObjectOptions {
4649            user_defined: materialized_copy_user_defined(src_info.user_defined.as_ref()),
4650            versioned: dst_opts.versioned,
4651            shard_integrity_write_mode: dst_opts.shard_integrity_write_mode,
4652            version_id: dst_opts.version_id.clone(),
4653            no_lock: dst_opts.no_lock,
4654            mod_time: dst_opts.mod_time,
4655            http_preconditions: dst_opts.http_preconditions.clone(),
4656            expected_current_version_id: dst_opts.expected_current_version_id.clone(),
4657            expected_bucket_incarnation_id: dst_opts.expected_bucket_incarnation_id,
4658            namespace_lock_fence: dst_opts.namespace_lock_fence.clone(),
4659            bucket_lifecycle_lock_fence: dst_opts.bucket_lifecycle_lock_fence.clone(),
4660            object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
4661            ..Default::default()
4662        };
4663        if !self.single_pool() {
4664            put_opts.decommission_capacity_admission =
4665                crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
4666        }
4667
4668        if let Some(put_object_reader) = src_info.put_object_reader.as_mut() {
4669            let receipt_sink = install_tier_free_version_receipt_sink(&mut put_opts);
4670            let result = self.pools[pool_idx]
4671                .put_object(dst_bucket, dst_object_name, put_object_reader, &put_opts)
4672                .await;
4673            enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4674            return result;
4675        }
4676
4677        Err(StorageError::InvalidArgument(
4678            src_bucket.to_owned(),
4679            src_object.to_owned(),
4680            "put_object_reader is none".to_owned(),
4681        ))
4682    }
4683
4684    /// Best-effort purge of an orphan directory prefix — an on-disk tree of empty
4685    /// directories or committed delete residue with no `xl.meta` anywhere. Orphan fragments can sit
4686    /// on any erasure set of any pool (they are left behind by whichever sets stored
4687    /// the now-deleted children), so every set is swept. Returns true when at least
4688    /// one set removed an orphan tree. Hard per-set failures are logged and skipped:
4689    /// the caller falls back to surfacing the original NotFound.
4690    pub(super) async fn purge_orphan_dir_object(&self, bucket: &str, object: &str) -> bool {
4691        let prefix = decode_dir_object(object);
4692        let mut purged = false;
4693        for pool in self.pools.iter() {
4694            for set in pool.disk_set.iter() {
4695                match set.purge_orphan_dir_object(bucket, &prefix).await {
4696                    Ok(set_purged) => purged |= set_purged,
4697                    Err(err) => {
4698                        warn!(
4699                            bucket,
4700                            prefix,
4701                            pool_index = pool.pool_idx,
4702                            error = ?err,
4703                            "failed to purge orphan directory prefix"
4704                        );
4705                    }
4706                }
4707            }
4708        }
4709        purged
4710    }
4711
4712    pub async fn delete_object_with_tier_delete_journal(
4713        self: &Arc<Self>,
4714        bucket: &str,
4715        object: &str,
4716        opts: ObjectOptions,
4717    ) -> Result<ObjectInfo> {
4718        let result = self
4719            .handle_delete_object_with_journal(bucket, object, opts, Some(Arc::clone(self)))
4720            .await;
4721        if result.is_ok() {
4722            list_objects::observe_list_objects_mutation(self, bucket).await;
4723        }
4724        result
4725    }
4726
4727    pub async fn delete_objects_with_tier_delete_journal(
4728        self: &Arc<Self>,
4729        bucket: &str,
4730        objects: Vec<ObjectToDelete>,
4731        opts: ObjectOptions,
4732    ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
4733        let result = self
4734            .handle_delete_objects_with_journal(bucket, objects, opts, Some(Arc::clone(self)))
4735            .await;
4736        let success_count = result.1.iter().filter(|err| err.is_none()).count();
4737        if success_count > 0 {
4738            list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
4739        }
4740        result
4741    }
4742
4743    pub async fn delete_objects_with_tier_delete_journal_and_accounting(
4744        self: &Arc<Self>,
4745        bucket: &str,
4746        objects: Vec<ObjectToDelete>,
4747        opts: ObjectOptions,
4748    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
4749        let result = self
4750            .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, Some(Arc::clone(self)))
4751            .await;
4752        let success_count = result.1.iter().filter(|err| err.is_none()).count();
4753        if success_count > 0 {
4754            list_objects::observe_list_objects_mutations(self, bucket, success_count).await;
4755        }
4756        result
4757    }
4758
4759    #[instrument(skip(self))]
4760    pub(super) async fn handle_delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
4761        self.handle_delete_object_with_journal(bucket, object, opts, None).await
4762    }
4763
4764    pub(super) async fn handle_delete_object_with_journal(
4765        &self,
4766        bucket: &str,
4767        object: &str,
4768        opts: ObjectOptions,
4769        tier_journal_api: Option<Arc<ECStore>>,
4770    ) -> Result<ObjectInfo> {
4771        Box::pin(async move {
4772            let mut opts = opts;
4773            let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
4774            let result = self
4775                .handle_delete_object_with_journal_inner(bucket, object, opts, tier_journal_api)
4776                .await;
4777            enqueue_recorded_tier_free_versions(self, receipt_sink).await;
4778            result
4779        })
4780        .await
4781    }
4782
4783    async fn handle_delete_object_with_journal_inner(
4784        &self,
4785        bucket: &str,
4786        object: &str,
4787        opts: ObjectOptions,
4788        tier_journal_api: Option<Arc<ECStore>>,
4789    ) -> Result<ObjectInfo> {
4790        check_del_obj_args(bucket, object)?;
4791
4792        if opts.lifecycle_delete_all.is_some() && self.ctx.lock_manager().is_disabled() {
4793            return Err(Error::other("lifecycle delete-all requires namespace locking"));
4794        }
4795
4796        let _bucket_lifecycle_guard =
4797            if is_meta_bucketname(bucket) || (opts.delete_prefix && opts.bucket_lifecycle_lock_fence.is_some()) {
4798                None
4799            } else if opts.delete_prefix {
4800                Some(self.acquire_bucket_lifecycle_write_lock(bucket).await?)
4801            } else {
4802                Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
4803            };
4804        let object = if opts.delete_prefix && !opts.delete_prefix_object {
4805            object.to_owned()
4806        } else {
4807            encode_dir_object(object)
4808        };
4809        let object = object.as_str();
4810        let mut opts = opts;
4811        ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
4812        let delete_all_configs = if opts.lifecycle_delete_all.is_some() {
4813            Some(get_expiry_configs(self, bucket).await?)
4814        } else {
4815            None
4816        };
4817        opts.tier_delete_journal_api = tier_journal_api.clone();
4818        if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
4819            opts.add_bucket_lifecycle_lock_guard(guard);
4820        }
4821
4822        if !is_meta_bucketname(bucket) {
4823            get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await?;
4824        }
4825        let _object_lock_metadata_guard = if !is_meta_bucketname(bucket) {
4826            Some(acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await?)
4827        } else {
4828            None
4829        };
4830        if let Some(guard) = _object_lock_metadata_guard.as_ref() {
4831            opts.add_namespace_lock_guard(guard);
4832        }
4833        let current_bucket_incarnation_id = if _object_lock_metadata_guard.is_some() {
4834            let (state, incarnation_id, config_revision) =
4835                get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await?;
4836            opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
4837                self.id,
4838                bucket,
4839                incarnation_id,
4840                config_revision,
4841                state,
4842            )));
4843            Some(incarnation_id)
4844        } else {
4845            None
4846        };
4847        if let (Some(expected), Some(current)) = (opts.expected_bucket_incarnation_id, current_bucket_incarnation_id)
4848            && expected != current
4849        {
4850            return Err(StorageError::BucketNotFound(bucket.to_string()));
4851        }
4852        if opts.expected_bucket_incarnation_id.is_none() {
4853            opts.expected_bucket_incarnation_id = current_bucket_incarnation_id;
4854        }
4855        #[cfg(any(test, feature = "test-util"))]
4856        pause_delete_after_object_lock_snapshot(bucket).await;
4857
4858        if opts.delete_prefix && !opts.delete_prefix_object {
4859            delete_recursive_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?;
4860            return Ok(ObjectInfo::default());
4861        }
4862
4863        let _object_lock_guard = if opts.expected_current_version_id.is_none() {
4864            self.acquire_object_write_lock_if_needed("delete_object", bucket, object, &mut opts)
4865                .await?
4866        } else {
4867            None
4868        };
4869        #[cfg(any(test, feature = "test-util"))]
4870        if _object_lock_guard.is_some() {
4871            notify_delete_namespace_acquired(bucket);
4872        }
4873        if let Some(trigger) = opts.lifecycle_delete_all.as_ref() {
4874            let configs = delete_all_configs.as_ref().ok_or(StorageError::PreconditionFailed)?;
4875            let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id.ok_or(StorageError::PreconditionFailed)?;
4876            if configs.table_bucket_enabled || configs.bucket_incarnation_id != expected_bucket_incarnation_id {
4877                return Err(StorageError::PreconditionFailed);
4878            }
4879            let lifecycle = configs.lifecycle.as_ref().ok_or(StorageError::PreconditionFailed)?;
4880            let (mut current, _) = self
4881                .get_latest_object_info_with_idx(
4882                    bucket,
4883                    object,
4884                    &ObjectOptions {
4885                        no_lock: true,
4886                        metadata_cache_safe: false,
4887                        ..Default::default()
4888                    },
4889                )
4890                .await?;
4891            let current_version_id = current.version_id.filter(|version_id| !version_id.is_nil());
4892            if current_version_id != trigger.version_id || current.delete_marker != trigger.delete_marker {
4893                return Err(StorageError::PreconditionFailed);
4894            }
4895            current.name = decode_dir_object(&current.name);
4896            let current_event = eval_action_from_lifecycle(lifecycle, configs.object_lock.as_deref(), &current).await;
4897            if current_event.action != trigger.action || current_event.rule_id != trigger.rule_id {
4898                return Err(StorageError::PreconditionFailed);
4899            }
4900        }
4901        if opts.delete_prefix {
4902            delete_prefix_with_tier_delete_journal(self, bucket, object, &opts, tier_journal_api.as_ref()).await?;
4903            return Ok(ObjectInfo::default());
4904        }
4905
4906        let creates_latest_marker = latest_versioned_delete_creates_distinct_marker(&opts)
4907            || (opts.version_suspended && should_create_delete_marker_for_missing_object(&opts));
4908        let mut gopts = delete_pool_lookup_opts(&opts, true);
4909        if creates_latest_marker {
4910            // An unwritable source still owns its current version. Hiding it
4911            // during lookup would turn a rejected write into a new-pool marker.
4912            gopts.skip_rebalancing = false;
4913        }
4914
4915        if opts.data_movement {
4916            let existing_pool_info = self.get_pool_info_existing_with_opts(bucket, object, &gopts).await;
4917            let existing_pool_idx = existing_pool_info
4918                .as_ref()
4919                .map(|(pinfo, _)| pinfo.index)
4920                .map_err(Clone::clone);
4921            let selected_target_pool_idx =
4922                match select_data_movement_target_pool(existing_pool_idx, opts.src_pool_idx, opts.delete_marker)? {
4923                    Some(pool_idx) => pool_idx,
4924                    None => {
4925                        if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) {
4926                            self.select_decommission_capacity_target_pool(owner, 0).await?
4927                        } else {
4928                            self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx)
4929                                .await
4930                                .ok_or(Error::DiskFull)?
4931                        }
4932                    }
4933                };
4934            let resume_target_pool_idx = if selected_target_pool_idx == opts.src_pool_idx {
4935                if let Some(owner) = DecommissionCapacityOwner::from_options(&opts) {
4936                    Some(self.select_decommission_capacity_target_pool(owner, 0).await?)
4937                } else {
4938                    self.get_available_pool_idx_excluding(bucket, object, 0, opts.src_pool_idx)
4939                        .await
4940                }
4941            } else {
4942                None
4943            };
4944            let target_pool_idx =
4945                resolve_data_movement_resume_target_pool(selected_target_pool_idx, resume_target_pool_idx, opts.src_pool_idx);
4946            let mut delete_marker_target_opts = None;
4947
4948            if opts.delete_marker && should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) {
4949                let source = self
4950                    .find_data_movement_target_info(bucket, object, opts.src_pool_idx, &opts)
4951                    .await?;
4952                let Some(source) = source else {
4953                    return Err(StorageError::DataMovementOverwriteErr(
4954                        bucket.to_owned(),
4955                        object.to_owned(),
4956                        opts.version_id.unwrap_or_default(),
4957                    ));
4958                };
4959                if !is_expected_data_movement_delete_marker_source(&source, opts.mod_time) {
4960                    return Err(StorageError::DataMovementOverwriteErr(
4961                        bucket.to_owned(),
4962                        object.to_owned(),
4963                        opts.version_id.unwrap_or_default(),
4964                    ));
4965                }
4966                let Some(target_opts) = current_data_movement_delete_marker_opts(&source, &opts) else {
4967                    return Err(StorageError::DataMovementOverwriteErr(
4968                        bucket.to_owned(),
4969                        object.to_owned(),
4970                        opts.version_id.unwrap_or_default(),
4971                    ));
4972                };
4973                let target = self
4974                    .find_data_movement_target_info(bucket, object, target_pool_idx, &target_opts)
4975                    .await?;
4976                if let Some(target) = target {
4977                    if is_equivalent_data_movement_delete_marker(&source, &target) {
4978                        if let Some(owner) = DecommissionCapacityOwner::from_options(&target_opts) {
4979                            self.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, 0)
4980                                .await?;
4981                        }
4982                        let mut target = target;
4983                        target.name = decode_dir_object(object);
4984                        return Ok(target);
4985                    }
4986                    return Err(StorageError::DataMovementOverwriteErr(
4987                        bucket.to_owned(),
4988                        object.to_owned(),
4989                        opts.version_id.unwrap_or_default(),
4990                    ));
4991                }
4992                delete_marker_target_opts = Some(target_opts);
4993            }
4994
4995            if !should_check_data_movement_resume_target(opts.src_pool_idx, target_pool_idx) {
4996                if let Ok((source_pool_info, _)) = existing_pool_info
4997                    && opts.delete_marker
4998                    && is_data_movement_delete_marker(&source_pool_info.object_info)
4999                    && self
5000                        .has_equivalent_data_movement_delete_marker(
5001                            bucket,
5002                            object,
5003                            &source_pool_info.object_info,
5004                            &opts,
5005                            target_pool_idx,
5006                        )
5007                        .await?
5008                {
5009                    let mut obj = source_pool_info.object_info;
5010                    obj.name = decode_dir_object(object);
5011                    return Ok(obj);
5012                }
5013
5014                return Err(StorageError::DataMovementOverwriteErr(
5015                    bucket.to_owned(),
5016                    object.to_owned(),
5017                    opts.version_id.unwrap_or_default(),
5018                ));
5019            }
5020
5021            let target_opts = delete_marker_target_opts.unwrap_or(opts);
5022            let mut obj = self
5023                .run_decommission_capacity_admitted_mutation_with_capacity_lease(
5024                    target_pool_idx,
5025                    DecommissionCapacityOwner::from_options(&target_opts),
5026                    None,
5027                    |capacity_lease| async move {
5028                        let mut target_opts = target_opts;
5029                        if let Some(capacity_lease) = capacity_lease {
5030                            target_opts.add_namespace_lock_lost_signal(capacity_lease);
5031                        }
5032                        self.pools[target_pool_idx].delete_object(bucket, object, target_opts).await
5033                    },
5034                )
5035                .await?;
5036            obj.name = decode_dir_object(obj.name.as_str());
5037            return Ok(obj);
5038        }
5039
5040        // Determine which pool contains it
5041        let existing_pool_info = if creates_latest_marker {
5042            self.get_pool_info_for_delete_marker(bucket, object, &gopts).await
5043        } else {
5044            self.get_pool_info_existing_with_opts(bucket, object, &gopts).await
5045        };
5046        let (mut pinfo, errs) = match existing_pool_info {
5047            Ok(res) => res,
5048            Err(err) if is_err_read_quorum(&err) => return Err(StorageError::ErasureWriteQuorum),
5049            Err(err) if is_err_object_not_found(&err) && should_create_delete_marker_for_missing_object(&opts) => {
5050                let target_pool_idx = self.get_pool_idx_no_lock(bucket, object, 0).await?;
5051                let pool = self.pools[target_pool_idx].clone();
5052                let mut obj = self
5053                    .run_external_decommission_capacity_object_mutation(
5054                        target_pool_idx,
5055                        bucket,
5056                        object,
5057                        object,
5058                        opts,
5059                        |opts| async move { pool.delete_object(bucket, object, opts).await },
5060                    )
5061                    .await?;
5062                #[cfg(test)]
5063                pause_versioned_delete_marker_after_commit(bucket, object).await;
5064                obj.name = decode_dir_object(object);
5065                return Ok(obj);
5066            }
5067            Err(err) => {
5068                // A folder key (`prefix/`) with no object metadata may still exist on
5069                // disk as an orphan empty-directory tree (issue #4189): listings show
5070                // it as a common prefix, but no regular delete path can remove it.
5071                // Purge the orphan tree so folder deletes actually take effect.
5072                if should_purge_orphan_dir_on_missing(&err, object) && self.purge_orphan_dir_object(bucket, object).await {
5073                    return Ok(ObjectInfo {
5074                        bucket: bucket.to_owned(),
5075                        name: decode_dir_object(object),
5076                        ..Default::default()
5077                    });
5078                }
5079                return Err(err);
5080            }
5081        };
5082
5083        if creates_latest_marker && self.is_suspended(pinfo.index).await {
5084            let has_active_reservation = self
5085                .pool_meta
5086                .read()
5087                .await
5088                .has_active_decommission_capacity_reservation(pinfo.index);
5089            if has_active_reservation {
5090                pinfo.index = self.get_pool_idx_no_lock(bucket, object, 0).await?;
5091            }
5092        }
5093
5094        if pinfo.object_info.delete_marker && opts.version_id.is_none() && !creates_latest_marker {
5095            pinfo.object_info.name = decode_dir_object(object);
5096            return Ok(pinfo.object_info);
5097        }
5098
5099        if opts.data_movement && opts.src_pool_idx == pinfo.index {
5100            return Err(StorageError::DataMovementOverwriteErr(
5101                bucket.to_owned(),
5102                object.to_owned(),
5103                opts.version_id.unwrap_or_default(),
5104            ));
5105        }
5106
5107        if should_delete_from_all_pools(&opts, errs.len()) {
5108            let mut obj = self
5109                .delete_object_from_all_pools(bucket, object, &opts, &pinfo.object_info, errs)
5110                .await?;
5111            obj.name = decode_dir_object(object);
5112            return Ok(obj);
5113        }
5114
5115        for pool in self.pools.iter() {
5116            if creates_latest_marker && pool.pool_idx != pinfo.index {
5117                continue;
5118            }
5119            if self.is_suspended(pool.pool_idx).await || self.is_pool_rebalancing(pool.pool_idx).await {
5120                if creates_latest_marker {
5121                    return Err(StorageError::SlowDown);
5122                }
5123                continue;
5124            }
5125
5126            let pool_idx = pool.pool_idx;
5127            let pool = pool.clone();
5128            match self
5129                .run_external_decommission_capacity_object_delete(
5130                    pool_idx,
5131                    bucket,
5132                    object,
5133                    object,
5134                    opts.clone(),
5135                    |opts| async move { pool.delete_object(bucket, object, opts).await },
5136                )
5137                .await
5138            {
5139                Ok(res) => {
5140                    #[cfg(test)]
5141                    pause_versioned_delete_marker_after_commit(bucket, object).await;
5142                    let mut obj = res;
5143                    obj.name = decode_dir_object(object);
5144                    return Ok(obj);
5145                }
5146                Err(err) => {
5147                    if creates_latest_marker || (!is_err_object_not_found(&err) && !is_err_version_not_found(&err)) {
5148                        return Err(err);
5149                    }
5150                }
5151            }
5152        }
5153
5154        if let Some(ver) = opts.version_id {
5155            return Err(StorageError::VersionNotFound(bucket.to_owned(), object.to_owned(), ver));
5156        }
5157
5158        Err(StorageError::ObjectNotFound(bucket.to_owned(), object.to_owned()))
5159    }
5160
5161    #[instrument(skip(self, objects, opts))]
5162    pub(super) async fn handle_delete_objects(
5163        &self,
5164        bucket: &str,
5165        objects: Vec<ObjectToDelete>,
5166        opts: ObjectOptions,
5167    ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
5168        self.handle_delete_objects_with_journal(bucket, objects, opts, None).await
5169    }
5170
5171    pub(super) async fn handle_delete_objects_with_journal(
5172        &self,
5173        bucket: &str,
5174        objects: Vec<ObjectToDelete>,
5175        opts: ObjectOptions,
5176        tier_journal_api: Option<Arc<ECStore>>,
5177    ) -> (Vec<DeletedObject>, Vec<Option<Error>>) {
5178        let (deleted, errors, _) = self
5179            .handle_delete_objects_with_journal_and_accounting(bucket, objects, opts, tier_journal_api)
5180            .await;
5181        (deleted, errors)
5182    }
5183
5184    pub(super) async fn handle_delete_objects_with_journal_and_accounting(
5185        &self,
5186        bucket: &str,
5187        objects: Vec<ObjectToDelete>,
5188        opts: ObjectOptions,
5189        tier_journal_api: Option<Arc<ECStore>>,
5190    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
5191        Box::pin(async move {
5192            let mut opts = opts;
5193            let receipt_sink = install_tier_free_version_receipt_sink(&mut opts);
5194            let result = self
5195                .handle_delete_objects_with_journal_and_accounting_inner(bucket, objects, opts, tier_journal_api)
5196                .await;
5197            enqueue_recorded_tier_free_versions(self, receipt_sink).await;
5198            result
5199        })
5200        .await
5201    }
5202
5203    async fn handle_delete_objects_with_journal_and_accounting_inner(
5204        &self,
5205        bucket: &str,
5206        objects: Vec<ObjectToDelete>,
5207        opts: ObjectOptions,
5208        tier_journal_api: Option<Arc<ECStore>>,
5209    ) -> (Vec<DeletedObject>, Vec<Option<Error>>, Vec<Option<DeleteAccounting>>) {
5210        // encode object name
5211        let objects: Vec<ObjectToDelete> = objects
5212            .iter()
5213            .map(|v| {
5214                let mut v = v.clone();
5215                v.object_name = encode_dir_object(v.object_name.as_str());
5216                v
5217            })
5218            .collect();
5219
5220        // Default return value
5221        let mut del_objects = vec![DeletedObject::default(); objects.len()];
5222        let mut accounting = vec![None; objects.len()];
5223
5224        let mut del_errs = Vec::with_capacity(objects.len());
5225        for _ in 0..objects.len() {
5226            del_errs.push(None)
5227        }
5228
5229        let mut opts = opts;
5230        opts.tier_delete_journal_api = tier_journal_api;
5231        let _bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
5232            None
5233        } else {
5234            match self.acquire_bucket_lifecycle_read_lock(bucket).await {
5235                Ok(guard) => Some(guard),
5236                Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5237            }
5238        };
5239        if let Some(guard) = _bucket_lifecycle_guard.as_ref() {
5240            opts.add_bucket_lifecycle_lock_guard(guard);
5241        }
5242        if opts.delete_replication_config_snapshot.is_none() {
5243            match ReplicationObjectBridge::delete_request_config_in(&self.ctx, bucket).await {
5244                Ok(snapshot) => opts.delete_replication_config_snapshot = Some(Arc::new(snapshot)),
5245                Err(err) => {
5246                    let message = err.to_string();
5247                    let errors = (0..objects.len()).map(|_| Some(Error::other(message.clone()))).collect();
5248                    return (del_objects, errors, accounting);
5249                }
5250            }
5251        }
5252        if !is_meta_bucketname(bucket)
5253            && let Err(err) = get_cached_bucket_incarnation_id_in(&self.ctx, bucket).await
5254        {
5255            return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err);
5256        }
5257        let _object_lock_metadata_guard = if is_meta_bucketname(bucket) {
5258            None
5259        } else {
5260            Some(match acquire_bucket_metadata_transaction_read_lock_in(&self.ctx, bucket).await {
5261                Ok(guard) => guard,
5262                Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5263            })
5264        };
5265        if let Some(guard) = _object_lock_metadata_guard.as_ref() {
5266            opts.add_namespace_lock_guard(guard);
5267        }
5268        let current_bucket_incarnation_id = if _object_lock_metadata_guard.is_some() {
5269            let (state, incarnation_id, config_revision) =
5270                match get_object_lock_config_and_incarnation_from_disk_in(&self.ctx, bucket).await {
5271                    Ok(snapshot) => snapshot,
5272                    Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5273                };
5274            opts.object_lock_config_snapshot = Some(Arc::new(ObjectLockConfigSnapshot::for_store_bucket(
5275                self.id,
5276                bucket,
5277                incarnation_id,
5278                config_revision,
5279                state,
5280            )));
5281            Some(incarnation_id)
5282        } else {
5283            None
5284        };
5285        if let (Some(expected), Some(current)) = (opts.expected_bucket_incarnation_id, current_bucket_incarnation_id)
5286            && expected != current
5287        {
5288            return return_batch_delete_lock_error_with_accounting(
5289                objects.as_slice(),
5290                StorageError::BucketNotFound(bucket.to_string()),
5291            );
5292        }
5293        if opts.expected_bucket_incarnation_id.is_none() {
5294            opts.expected_bucket_incarnation_id = current_bucket_incarnation_id;
5295        }
5296        #[cfg(any(test, feature = "test-util"))]
5297        if current_bucket_incarnation_id.is_some() {
5298            pause_delete_after_object_lock_snapshot(bucket).await;
5299        }
5300        let _object_lock_guards = match self.acquire_delete_objects_write_locks(bucket, &objects, &mut opts).await {
5301            Ok(guards) => guards,
5302            Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5303        };
5304        #[cfg(any(test, feature = "test-util"))]
5305        if !_object_lock_guards.is_empty() {
5306            notify_delete_namespace_acquired(bucket);
5307        }
5308
5309        let delete_config_snapshot = opts
5310            .delete_replication_config_snapshot
5311            .as_deref()
5312            .expect("batch delete replication config snapshot should be loaded");
5313        let latest_marker_objects = objects
5314            .iter()
5315            .map(|object| batch_delete_creates_latest_marker(object, delete_config_snapshot))
5316            .collect::<Vec<_>>();
5317        let marker_target_results = join_all(objects.iter().zip(&latest_marker_objects).map(
5318            |(object, creates_marker)| async move {
5319                if *creates_marker {
5320                    Some(self.get_pool_idx_no_lock(bucket, &object.object_name, 0).await)
5321                } else {
5322                    None
5323                }
5324            },
5325        ))
5326        .await;
5327        let mut marker_target_pool_indices = Vec::with_capacity(objects.len());
5328        for (idx, target_result) in marker_target_results.into_iter().enumerate() {
5329            match target_result {
5330                Some(Ok(pool_idx)) => marker_target_pool_indices.push(Some(pool_idx)),
5331                Some(Err(err)) => {
5332                    del_errs[idx] = Some(err);
5333                    marker_target_pool_indices.push(None);
5334                }
5335                None => marker_target_pool_indices.push(None),
5336            }
5337        }
5338
5339        let _capacity_fence = if !self.single_pool() && latest_marker_objects.iter().any(|creates_marker| *creates_marker) {
5340            // Only marker destinations can grow. Other pools participate in
5341            // exact deletion under the same metadata read fence and must not
5342            // be treated as publication targets merely because they retire.
5343            let mut target_pool_indices = marker_target_pool_indices.iter().flatten().copied().collect::<Vec<_>>();
5344            target_pool_indices.sort_unstable();
5345            target_pool_indices.dedup();
5346            match self
5347                .acquire_external_decommission_capacity_fence(&target_pool_indices, DecommissionCapacityAdmission::BatchDelete)
5348                .await
5349            {
5350                Ok(fence) => Some(fence),
5351                Err(err) => return return_batch_delete_lock_error_with_accounting(objects.as_slice(), err),
5352            }
5353        } else {
5354            None
5355        };
5356
5357        let mut futures = Vec::with_capacity(self.pools.len());
5358        for pool in self.pools.iter() {
5359            if self.is_pool_rebalancing(pool.pool_idx).await {
5360                continue;
5361            }
5362
5363            let (object_indices, pool_objects): (Vec<_>, Vec<_>) = objects
5364                .iter()
5365                .enumerate()
5366                .filter(|(idx, _)| {
5367                    batch_delete_targets_pool(latest_marker_objects[*idx], marker_target_pool_indices[*idx], pool.pool_idx)
5368                })
5369                .map(|(idx, object)| (idx, object.clone()))
5370                .unzip();
5371            if pool_objects.is_empty() {
5372                continue;
5373            }
5374
5375            let pool_opts = opts.clone();
5376            futures.push(async move {
5377                #[cfg(test)]
5378                let pool_object_names = pool_objects
5379                    .iter()
5380                    .map(|object| object.object_name.clone())
5381                    .collect::<Vec<_>>();
5382                let result = pool.delete_objects_with_accounting(bucket, pool_objects, pool_opts).await;
5383                #[cfg(test)]
5384                let result = {
5385                    let mut result = result;
5386                    let (deleted, errors, _) = &mut result;
5387                    inject_batch_delete_pool_errors(bucket, pool.pool_idx, &pool_object_names, deleted, errors);
5388                    result
5389                };
5390                (object_indices, result)
5391            });
5392        }
5393
5394        let results = join_all(futures).await;
5395
5396        for idx in 0..del_objects.len() {
5397            let pool_results = results.iter().filter_map(|(object_indices, (dels, errs, _))| {
5398                let pool_object_idx = object_indices.binary_search(&idx).ok()?;
5399                Some((&dels[pool_object_idx], &errs[pool_object_idx]))
5400            });
5401            let (deleted, error, attempted) = resolve_batch_delete_pool_results(del_errs[idx].take(), pool_results);
5402            if let Some(deleted) = deleted {
5403                del_objects[idx] = deleted;
5404            }
5405            del_errs[idx] = error;
5406
5407            if !attempted && del_errs[idx].is_none() && latest_marker_objects[idx] {
5408                del_objects[idx] = DeletedObject {
5409                    object_name: objects[idx].object_name.clone(),
5410                    version_id: objects[idx].version_id,
5411                    ..Default::default()
5412                };
5413                del_errs[idx] = Some(StorageError::ObjectNotFound(bucket.to_owned(), objects[idx].object_name.clone()));
5414            }
5415        }
5416
5417        for (object_indices, (_, _, pool_accounting)) in &results {
5418            for (pool_object_idx, object_idx) in object_indices.iter().enumerate() {
5419                accounting[*object_idx] = pool_accounting.get(pool_object_idx).cloned().flatten();
5420            }
5421        }
5422
5423        #[cfg(test)]
5424        for (idx, object) in objects.iter().enumerate() {
5425            if del_errs[idx].is_none() && del_objects[idx].delete_marker {
5426                pause_versioned_delete_marker_after_commit(bucket, &object.object_name).await;
5427            }
5428        }
5429
5430        del_objects.iter_mut().for_each(|v| {
5431            v.object_name = decode_dir_object(&v.object_name);
5432        });
5433
5434        (del_objects, del_errs, accounting)
5435
5436        // let mut futures = Vec::with_capacity(objects.len());
5437
5438        // for obj in objects.iter() {
5439        //     futures.push(async move {
5440        //         self.internal_get_pool_info_existing_with_opts(
5441        //             bucket,
5442        //             &obj.object_name,
5443        //             &ObjectOptions {
5444        //                 no_lock: true,
5445        //                 ..Default::default()
5446        //             },
5447        //         )
5448        //         .await
5449        //     });
5450        // }
5451
5452        // let results = join_all(futures).await;
5453
5454        // // let mut jhs = Vec::new();
5455        // // let semaphore = Arc::new(Semaphore::new(num_cpus::get()));
5456        // // let pools = Arc::new(self.pools.clone());
5457
5458        // // for obj in objects.iter() {
5459        // //     let (semaphore, pools, bucket, object_name, opt) = (
5460        // //         semaphore.clone(),
5461        // //         pools.clone(),
5462        // //         bucket.to_string(),
5463        // //         obj.object_name.to_string(),
5464        // //         ObjectOptions::default(),
5465        // //     );
5466
5467        // //     let jh = tokio::spawn(async move {
5468        // //         let _permit = semaphore.acquire().await.unwrap();
5469        // //         self.internal_get_pool_info_existing_with_opts(pools.as_ref(), &bucket, &object_name, &opt)
5470        // //             .await
5471        // //     });
5472        // //     jhs.push(jh);
5473        // // }
5474        // // let mut results = Vec::new();
5475        // // for jh in jhs {
5476        // //     results.push(jh.await.unwrap());
5477        // // }
5478
5479        // // Record the mapping pool_idx -> object index
5480        // let mut pool_obj_idx_map = HashMap::new();
5481        // let mut orig_index_map = HashMap::new();
5482
5483        // for (i, res) in results.into_iter().enumerate() {
5484        //     match res {
5485        //         Ok((pinfo, _)) => {
5486        //             if let Some(obj) = objects.get(i) {
5487        //                 if pinfo.object_info.delete_marker && obj.version_id.is_none() {
5488        //                     del_objects[i] = DeletedObject {
5489        //                         delete_marker: pinfo.object_info.delete_marker,
5490        //                         delete_marker_version_id: pinfo.object_info.version_id.map(|v| v.to_string()),
5491        //                         object_name: decode_dir_object(&pinfo.object_info.name),
5492        //                         delete_marker_mtime: pinfo.object_info.mod_time,
5493        //                         ..Default::default()
5494        //                     };
5495        //                     continue;
5496        //                 }
5497
5498        //                 if !pool_obj_idx_map.contains_key(&pinfo.index) {
5499        //                     pool_obj_idx_map.insert(pinfo.index, vec![obj.clone()]);
5500        //                 } else if let Some(val) = pool_obj_idx_map.get_mut(&pinfo.index) {
5501        //                     val.push(obj.clone());
5502        //                 }
5503
5504        //                 if !orig_index_map.contains_key(&pinfo.index) {
5505        //                     orig_index_map.insert(pinfo.index, vec![i]);
5506        //                 } else if let Some(val) = orig_index_map.get_mut(&pinfo.index) {
5507        //                     val.push(i);
5508        //                 }
5509        //             }
5510        //         }
5511        //         Err(e) => {
5512        //             if !is_err_object_not_found(&e) && is_err_version_not_found(&e) {
5513        //                 del_errs[i] = Some(e)
5514        //             }
5515
5516        //             if let Some(obj) = objects.get(i) {
5517        //                 del_objects[i] = DeletedObject {
5518        //                     object_name: decode_dir_object(&obj.object_name),
5519        //                     version_id: obj.version_id.map(|v| v.to_string()),
5520        //                     ..Default::default()
5521        //                 }
5522        //             }
5523        //         }
5524        //     }
5525        // }
5526
5527        // if !pool_obj_idx_map.is_empty() {
5528        //     for (i, sets) in self.pools.iter().enumerate() {
5529        //         // Retrieve the object index for a pool idx
5530        //         if let Some(objs) = pool_obj_idx_map.get(&i) {
5531        //             // Fetch the corresponding object (should never be None)
5532        //             // let objs: Vec<ObjectToDelete> = obj_idxs.iter().filter_map(|&idx| objects.get(idx).cloned()).collect();
5533
5534        //             if objs.is_empty() {
5535        //                 continue;
5536        //             }
5537
5538        //             let (pdel_objs, perrs) = sets.delete_objects(bucket, objs.clone(), opts.clone()).await?;
5539
5540        //             // Insert simultaneously (should never be None)
5541        //             let org_indexes = orig_index_map.get(&i).unwrap();
5542
5543        //             // perrs should follow the same order as obj_idxs
5544        //             for (i, err) in perrs.into_iter().enumerate() {
5545        //                 let obj_idx = org_indexes[i];
5546
5547        //                 if err.is_some() {
5548        //                     del_errs[obj_idx] = err;
5549        //                 }
5550
5551        //                 let mut dobj = pdel_objs.get(i).unwrap().clone();
5552        //                 dobj.object_name = decode_dir_object(&dobj.object_name);
5553
5554        //                 del_objects[obj_idx] = dobj;
5555        //             }
5556        //         }
5557        //     }
5558        // }
5559
5560        // Ok((del_objects, del_errs))
5561    }
5562
5563    #[instrument(skip(self))]
5564    pub(super) async fn handle_add_partial(&self, bucket: &str, object: &str, version_id: &str) -> Result<()> {
5565        let object = encode_dir_object(object);
5566
5567        if self.single_pool() {
5568            let _ = self.pools[0].add_partial(bucket, object.as_str(), version_id).await;
5569            return Ok(());
5570        }
5571
5572        let opts = ObjectOptions {
5573            version_id: Some(version_id.to_string()),
5574            ..Default::default()
5575        };
5576        let (_, idx) = self
5577            .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(&opts, opts.no_lock))
5578            .await?;
5579
5580        let _ = self.pools[idx].add_partial(bucket, object.as_str(), version_id).await;
5581        Ok(())
5582    }
5583
5584    #[instrument(skip(self))]
5585    pub(super) async fn handle_transition_object(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
5586        let object = encode_dir_object(object);
5587        if self.single_pool() {
5588            return self.pools[0].transition_object(bucket, &object, opts).await;
5589        }
5590
5591        let opts = transition_restore_pool_opts(opts);
5592        let (_, idx) = self
5593            .get_latest_accessible_object_info_with_idx(bucket, &object, &opts)
5594            .await?;
5595
5596        self.pools[idx].transition_object(bucket, &object, &opts).await
5597    }
5598
5599    #[instrument(skip(self))]
5600    pub(super) async fn handle_restore_transitioned_object(
5601        self: Arc<Self>,
5602        bucket: &str,
5603        object: &str,
5604        opts: &ObjectOptions,
5605    ) -> Result<()> {
5606        let object = encode_dir_object(object);
5607        let mut opts = transition_restore_pool_opts(opts);
5608        if !is_meta_bucketname(bucket) && opts.expected_bucket_incarnation_id.is_none() {
5609            opts.expected_bucket_incarnation_id = Some(self.bucket_incarnation_id(bucket).await?);
5610        }
5611        let bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
5612            None
5613        } else {
5614            Some(self.acquire_bucket_lifecycle_read_lock(bucket).await?)
5615        };
5616        if let Some(guard) = bucket_lifecycle_guard.as_ref() {
5617            opts.add_bucket_lifecycle_lock_guard(guard);
5618        }
5619        if !is_meta_bucketname(bucket) {
5620            let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
5621            if opts.expected_bucket_incarnation_id != Some(current_incarnation_id) {
5622                return Err(StorageError::BucketNotFound(bucket.to_string()));
5623            }
5624        }
5625        if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
5626            let expected_incarnation_id = opts
5627                .expected_bucket_incarnation_id
5628                .ok_or_else(|| Error::other("restore is missing its bucket incarnation snapshot"))?;
5629            let lifecycle_fence = opts
5630                .bucket_lifecycle_lock_fence
5631                .as_ref()
5632                .ok_or_else(|| Error::other("restore is missing its bucket lifecycle fence"))?;
5633            let snapshot = match opts.object_lock_config_snapshot.as_ref() {
5634                Some(snapshot) => Arc::clone(snapshot),
5635                None => {
5636                    self.object_lock_config_snapshot_under_lifecycle_fence(bucket, lifecycle_fence)
5637                        .await?
5638                }
5639            };
5640            if !snapshot.is_valid_for_destructive_put(self.id, bucket, expected_incarnation_id) {
5641                return Err(Error::other("restore Object Lock snapshot does not match the target bucket generation"));
5642            }
5643            snapshot.add_lock_fences(&mut opts);
5644            opts.object_lock_config_snapshot = Some(snapshot);
5645        }
5646        // Deliberately NOT holding the object write lock across the tier
5647        // copy-back (backlog#1304): non-SELECT restore-vs-restore is
5648        // serialized by the accept path's compare-and-set of the ongoing flag
5649        // (see acquire_restore_accept_guard), and while ongoing-request="true"
5650        // the restore header parses with no expiry, so DeleteRestoredAction
5651        // cannot fire mid-copy-back. Torn-write protection against concurrent
5652        // readers and writers stays with the inner put_object /
5653        // complete_multipart_upload commit phases, which take this object's
5654        // write lock themselves. A delete (user or lifecycle) landing between
5655        // the tier read and that commit can still be overwritten by the
5656        // commit — the same window MinIO accepts; a commit-time existence
5657        // re-check is tracked separately. Holding the lock here instead
5658        // (#4877) blocked HEAD/get_object_info for the whole copy-back and
5659        // self-deadlocked on the inner commits.
5660        let object_name = object.as_str();
5661        if self.single_pool() {
5662            return self.pools[0]
5663                .clone()
5664                .restore_transitioned_object(bucket, object_name, &opts)
5665                .await;
5666        }
5667
5668        let (_, idx) = self
5669            .get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(&opts, opts.no_lock))
5670            .await?;
5671
5672        opts.decommission_capacity_admission = Some(Arc::clone(&self));
5673        self.pools[idx]
5674            .clone()
5675            .restore_transitioned_object(bucket, object_name, &opts)
5676            .await
5677    }
5678
5679    #[instrument(skip(self))]
5680    pub(super) async fn handle_put_object_metadata(
5681        &self,
5682        bucket: &str,
5683        object: &str,
5684        opts: &ObjectOptions,
5685    ) -> Result<ObjectInfo> {
5686        let object = encode_dir_object(object);
5687        let mut opts = opts.clone();
5688        opts.metadata_chg = true;
5689        let bucket_lifecycle_guard = if is_meta_bucketname(bucket) {
5690            None
5691        } else {
5692            let guard = self.acquire_bucket_lifecycle_read_lock(bucket).await?;
5693            let current_incarnation_id = get_bucket_incarnation_id_in(&self.ctx, bucket).await?;
5694            if opts
5695                .expected_bucket_incarnation_id
5696                .is_some_and(|expected| expected != current_incarnation_id)
5697            {
5698                return Err(StorageError::BucketNotFound(bucket.to_string()));
5699            }
5700            opts.expected_bucket_incarnation_id = Some(current_incarnation_id);
5701            opts.add_bucket_lifecycle_lock_guard(&guard);
5702            if guard.is_lock_lost() {
5703                return Err(Error::other("bucket lifecycle lock was lost before the metadata update"));
5704            }
5705            Some(guard)
5706        };
5707
5708        if self.single_pool() {
5709            let pool = self.pools[0].clone();
5710            let capacity_object = object.clone();
5711            return self
5712                .run_external_decommission_capacity_object_mutation(
5713                    0,
5714                    bucket,
5715                    capacity_object.as_str(),
5716                    capacity_object.as_str(),
5717                    opts,
5718                    |opts| async move { pool.put_object_metadata(bucket, object.as_str(), &opts).await },
5719                )
5720                .await;
5721        }
5722
5723        let (_, idx) = self
5724            .get_latest_accessible_object_info_with_idx(bucket, object.as_str(), &writer_pool_lookup_opts(&opts, opts.no_lock))
5725            .await?;
5726
5727        let pool = self.pools[idx].clone();
5728        let capacity_object = object.clone();
5729        let result = self
5730            .run_external_decommission_capacity_object_mutation(
5731                idx,
5732                bucket,
5733                capacity_object.as_str(),
5734                capacity_object.as_str(),
5735                opts,
5736                |opts| async move { pool.put_object_metadata(bucket, object.as_str(), &opts).await },
5737            )
5738            .await;
5739        drop(bucket_lifecycle_guard);
5740        result
5741    }
5742
5743    #[instrument(skip(self))]
5744    pub(super) async fn handle_get_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<String> {
5745        let object = encode_dir_object(object);
5746
5747        if self.single_pool() {
5748            return self.pools[0].get_object_tags(bucket, object.as_str(), opts).await;
5749        }
5750
5751        let (oi, _) = self.get_latest_accessible_object_info_with_idx(bucket, &object, opts).await?;
5752        Ok((*oi.user_tags).clone())
5753    }
5754
5755    #[instrument(level = "debug", skip(self))]
5756    pub(super) async fn handle_put_object_tags(
5757        &self,
5758        bucket: &str,
5759        object: &str,
5760        tags: &str,
5761        opts: &ObjectOptions,
5762    ) -> Result<ObjectInfo> {
5763        let object = encode_dir_object(object);
5764        let object_name = object.as_str();
5765
5766        if self.single_pool() {
5767            return self
5768                .run_external_decommission_capacity_object_mutation(
5769                    0,
5770                    bucket,
5771                    object_name,
5772                    object_name,
5773                    opts.clone(),
5774                    |opts| async move { self.pools[0].put_object_tags(bucket, object_name, tags, &opts).await },
5775                )
5776                .await;
5777        }
5778
5779        let (_, idx) = self
5780            .get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(opts, opts.no_lock))
5781            .await?;
5782
5783        self.run_external_decommission_capacity_object_mutation(
5784            idx,
5785            bucket,
5786            object_name,
5787            object_name,
5788            opts.clone(),
5789            |opts| async move { self.pools[idx].put_object_tags(bucket, object_name, tags, &opts).await },
5790        )
5791        .await
5792    }
5793
5794    #[instrument(skip(self))]
5795    pub(super) async fn handle_delete_object_version(
5796        &self,
5797        bucket: &str,
5798        object: &str,
5799        fi: &FileInfo,
5800        force_del_marker: bool,
5801    ) -> Result<()> {
5802        check_del_obj_args(bucket, object)?;
5803
5804        let object = encode_dir_object(object);
5805
5806        if self.single_pool() {
5807            return self.pools[0]
5808                .delete_object_version(bucket, object.as_str(), fi, force_del_marker)
5809                .await;
5810        }
5811        Err(StorageError::NotImplemented)
5812    }
5813
5814    #[instrument(skip(self))]
5815    pub(super) async fn handle_delete_object_tags(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<ObjectInfo> {
5816        let object = encode_dir_object(object);
5817        let object_name = object.as_str();
5818
5819        if self.single_pool() {
5820            return self
5821                .run_external_decommission_capacity_object_mutation(
5822                    0,
5823                    bucket,
5824                    object_name,
5825                    object_name,
5826                    opts.clone(),
5827                    |opts| async move { self.pools[0].delete_object_tags(bucket, object_name, &opts).await },
5828                )
5829                .await;
5830        }
5831
5832        let (_, idx) = self
5833            .get_latest_accessible_object_info_with_idx(bucket, object_name, &writer_pool_lookup_opts(opts, opts.no_lock))
5834            .await?;
5835
5836        self.run_external_decommission_capacity_object_mutation(
5837            idx,
5838            bucket,
5839            object_name,
5840            object_name,
5841            opts.clone(),
5842            |opts| async move { self.pools[idx].delete_object_tags(bucket, object_name, &opts).await },
5843        )
5844        .await
5845    }
5846
5847    pub(super) async fn handle_verify_object_integrity(&self, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
5848        let get_object_reader = <Self as crate::storage_api_contracts::object::ObjectIO>::get_object_reader(
5849            self,
5850            bucket,
5851            object,
5852            None,
5853            HeaderMap::new(),
5854            opts,
5855        )
5856        .await?;
5857        // Stream to sink to avoid loading entire object into memory during verification
5858        let mut reader = get_object_reader.stream;
5859        tokio::io::copy(&mut reader, &mut tokio::io::sink()).await?;
5860        Ok(())
5861    }
5862}
5863
5864#[cfg(test)]
5865mod tests {
5866    use super::*;
5867    use crate::bucket::lifecycle::core::TRANSITION_COMPLETE;
5868    use crate::bucket::lifecycle::tier_sweeper::TierDeleteJournalState;
5869    use crate::bucket::metadata_sys::ObjectLockConfigState;
5870    use crate::bucket::replication::{
5871        ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_state_to_filemeta, replication_statuses_map,
5872        version_purge_statuses_map,
5873    };
5874    use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
5875    use crate::core::sets::make_local_two_set_sets_with_ctx;
5876    use crate::ecstore_validation_blackbox::{RefreshLossLockClient, make_local_set_disks, make_local_set_disks_with_ctx};
5877    use crate::layout::{
5878        endpoints::{Endpoints, PoolEndpoints, SetupType},
5879        format::FormatV3,
5880    };
5881    use crate::object_api::{
5882        GetObjectBodyCacheHook, GetObjectBodyCacheHookLookup, GetObjectBodySource, clear_get_object_body_cache_hook,
5883        lookup_get_object_body_cache_hook, register_get_object_body_cache_hook,
5884    };
5885    use crate::set_disk::same_distributed_lock_domain;
5886    use crate::set_disk::{SetDisks, disk_call_counters};
5887    use crate::storage_api_contracts::bucket::MakeBucketOptions;
5888    use crate::storage_api_contracts::lifecycle::TransitionedObject;
5889    use bytes::Bytes;
5890    use std::future::Future as _;
5891    use std::io::Cursor;
5892    use std::sync::Arc;
5893    use std::sync::atomic::{AtomicUsize, Ordering};
5894    use tokio::io::AsyncReadExt;
5895
5896    #[test]
5897    fn predecessor_replay_scope_distinguishes_exact_object_from_recursive_prefix() {
5898        let source = TierDeleteSourceIdentity {
5899            bucket: "bucket".to_string(),
5900            object: "directory/child".to_string(),
5901            version_id: Some("version".to_string()),
5902            versioned: true,
5903            version_suspended: false,
5904            data_dir: Some("data-dir".to_string()),
5905            etag: Some("etag".to_string()),
5906            mod_time: Some("mod-time".to_string()),
5907        };
5908        let persisted_prefix = encode_dir_object("directory/");
5909
5910        assert!(tier_delete_source_matches_replay_scope(&source, "bucket", &persisted_prefix, false,));
5911        assert!(!tier_delete_source_matches_replay_scope(&source, "bucket", &persisted_prefix, true,));
5912        let mut exact_source = source;
5913        exact_source.object = "directory/".to_string();
5914        assert!(tier_delete_source_matches_replay_scope(&exact_source, "bucket", &persisted_prefix, true,));
5915    }
5916
5917    struct WaitForLockLossReader {
5918        inner: Cursor<Vec<u8>>,
5919        poll_started: Option<tokio::sync::oneshot::Sender<()>>,
5920        resume: ParkingMutex<std::sync::mpsc::Receiver<()>>,
5921    }
5922
5923    impl AsyncRead for WaitForLockLossReader {
5924        fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
5925            let poll = Pin::new(&mut self.inner).poll_read(cx, buf);
5926            if let Some(poll_started) = self.poll_started.take() {
5927                let _ = poll_started.send(());
5928                if self.resume.lock().recv_timeout(Duration::from_secs(10)).is_err() {
5929                    return Poll::Ready(Err(std::io::Error::new(
5930                        std::io::ErrorKind::TimedOut,
5931                        "lock-loss signal was not observed during read",
5932                    )));
5933                }
5934            }
5935            poll
5936        }
5937    }
5938
5939    struct PermanentlyPendingReader {
5940        poll_started: Option<tokio::sync::oneshot::Sender<()>>,
5941    }
5942
5943    impl AsyncRead for PermanentlyPendingReader {
5944        fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
5945            if let Some(poll_started) = self.poll_started.take() {
5946                let _ = poll_started.send(());
5947            }
5948            Poll::Pending
5949        }
5950    }
5951
5952    struct CommitLateBodyReader {
5953        inner: Cursor<Vec<u8>>,
5954        poll_started: Option<tokio::sync::oneshot::Sender<()>>,
5955        release: Option<tokio::sync::oneshot::Receiver<()>>,
5956    }
5957
5958    impl AsyncRead for CommitLateBodyReader {
5959        fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
5960            if let Some(poll_started) = self.poll_started.take() {
5961                let _ = poll_started.send(());
5962            }
5963            if let Some(release) = self.release.as_mut() {
5964                match Pin::new(release).poll(cx) {
5965                    Poll::Pending => return Poll::Pending,
5966                    Poll::Ready(Ok(())) => self.release = None,
5967                    Poll::Ready(Err(_)) => {
5968                        return Poll::Ready(Err(std::io::Error::new(
5969                            std::io::ErrorKind::BrokenPipe,
5970                            "commit-late body release sender was dropped",
5971                        )));
5972                    }
5973                }
5974            }
5975            Pin::new(&mut self.inner).poll_read(cx, buf)
5976        }
5977    }
5978
5979    struct CountingMissHook {
5980        calls: AtomicUsize,
5981    }
5982
5983    async fn refresh_failure_test_guard(
5984        owner: &'static str,
5985    ) -> (
5986        ObjectLockDiagGuard,
5987        Arc<rustfs_lock::distributed_lock::LockLostSignal>,
5988        Arc<RefreshLossLockClient>,
5989    ) {
5990        let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
5991            rustfs_lock::FastObjectLockManager::new(),
5992        )));
5993        let client = Arc::new(RefreshLossLockClient::with_manager(manager));
5994        let namespace_lock = rustfs_lock::NamespaceLock::with_clients_and_quorum(
5995            owner.to_string(),
5996            vec![Arc::clone(&client) as Arc<dyn rustfs_lock::LockClient>],
5997            1,
5998        );
5999        let request =
6000            rustfs_lock::LockRequest::new(rustfs_lock::ObjectKey::new("bucket", "object"), rustfs_lock::LockType::Shared, owner)
6001                .with_ttl(Duration::from_secs(2))
6002                .with_refresh_interval(Duration::from_millis(20));
6003        let guard = namespace_lock
6004            .acquire_guard(&request)
6005            .await
6006            .expect("distributed read lock acquisition should succeed")
6007            .expect("distributed read lock should reach quorum");
6008        let guard = ObjectLockDiagGuard::new(
6009            guard,
6010            false,
6011            "SelectObjectContent",
6012            Some("bucket".to_string()),
6013            Some("object".to_string()),
6014            Some(owner.to_string()),
6015            ObjectLockDiagMode::Read,
6016        );
6017        let signal = guard
6018            .lock_lost_signal()
6019            .expect("a distributed guard should expose a lock-loss signal");
6020        (guard, signal, client)
6021    }
6022
6023    async fn refresh_failure_test_lease(
6024        owner: &'static str,
6025    ) -> (
6026        Arc<SelectObjectSnapshotLease>,
6027        Arc<rustfs_lock::distributed_lock::LockLostSignal>,
6028        Arc<RefreshLossLockClient>,
6029    ) {
6030        let (guard, signal, client) = refresh_failure_test_guard(owner).await;
6031        (Arc::new(SelectObjectSnapshotLease::new(vec![guard])), signal, client)
6032    }
6033
6034    #[async_trait::async_trait]
6035    impl GetObjectBodyCacheHook for CountingMissHook {
6036        async fn lookup(&self, _bucket: &str, _object: &str, _info: &ObjectInfo) -> Option<Bytes> {
6037            self.calls.fetch_add(1, Ordering::Relaxed);
6038            None
6039        }
6040    }
6041
6042    struct BodyCacheHookGuard;
6043
6044    #[test]
6045    fn select_snapshot_deduplicates_same_distributed_lock_domain() {
6046        let first: Arc<dyn rustfs_lock::LockClient> = Arc::new(rustfs_lock::LocalClient::new());
6047        let second: Arc<dyn rustfs_lock::LockClient> = Arc::new(rustfs_lock::LocalClient::new());
6048        let other: Arc<dyn rustfs_lock::LockClient> = Arc::new(rustfs_lock::LocalClient::new());
6049
6050        assert!(same_distributed_lock_domain(
6051            &[Arc::clone(&first), Arc::clone(&second)],
6052            &[Arc::clone(&second), Arc::clone(&first)]
6053        ));
6054        assert!(same_distributed_lock_domain(
6055            &[Arc::clone(&first), Arc::clone(&first), Arc::clone(&second)],
6056            &[Arc::clone(&second), Arc::clone(&first)]
6057        ));
6058        assert!(!same_distributed_lock_domain(&[first, second], &[other]));
6059    }
6060
6061    #[tokio::test]
6062    async fn publication_commit_locks_dedupe_dist_sets_with_same_clients() {
6063        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6064        let (_dirs, original_sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await;
6065        let mut second_set = (*original_sets.disk_set[1]).clone();
6066        second_set.lockers = original_sets.disk_set[0].lockers.clone();
6067        let mut sets = (*original_sets).clone();
6068        sets.disk_set[1] = Arc::new(second_set);
6069        let sets = Arc::new(sets);
6070        ctx.update_erasure_type(SetupType::DistErasure).await;
6071
6072        assert!(
6073            sets.disk_set[0]
6074                .lockers
6075                .iter()
6076                .zip(&sets.disk_set[1].lockers)
6077                .all(|(fixed, hashed)| Arc::ptr_eq(fixed, hashed)),
6078            "the regression requires identical distributed lock clients"
6079        );
6080        assert_ne!(sets.disk_set[0].set_index, sets.disk_set[1].set_index);
6081
6082        let pool_config = sets.endpoints.clone();
6083        let store = new_prepared_reader_test_store_from_pools(vec![Arc::clone(&sets)], vec![pool_config], ctx);
6084        let object = (0..1_000)
6085            .map(|index| format!("decommission-dist-domain-{index}.bin"))
6086            .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1]))
6087            .expect("a key should hash to the second set namespace");
6088        let publication_guards = store
6089            .acquire_data_movement_publication_write_locks("bucket", &encode_dir_object(&object), 0, 0, true)
6090            .await
6091            .expect("the publication commit lock set should be acquired");
6092        let target_lock = sets.disk_set[1]
6093            .new_ns_lock("bucket", &object)
6094            .await
6095            .expect("the hashed-set namespace lock should be created");
6096        let target_err = target_lock
6097            .get_write_lock(Duration::from_millis(50))
6098            .await
6099            .expect_err("the fixed write fence must conflict through the shared clients");
6100        assert!(matches!(target_err, rustfs_lock::LockError::Timeout { .. }));
6101
6102        drop(publication_guards);
6103
6104        let cleanup_object = (0..1_000)
6105            .map(|index| format!("decommission-dist-cleanup-{index}.bin"))
6106            .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1]))
6107            .expect("a cleanup key should hash to the second set namespace");
6108        let source_fence = store
6109            .acquire_decommission_source_cleanup_fence("bucket", &cleanup_object, sets.disk_set[1].as_ref())
6110            .await
6111            .expect("the fixed distributed cleanup fence should be acquired");
6112        assert!(source_fence.source_lock_covered(), "source cleanup must reuse the covering fixed fence");
6113        let source_lock = sets.disk_set[1]
6114            .new_ns_lock("bucket", &cleanup_object)
6115            .await
6116            .expect("the source-set namespace lock should be created");
6117        let source_err = source_lock
6118            .get_read_lock(Duration::from_millis(50))
6119            .await
6120            .expect_err("the fixed write fence must conflict through the shared clients");
6121        assert!(matches!(source_err, rustfs_lock::LockError::Timeout { .. }));
6122    }
6123
6124    #[tokio::test]
6125    async fn physical_absence_proof_blocks_decommission_remote_tuple_publication() {
6126        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6127        let (_dirs, sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await;
6128        ctx.update_erasure_type(SetupType::DistErasure).await;
6129        let pool_config = sets.endpoints.clone();
6130        let store = new_prepared_reader_test_store_from_pools(vec![Arc::clone(&sets)], vec![pool_config], ctx);
6131        let bucket = "remote-tuple-publication-fence";
6132        let object = "transitioned.bin";
6133        let encoded = encode_dir_object(object);
6134        let mut proof_opts = ObjectOptions::default();
6135        let proof = store
6136            .acquire_all_physical_object_read_locks("tier_delete_journal_recovery", bucket, &encoded, &mut proof_opts)
6137            .await
6138            .expect("the all-physical absence proof should acquire every read domain");
6139
6140        // Keep polling the same distributed-lock request after the bounded
6141        // observation below.  Dropping a timed-out request can race with the
6142        // lock service granting it and would turn the test itself into an
6143        // abandoned-owner scenario.
6144        let mut publication = Box::pin(store.acquire_data_movement_publication_write_locks(bucket, &encoded, 0, 0, true));
6145        let blocked = tokio::time::timeout(Duration::from_millis(50), &mut publication).await;
6146        assert!(
6147            blocked.is_err(),
6148            "a decommission publisher must remain blocked while the remote-delete absence proof is held"
6149        );
6150
6151        drop(proof);
6152        publication
6153            .await
6154            .expect("the publisher should acquire its write fence after the proof releases");
6155    }
6156
6157    #[cfg(feature = "test-util")]
6158    #[tokio::test]
6159    async fn restored_transitioned_rebalance_revalidates_source_after_recovery_lock() {
6160        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6161        let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6162        let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6163        let store = Arc::new(
6164            new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], Arc::clone(&ctx)).await,
6165        );
6166        let bucket = "restored-transitioned-rebalance-publication";
6167        let object = "restored.bin";
6168        for set in [&first_set, &second_set] {
6169            set.make_bucket(bucket, &MakeBucketOptions::default())
6170                .await
6171                .expect("publication-race bucket should be created");
6172        }
6173
6174        let tier_name = "RESTORED-REBALANCE";
6175        crate::services::tier::test_util::register_mock_tier(&store.tier_config_mgr(), tier_name).await;
6176        let lease = TierConfigMgr::acquire_operation_lease(&store.tier_config_mgr(), tier_name)
6177            .await
6178            .expect("mock tier generation should be available");
6179        let backend_identity = lease.backend_identity();
6180        drop(lease);
6181
6182        let mut metadata = HashMap::new();
6183        for (suffix, value) in [
6184            (rustfs_utils::http::SUFFIX_TRANSITION_STATUS, TRANSITION_COMPLETE.to_string()),
6185            (rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME, "remote/restored.bin".to_string()),
6186            (rustfs_utils::http::SUFFIX_TRANSITION_TIER, tier_name.to_string()),
6187            (rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID, "remote-version".to_string()),
6188            (
6189                rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
6190                rustfs_filemeta::TransitionVersionState::Exact.as_str().to_string(),
6191            ),
6192            (
6193                rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6194                rustfs_utils::crypto::hex(backend_identity),
6195            ),
6196        ] {
6197            rustfs_utils::http::metadata_compat::insert_str(&mut metadata, suffix, value);
6198        }
6199        metadata.insert(
6200            "x-amz-restore".to_string(),
6201            "ongoing-request=\"false\", expiry-date=\"2099-01-01T00:00:00Z\"".to_string(),
6202        );
6203        let mut body = PutObjReader::from_vec(b"restored local body".to_vec());
6204        first_set
6205            .put_object(
6206                bucket,
6207                object,
6208                &mut body,
6209                &ObjectOptions {
6210                    user_defined: metadata,
6211                    no_lock: true,
6212                    ..Default::default()
6213                },
6214            )
6215            .await
6216            .expect("restored-transitioned source should be written");
6217        let source = first_set
6218            .get_object_info(
6219                bucket,
6220                object,
6221                &ObjectOptions {
6222                    include_part_checksums: true,
6223                    no_lock: true,
6224                    ..Default::default()
6225                },
6226            )
6227            .await
6228            .expect("restored-transitioned source metadata should be readable");
6229        assert_eq!(source.transitioned_object.status, TRANSITION_COMPLETE);
6230        assert!(source.data_dir.is_some(), "fixture must retain restored local data");
6231
6232        let publisher_first = store
6233            .acquire_remote_tuple_publication_fence(bucket, 0, &source, false)
6234            .await
6235            .expect("rebalance should capture an exact publication capability");
6236        let encoded = encode_dir_object(object);
6237        let publisher_guard = publisher_first
6238            .into_commit_guard(1, bucket, &encoded)
6239            .await
6240            .expect("unchanged source should validate at the publication boundary");
6241        let mut publisher_first_proof_opts = ObjectOptions::default();
6242        let mut publisher_first_proof = Box::pin(store.acquire_all_physical_object_read_locks(
6243            "tier_delete_journal_recovery",
6244            bucket,
6245            &encoded,
6246            &mut publisher_first_proof_opts,
6247        ));
6248        assert!(
6249            tokio::time::timeout(Duration::from_millis(50), &mut publisher_first_proof)
6250                .await
6251                .is_err(),
6252            "recovery must wait while a validated publisher owns the commit scope"
6253        );
6254        drop(publisher_guard);
6255        drop(
6256            publisher_first_proof
6257                .await
6258                .expect("recovery proof should acquire after publisher commit scope releases"),
6259        );
6260
6261        let publication = store
6262            .acquire_remote_tuple_publication_fence(bucket, 0, &source, false)
6263            .await
6264            .expect("recovery-first race should capture the old source identity");
6265
6266        // The pre-copy capability itself must not hold a namespace write lock.
6267        let pre_commit_reader = first_set
6268            .new_ns_lock(bucket, &encoded)
6269            .await
6270            .expect("source read lock should be created")
6271            .get_read_lock(Duration::from_millis(50))
6272            .await
6273            .expect("body staging must not be blocked by an early publication write lock");
6274        drop(pre_commit_reader);
6275
6276        first_set
6277            .delete_object(
6278                bucket,
6279                &encoded,
6280                ObjectOptions {
6281                    delete_prefix: true,
6282                    delete_prefix_object: true,
6283                    data_movement: true,
6284                    no_lock: true,
6285                    ..Default::default()
6286                },
6287            )
6288            .await
6289            .expect("source cleanup should remove the old physical owner");
6290
6291        let mut proof_opts = ObjectOptions::default();
6292        let proof = store
6293            .acquire_all_physical_object_read_locks("tier_delete_journal_recovery", bucket, &encoded, &mut proof_opts)
6294            .await
6295            .expect("recovery should hold its all-physical absence proof");
6296        let mut commit = Box::pin(publication.into_commit_guard(1, bucket, &encoded));
6297        assert!(
6298            tokio::time::timeout(Duration::from_millis(50), &mut commit).await.is_err(),
6299            "publisher must wait while recovery owns the physical absence proof"
6300        );
6301        drop(proof);
6302        let err = match commit.await {
6303            Ok(_) => panic!("publisher must reject its stale source snapshot after recovery releases"),
6304            Err(err) => err,
6305        };
6306        assert!(matches!(err, Error::DataMovementOverwriteErr(_, _, _)));
6307        assert!(
6308            second_set
6309                .get_object_info(bucket, object, &ObjectOptions::default())
6310                .await
6311                .is_err(),
6312            "stale publication must not create a target owner"
6313        );
6314    }
6315
6316    #[tokio::test]
6317    async fn data_movement_put_defers_publication_locks_until_body_complete() {
6318        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6319        let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6320        let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6321        let store =
6322            Arc::new(new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await);
6323        let bucket = "commit-late-data-movement-put";
6324        let object = "slow-body.bin";
6325        for set in [&first_set, &second_set] {
6326            set.make_bucket(bucket, &MakeBucketOptions::default())
6327                .await
6328                .expect("commit-late PUT bucket should be created");
6329        }
6330
6331        let payload = b"data movement body".to_vec();
6332        let mut source_body = PutObjReader::from_vec(payload.clone());
6333        first_set
6334            .put_object(bucket, object, &mut source_body, &ObjectOptions::default())
6335            .await
6336            .expect("data-movement source should be written");
6337        let source = first_set
6338            .get_object_info(bucket, object, &ObjectOptions::default())
6339            .await
6340            .expect("data-movement source should be readable");
6341        let publication = store
6342            .acquire_remote_tuple_publication_fence(bucket, 0, &source, true)
6343            .await
6344            .expect("decommission should capture its publication capability");
6345
6346        let payload_len = i64::try_from(payload.len()).expect("test payload length should fit in i64");
6347        let (poll_started_tx, poll_started_rx) = tokio::sync::oneshot::channel();
6348        let (release_tx, release_rx) = tokio::sync::oneshot::channel();
6349        let slow_body = CommitLateBodyReader {
6350            inner: Cursor::new(payload),
6351            poll_started: Some(poll_started_tx),
6352            release: Some(release_rx),
6353        };
6354        let target_set = Arc::clone(&store.pools[1]);
6355        let target_opts = ObjectOptions {
6356            data_movement: true,
6357            src_pool_idx: 0,
6358            http_preconditions: Some(crate::data_movement::data_movement_target_precondition()),
6359            mod_time: source.mod_time,
6360            preserve_etag: source.etag.clone(),
6361            user_defined: source.user_defined.as_ref().clone(),
6362            ..Default::default()
6363        };
6364        let worker = tokio::spawn(async move {
6365            let hash_reader = rustfs_rio::HashReader::from_stream(slow_body, payload_len, payload_len, None, None, false)
6366                .expect("slow body should produce a valid hash reader");
6367            let mut reader = PutObjReader::new(hash_reader);
6368            target_set
6369                .put_object_with_old_current_size_for_data_movement(bucket, object, &mut reader, &target_opts, publication)
6370                .await
6371        });
6372
6373        tokio::time::timeout(Duration::from_secs(5), poll_started_rx)
6374            .await
6375            .expect("target PUT should start polling its body")
6376            .expect("target PUT should retain the poll observer");
6377        let source_head = tokio::time::timeout(
6378            Duration::from_millis(250),
6379            first_set.get_object_info(bucket, object, &ObjectOptions::default()),
6380        )
6381        .await
6382        .expect("same-key HEAD must not wait behind body staging")
6383        .expect("the source must remain readable during body staging");
6384        assert_eq!(source_head.etag, source.etag);
6385        let target_head = tokio::time::timeout(
6386            Duration::from_millis(250),
6387            second_set.get_object_info(bucket, object, &ObjectOptions::default()),
6388        )
6389        .await
6390        .expect("target HEAD must not wait behind body staging")
6391        .expect_err("the target must remain unpublished before body EOF");
6392        assert!(is_err_object_not_found(&target_head) || is_err_version_not_found(&target_head));
6393
6394        release_tx
6395            .send(())
6396            .expect("target PUT should still be waiting on the body gate");
6397        worker
6398            .await
6399            .expect("target PUT task should join")
6400            .expect("target PUT should commit after body EOF");
6401        second_set
6402            .get_object_info(bucket, object, &ObjectOptions::default())
6403            .await
6404            .expect("target metadata should be visible after the commit boundary");
6405    }
6406
6407    #[test]
6408    fn materialized_copy_drops_remote_tuple_and_free_version_compatibility_keys() {
6409        let protected_suffixes = [
6410            rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
6411            rustfs_utils::http::SUFFIX_TRANSITIONED_OBJECTNAME,
6412            rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_ID,
6413            rustfs_utils::http::SUFFIX_TRANSITIONED_VERSION_STATE,
6414            rustfs_utils::http::SUFFIX_TRANSITION_TIER,
6415            rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6416            rustfs_utils::http::SUFFIX_TRANSITION_TRANSACTION_ID,
6417            rustfs_utils::http::SUFFIX_FREE_VERSION,
6418            rustfs_utils::http::SUFFIX_TIER_FV_ID,
6419            rustfs_utils::http::SUFFIX_TIER_FV_MARKER,
6420        ];
6421        let mut source = HashMap::from([("content-type".to_string(), "application/octet-stream".to_string())]);
6422        for suffix in protected_suffixes {
6423            rustfs_utils::http::metadata_compat::insert_str(&mut source, suffix, format!("source-{suffix}"));
6424        }
6425        // Compatibility readers are case-insensitive, so the scrubber must
6426        // remove non-canonical casing as well as both canonical prefixes.
6427        source.insert(
6428            format!(
6429                "{}{}",
6430                rustfs_utils::http::MINIO_INTERNAL_PREFIX.to_ascii_uppercase(),
6431                rustfs_utils::http::SUFFIX_TRANSITION_STATUS.to_ascii_uppercase()
6432            ),
6433            "complete".to_string(),
6434        );
6435
6436        let copied = materialized_copy_user_defined(&source);
6437
6438        assert_eq!(copied.get("content-type").map(String::as_str), Some("application/octet-stream"));
6439        for suffix in protected_suffixes {
6440            assert!(
6441                !rustfs_utils::http::metadata_compat::contains_key_str(&copied, suffix),
6442                "materialized copy retained protected internal suffix {suffix}"
6443            );
6444        }
6445    }
6446
6447    #[test]
6448    fn remote_tuple_publication_target_is_exact_legacy_compatible_and_ordinary_zero_cost() {
6449        let mut ordinary_metadata = HashMap::new();
6450        rustfs_utils::http::metadata_compat::insert_str(
6451            &mut ordinary_metadata,
6452            rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6453            "malformed-but-irrelevant".to_string(),
6454        );
6455        let ordinary = ObjectInfo {
6456            user_defined: Arc::new(ordinary_metadata),
6457            ..Default::default()
6458        };
6459        assert!(
6460            remote_tuple_publication_target(&ordinary)
6461                .expect("ordinary objects must not parse or lock tier metadata")
6462                .is_none()
6463        );
6464
6465        let identity = [0xabu8; 32];
6466        let mut exact_metadata = HashMap::new();
6467        rustfs_utils::http::metadata_compat::insert_str(
6468            &mut exact_metadata,
6469            rustfs_utils::http::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6470            rustfs_utils::crypto::hex(identity),
6471        );
6472        let transitioned = ObjectInfo {
6473            transitioned_object: TransitionedObject {
6474                status: rustfs_filemeta::TRANSITION_COMPLETE.to_string(),
6475                name: "remote/object".to_string(),
6476                tier: "WARM".to_string(),
6477                ..Default::default()
6478            },
6479            user_defined: Arc::new(exact_metadata),
6480            ..Default::default()
6481        };
6482        let (tier, recorded_identity) = remote_tuple_publication_target(&transitioned)
6483            .expect("exact transitioned metadata should parse")
6484            .expect("transitioned object should require a publication fence");
6485        assert_eq!(tier, "WARM");
6486        assert_eq!(recorded_identity, Some(identity));
6487
6488        let legacy = ObjectInfo {
6489            user_defined: Arc::new(HashMap::new()),
6490            ..transitioned.clone()
6491        };
6492        assert_eq!(
6493            remote_tuple_publication_target(&legacy)
6494                .expect("legacy transitioned metadata should remain compatible")
6495                .map(|(_, identity)| identity),
6496            Some(None)
6497        );
6498
6499        let mut malformed = transitioned;
6500        malformed.transitioned_object.tier.clear();
6501        assert!(remote_tuple_publication_target(&malformed).is_err());
6502    }
6503
6504    #[test]
6505    fn publication_source_match_rejects_new_physical_generation_with_same_logical_identity() {
6506        let expected = ObjectInfo {
6507            data_dir: Some(Uuid::new_v4()),
6508            ..Default::default()
6509        };
6510        assert!(remote_tuple_publication_object_source_matches(&expected, &expected));
6511
6512        let current = ObjectInfo {
6513            data_dir: Some(Uuid::new_v4()),
6514            ..expected.clone()
6515        };
6516        assert!(
6517            crate::data_movement::is_equivalent_data_movement_object_identity(&expected, &current, true, true),
6518            "the logical target-equivalence check deliberately ignores target data-dir allocation"
6519        );
6520        assert!(
6521            !remote_tuple_publication_object_source_matches(&expected, &current),
6522            "publication revalidation must additionally bind the physical source generation"
6523        );
6524    }
6525
6526    #[test]
6527    fn exact_delete_capacity_target_requires_matching_namespace_and_object_identity() {
6528        let source = ObjectInfo {
6529            bucket: "bucket".to_string(),
6530            name: "directory/".to_string(),
6531            version_id: Some(Uuid::from_u128(1)),
6532            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
6533            size: 10,
6534            etag: Some("source-etag".to_string()),
6535            ..Default::default()
6536        };
6537        let target = ObjectInfo {
6538            name: rustfs_utils::path::encode_dir_object(&source.name),
6539            ..source.clone()
6540        };
6541        assert!(ECStore::is_equivalent_decommission_capacity_target(&source, &target));
6542
6543        let mismatched_identity = ObjectInfo {
6544            etag: Some("different-etag".to_string()),
6545            ..target.clone()
6546        };
6547        assert!(!ECStore::is_equivalent_decommission_capacity_target(&source, &mismatched_identity));
6548
6549        let wrong_bucket = ObjectInfo {
6550            bucket: "other-bucket".to_string(),
6551            ..target
6552        };
6553        assert!(!ECStore::is_equivalent_decommission_capacity_target(&source, &wrong_bucket));
6554    }
6555
6556    fn publication_part(number: usize) -> ObjectPartInfo {
6557        ObjectPartInfo {
6558            number,
6559            size: 100 + number,
6560            actual_size: i64::try_from(200 + number).expect("test part size should fit in i64"),
6561            etag: format!("part-etag-{number}"),
6562            ..Default::default()
6563        }
6564    }
6565
6566    fn publication_source(parts: Vec<ObjectPartInfo>) -> ObjectInfo {
6567        ObjectInfo {
6568            data_dir: Some(Uuid::new_v4()),
6569            version_id: Some(Uuid::new_v4()),
6570            size: 4096,
6571            actual_size: 4096,
6572            etag: Some("object-etag".to_string()),
6573            checksum: Some(Bytes::from_static(b"object-checksum")),
6574            mod_time: Some(time::OffsetDateTime::UNIX_EPOCH),
6575            parts: Arc::new(parts),
6576            ..Default::default()
6577        }
6578    }
6579
6580    #[test]
6581    fn publication_parts_match_is_order_independent_and_bijective() {
6582        let ordered = vec![publication_part(1), publication_part(2), publication_part(3)];
6583        let mut reversed = ordered.clone();
6584        reversed.reverse();
6585
6586        assert!(remote_tuple_publication_parts_match(&[], &[]));
6587        assert!(remote_tuple_publication_parts_match(&ordered, &ordered));
6588        assert!(remote_tuple_publication_parts_match(&ordered, &reversed));
6589        assert!(!remote_tuple_publication_parts_match(&ordered[..2], &ordered));
6590
6591        let expected_duplicate = vec![publication_part(1), publication_part(1)];
6592        let current_unique = vec![publication_part(1), publication_part(2)];
6593        assert!(
6594            !remote_tuple_publication_parts_match(&expected_duplicate, &current_unique),
6595            "two expected entries must not reuse the same current part"
6596        );
6597        assert!(!remote_tuple_publication_parts_match(&current_unique, &expected_duplicate));
6598        assert!(!remote_tuple_publication_parts_match(&expected_duplicate, &expected_duplicate));
6599    }
6600
6601    #[test]
6602    fn publication_parts_match_preserves_exact_part_identity_contract() {
6603        let expected = vec![publication_part(1)];
6604
6605        let mut different_number = expected.clone();
6606        different_number[0].number = 2;
6607        assert!(!remote_tuple_publication_parts_match(&expected, &different_number));
6608
6609        let mut different_size = expected.clone();
6610        different_size[0].size += 1;
6611        assert!(!remote_tuple_publication_parts_match(&expected, &different_size));
6612
6613        let mut different_actual_size = expected.clone();
6614        different_actual_size[0].actual_size += 1;
6615        assert!(!remote_tuple_publication_parts_match(&expected, &different_actual_size));
6616
6617        let mut different_etag = expected.clone();
6618        different_etag[0].etag.push_str("-changed");
6619        assert!(!remote_tuple_publication_parts_match(&expected, &different_etag));
6620
6621        let mut ignored_fields = expected.clone();
6622        ignored_fields[0].index = Some(Bytes::from_static(b"different-index"));
6623        ignored_fields[0].checksums = Some(HashMap::from([("CRC32C".to_string(), "different".to_string())]));
6624        ignored_fields[0].mod_time = Some(time::OffsetDateTime::UNIX_EPOCH);
6625        assert!(
6626            remote_tuple_publication_parts_match(&expected, &ignored_fields),
6627            "the publication fence must retain its existing checksum/index/mod-time compatibility contract"
6628        );
6629
6630        let zero_actual_size = vec![ObjectPartInfo {
6631            actual_size: 0,
6632            ..publication_part(1)
6633        }];
6634        assert!(
6635            !remote_tuple_publication_parts_match(&zero_actual_size, &expected),
6636            "the publication fence compares raw part actual sizes without the broader comparator's fallback"
6637        );
6638    }
6639
6640    #[test]
6641    fn publication_source_match_rejects_object_etag_and_identity_mutations() {
6642        let expected = publication_source(vec![publication_part(1)]);
6643        assert!(remote_tuple_publication_object_source_matches(&expected, &expected));
6644
6645        let mut current = expected.clone();
6646        current.etag = Some("changed-object-etag".to_string());
6647        assert!(!remote_tuple_publication_object_source_matches(&expected, &current));
6648
6649        let mut current = expected.clone();
6650        current.checksum = Some(Bytes::from_static(b"changed-checksum"));
6651        assert!(!remote_tuple_publication_object_source_matches(&expected, &current));
6652
6653        let mut current = expected.clone();
6654        current.actual_size += 1;
6655        assert!(!remote_tuple_publication_object_source_matches(&expected, &current));
6656
6657        let mut current = expected.clone();
6658        current.data_dir = Some(Uuid::new_v4());
6659        assert!(!remote_tuple_publication_object_source_matches(&expected, &current));
6660
6661        let mut current = expected.clone();
6662        current.version_id = Some(Uuid::new_v4());
6663        assert!(!remote_tuple_publication_object_source_matches(&expected, &current));
6664
6665        let mut current = expected.clone();
6666        current.mod_time = current.mod_time.map(|mod_time| mod_time + time::Duration::SECOND);
6667        assert!(!remote_tuple_publication_object_source_matches(&expected, &current));
6668    }
6669
6670    #[test]
6671    fn publication_source_match_preserves_effective_actual_size_compatibility() {
6672        let expected = publication_source(vec![publication_part(1)]);
6673        let current = ObjectInfo {
6674            actual_size: 0,
6675            ..expected.clone()
6676        };
6677
6678        assert_eq!(
6679            expected.get_actual_size().expect("expected size should be valid"),
6680            current.get_actual_size().expect("current size should be valid")
6681        );
6682        assert!(remote_tuple_publication_object_source_matches(&expected, &current));
6683    }
6684
6685    #[test]
6686    fn publication_source_match_rejects_duplicate_parts_through_the_full_fence() {
6687        let expected = publication_source(vec![publication_part(1), publication_part(1)]);
6688        let current = ObjectInfo {
6689            parts: Arc::new(vec![publication_part(1), publication_part(2)]),
6690            ..expected.clone()
6691        };
6692
6693        assert!(
6694            !remote_tuple_publication_object_source_matches(&expected, &current),
6695            "the full source fence must reject the original non-bijective false-positive"
6696        );
6697    }
6698
6699    #[test]
6700    fn publication_source_match_handles_10_000_reversed_parts_without_payload_cloning() {
6701        let parts = Arc::new((1..=10_000).map(publication_part).collect::<Vec<_>>());
6702        let mut reversed = parts.as_ref().clone();
6703        reversed.reverse();
6704        let expected = publication_source(Vec::new());
6705        let expected = ObjectInfo {
6706            parts: Arc::clone(&parts),
6707            ..expected
6708        };
6709        let current = ObjectInfo {
6710            parts: Arc::new(reversed),
6711            ..expected.clone()
6712        };
6713        let expected_parts = Arc::clone(&expected.parts);
6714
6715        assert!(remote_tuple_publication_object_source_matches(&expected, &current));
6716        assert!(Arc::ptr_eq(&expected.parts, &expected_parts));
6717    }
6718
6719    #[tokio::test]
6720    async fn publication_commit_guard_rejects_an_etag_only_source_change() {
6721        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
6722        let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6723        let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
6724        let store =
6725            Arc::new(new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await);
6726        let bucket = "publication-etag-source-change";
6727        let object = "source.bin";
6728        for set in [&first_set, &second_set] {
6729            set.make_bucket(bucket, &MakeBucketOptions::default())
6730                .await
6731                .expect("publication ETag test bucket should be created");
6732        }
6733
6734        let mut source_body = PutObjReader::from_vec(b"source body".to_vec());
6735        first_set
6736            .put_object(bucket, object, &mut source_body, &ObjectOptions::default())
6737            .await
6738            .expect("publication source should be written");
6739        let source = first_set
6740            .get_object_info(
6741                bucket,
6742                object,
6743                &ObjectOptions {
6744                    include_part_checksums: true,
6745                    ..Default::default()
6746                },
6747            )
6748            .await
6749            .expect("publication source should be readable");
6750        let publication = store
6751            .acquire_remote_tuple_publication_fence(bucket, 0, &source, false)
6752            .await
6753            .expect("the source snapshot should produce a publication capability");
6754
6755        let changed = first_set
6756            .put_object_metadata(
6757                bucket,
6758                object,
6759                &ObjectOptions {
6760                    eval_metadata: Some(HashMap::from([("etag".to_string(), "changed-etag".to_string())])),
6761                    ..Default::default()
6762                },
6763            )
6764            .await
6765            .expect("the source ETag should be updated in place");
6766        assert_ne!(source.etag, changed.etag);
6767        let unchanged_except_etag = ObjectInfo {
6768            etag: source.etag.clone(),
6769            ..changed.clone()
6770        };
6771        assert!(
6772            remote_tuple_publication_object_source_matches(&source, &unchanged_except_etag),
6773            "the persisted source update fixture must differ only by object ETag"
6774        );
6775
6776        let encoded = encode_dir_object(object);
6777        let err = match publication.into_commit_guard(1, bucket, &encoded).await {
6778            Ok(_) => panic!("the publication guard must reject an ETag-only source change"),
6779            Err(err) => err,
6780        };
6781        assert!(matches!(err, Error::DataMovementOverwriteErr(_, _, _)));
6782        assert!(
6783            second_set
6784                .get_object_info(bucket, object, &ObjectOptions::default())
6785                .await
6786                .is_err(),
6787            "a rejected publication must not create a target object"
6788        );
6789    }
6790
6791    #[tokio::test]
6792    async fn generic_data_movement_put_rejects_transition_ownership_without_capability() {
6793        let (_dirs, set) = make_local_set_disks(4, 2).await;
6794        let bucket = "unfenced-transition-publication";
6795        set.make_bucket(bucket, &MakeBucketOptions::default())
6796            .await
6797            .expect("unfenced publication bucket should be created");
6798        let mut metadata = HashMap::new();
6799        rustfs_utils::http::metadata_compat::insert_str(
6800            &mut metadata,
6801            rustfs_utils::http::SUFFIX_TRANSITION_STATUS,
6802            TRANSITION_COMPLETE.to_string(),
6803        );
6804        let mut body = PutObjReader::from_vec(b"must not publish".to_vec());
6805        let err = set
6806            .put_object(
6807                bucket,
6808                "unfenced.bin",
6809                &mut body,
6810                &ObjectOptions {
6811                    data_movement: true,
6812                    user_defined: metadata,
6813                    ..Default::default()
6814                },
6815            )
6816            .await
6817            .expect_err("transition ownership must require the typed publication entry point");
6818        assert!(err.to_string().contains("publication capability"));
6819    }
6820
6821    #[test]
6822    fn select_snapshot_version_matching_normalizes_null_and_uuid_forms() {
6823        let nil = Uuid::nil();
6824        for actual in [None, Some(nil)] {
6825            assert!(select_snapshot_version_matches(actual, "null"));
6826            assert!(select_snapshot_version_matches(actual, "NULL"));
6827            assert!(select_snapshot_version_matches(actual, &nil.to_string()));
6828        }
6829
6830        let version = Uuid::new_v4();
6831        assert!(select_snapshot_version_matches(Some(version), &version.to_string().to_uppercase()));
6832        assert!(!select_snapshot_version_matches(Some(version), "null"));
6833        assert!(!select_snapshot_version_matches(Some(version), "not-a-version"));
6834    }
6835
6836    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6837    async fn select_snapshot_reader_rolls_back_bytes_when_lock_is_lost_during_poll() {
6838        let (lease, signal, client) = refresh_failure_test_lease("select-snapshot-lock-loss").await;
6839
6840        let (poll_started_tx, poll_started_rx) = tokio::sync::oneshot::channel();
6841        let (resume_tx, resume_rx) = std::sync::mpsc::channel();
6842        let release_client = Arc::clone(&client);
6843        let release_lease = Arc::clone(&lease);
6844        let release_signal = Arc::clone(&signal);
6845        let release_task = tokio::spawn(async move {
6846            poll_started_rx.await.expect("reader poll should start");
6847            release_client.reject_refreshes();
6848            release_client
6849                .wait_for_rejected_refresh(Duration::from_secs(5))
6850                .await
6851                .expect("refresh rejection should be observed");
6852            tokio::time::timeout(Duration::from_secs(5), release_signal.notified())
6853                .await
6854                .expect("heartbeat should observe the rejected refresh");
6855            tokio::time::timeout(Duration::from_secs(5), async {
6856                while !release_lease.is_lost() {
6857                    tokio::task::yield_now().await;
6858                }
6859            })
6860            .await
6861            .expect("snapshot monitor should publish lock loss");
6862            resume_tx.send(()).expect("reader poll should still be waiting");
6863        });
6864
6865        let mut reader = SelectObjectSnapshotReader {
6866            inner: Box::new(WaitForLockLossReader {
6867                inner: Cursor::new(b"must-not-escape".to_vec()),
6868                poll_started: Some(poll_started_tx),
6869                resume: ParkingMutex::new(resume_rx),
6870            }),
6871            lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6872            lease,
6873        };
6874        let mut output = Vec::new();
6875        let error = reader
6876            .read_to_end(&mut output)
6877            .await
6878            .expect_err("a read that loses its lease must fail");
6879        release_task.await.expect("backend release task should not panic");
6880
6881        assert!(signal.is_lost(), "heartbeat should report the rejected refresh");
6882        assert!(output.is_empty(), "bytes read during the failed poll must be rolled back");
6883        assert_eq!(error.kind(), std::io::ErrorKind::Other);
6884    }
6885
6886    #[tokio::test(flavor = "current_thread")]
6887    async fn select_snapshot_reader_checks_guards_before_monitor_runs() {
6888        let (guard, signal, client) = refresh_failure_test_guard("select-snapshot-eof-fence").await;
6889        client.reject_refreshes();
6890        client
6891            .wait_for_rejected_refresh(Duration::from_secs(5))
6892            .await
6893            .expect("refresh rejection should be observed");
6894        tokio::time::timeout(Duration::from_secs(5), signal.notified())
6895            .await
6896            .expect("heartbeat should observe the rejected refresh");
6897
6898        let lease = Arc::new(SelectObjectSnapshotLease::new(vec![guard]));
6899        assert!(!lease.is_lost(), "current-thread monitor must not run before the synchronous read");
6900        let mut reader = SelectObjectSnapshotReader {
6901            inner: Box::new(Cursor::new(b"old-generation".to_vec())),
6902            lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6903            lease,
6904        };
6905        let mut output = Vec::new();
6906
6907        let error = reader
6908            .read_to_end(&mut output)
6909            .await
6910            .expect_err("EOF fence must reject a lease lost before its monitor is scheduled");
6911
6912        assert!(output.is_empty(), "bytes from a known-lost snapshot must not escape");
6913        assert_eq!(error.kind(), std::io::ErrorKind::Other);
6914    }
6915
6916    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
6917    async fn select_snapshot_lock_loss_is_broadcast_to_remaining_pending_readers() {
6918        let (first_guard, first_signal, first_client) = refresh_failure_test_guard("select-snapshot-pending-first-lock").await;
6919        let (second_guard, second_signal, second_client) =
6920            refresh_failure_test_guard("select-snapshot-pending-second-lock").await;
6921        let lease = Arc::new(SelectObjectSnapshotLease::new(vec![first_guard, second_guard]));
6922        let dropped_reader = SelectObjectSnapshotReader {
6923            inner: Box::new(PermanentlyPendingReader { poll_started: None }),
6924            lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6925            lease: Arc::clone(&lease),
6926        };
6927        drop(dropped_reader);
6928
6929        let (first_started_tx, first_started_rx) = tokio::sync::oneshot::channel();
6930        let (second_started_tx, second_started_rx) = tokio::sync::oneshot::channel();
6931        let mut first_reader = SelectObjectSnapshotReader {
6932            inner: Box::new(PermanentlyPendingReader {
6933                poll_started: Some(first_started_tx),
6934            }),
6935            lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6936            lease: Arc::clone(&lease),
6937        };
6938        let mut second_reader = SelectObjectSnapshotReader {
6939            inner: Box::new(PermanentlyPendingReader {
6940                poll_started: Some(second_started_tx),
6941            }),
6942            lock_loss_wake: SelectObjectSnapshotLockLossWake::new(lease.subscribe_lock_loss()),
6943            lease,
6944        };
6945        let first_read_task = tokio::spawn(async move {
6946            let mut byte = [0_u8; 1];
6947            first_reader.read(&mut byte).await
6948        });
6949        let second_read_task = tokio::spawn(async move {
6950            let mut byte = [0_u8; 1];
6951            second_reader.read(&mut byte).await
6952        });
6953
6954        first_started_rx.await.expect("first inner reader should reach Poll::Pending");
6955        second_started_rx
6956            .await
6957            .expect("second inner reader should reach Poll::Pending");
6958        second_client.reject_refreshes();
6959        second_client
6960            .wait_for_rejected_refresh(Duration::from_secs(5))
6961            .await
6962            .expect("refresh rejection should be observed");
6963        let (first_result, second_result) = tokio::join!(
6964            tokio::time::timeout(Duration::from_secs(5), first_read_task),
6965            tokio::time::timeout(Duration::from_secs(5), second_read_task),
6966        );
6967        for result in [first_result, second_result] {
6968            let error = result
6969                .expect("lock loss should wake every reader whose inner I/O remains pending")
6970                .expect("reader task should not panic")
6971                .expect_err("lost snapshot lease must fail every pending read");
6972            assert_eq!(error.kind(), std::io::ErrorKind::Other);
6973        }
6974
6975        assert!(!first_client.refreshes_rejected());
6976        assert!(!first_signal.is_lost());
6977        assert!(second_signal.is_lost());
6978    }
6979
6980    #[test]
6981    fn tier_delete_entry_is_prepared_and_prefix_legacy_fails_closed() {
6982        let identity = [9_u8; 32];
6983        let mut metadata = HashMap::new();
6984        rustfs_utils::http::metadata_compat::insert_str(
6985            &mut metadata,
6986            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
6987            rustfs_utils::crypto::hex(identity),
6988        );
6989        let version_id = Uuid::from_u128(1);
6990        let data_dir = Uuid::from_u128(2);
6991        let source = ObjectInfo {
6992            bucket: "bucket".to_string(),
6993            name: "object".to_string(),
6994            version_id: Some(version_id),
6995            data_dir: Some(data_dir),
6996            user_defined: Arc::new(metadata),
6997            transitioned_object: TransitionedObject {
6998                name: "remote/object".to_string(),
6999                version_id: "remote-version".to_string(),
7000                tier: "WARM".to_string(),
7001                status: TRANSITION_COMPLETE.to_string(),
7002                ..Default::default()
7003            },
7004            transition_version_state: rustfs_filemeta::TransitionVersionState::Exact,
7005            ..Default::default()
7006        };
7007        let entry = build_tier_delete_journal_entry(
7008            "bucket",
7009            "object",
7010            &ObjectOptions {
7011                version_id: Some(version_id.to_string()),
7012                versioned: true,
7013                ..Default::default()
7014            },
7015            &source,
7016        )
7017        .expect("transition source should produce a journal entry")
7018        .expect("completed transition should be journaled");
7019
7020        assert_eq!(entry.state, TierDeleteJournalState::Prepared);
7021        assert_eq!(entry.backend_identity, Some(identity));
7022        let data_dir_string = data_dir.to_string();
7023        assert_eq!(
7024            entry.source.as_ref().and_then(|source| source.data_dir.as_deref()),
7025            Some(data_dir_string.as_str())
7026        );
7027
7028        let mut unknown = source.clone();
7029        unknown.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
7030        assert!(
7031            build_tier_delete_journal_entry(
7032                "bucket",
7033                "object",
7034                &ObjectOptions {
7035                    delete_prefix: true,
7036                    versioned: true,
7037                    ..Default::default()
7038                },
7039                &unknown,
7040            )
7041            .expect_err("recursive prefix delete must fail closed for legacy Unknown metadata")
7042            .to_string()
7043            .contains("legacy Unknown")
7044        );
7045
7046        assert!(
7047            build_tier_delete_journal_entry(
7048                "bucket",
7049                "object",
7050                &ObjectOptions {
7051                    delete_prefix: true,
7052                    delete_prefix_object: true,
7053                    versioned: true,
7054                    lifecycle_delete_all: Some(crate::object_api::LifecycleDeleteAllRequest {
7055                        version_id: Some(version_id),
7056                        delete_marker: false,
7057                        action: rustfs_scanner_metrics::metrics::IlmAction::DeleteAllVersionsAction,
7058                        rule_id: "delete-all".to_string(),
7059                        phase: crate::object_api::LifecycleDeleteAllPhase::Preflight,
7060                    }),
7061                    ..Default::default()
7062                },
7063                &unknown,
7064            )
7065            .expect("lifecycle exact-object delete-all may use its free-version fallback")
7066            .is_none()
7067        );
7068
7069        let mut identity_less = source;
7070        identity_less.user_defined = Arc::new(HashMap::new());
7071        assert!(
7072            build_tier_delete_journal_entry(
7073                "bucket",
7074                "object",
7075                &ObjectOptions {
7076                    delete_prefix: true,
7077                    versioned: true,
7078                    ..Default::default()
7079                },
7080                &identity_less,
7081            )
7082            .expect_err("recursive prefix delete must fail closed without destination identity")
7083            .to_string()
7084            .contains("requires a stable transitioned source")
7085        );
7086    }
7087
7088    #[test]
7089    fn tier_delete_walk_only_accepts_explicit_limit_cancellation() {
7090        let cancelled = Error::OperationCanceled;
7091        assert!(tier_delete_walk_cancellation_is_expected(true, true, &cancelled));
7092        assert!(!tier_delete_walk_cancellation_is_expected(false, true, &cancelled));
7093        assert!(!tier_delete_walk_cancellation_is_expected(true, false, &cancelled));
7094        assert!(!tier_delete_walk_cancellation_is_expected(true, true, &Error::other("scan failed")));
7095    }
7096
7097    #[test]
7098    fn tier_delete_walk_results_prioritize_real_errors_over_cancellation() {
7099        let err = combine_tier_delete_walk_results([Err(Error::OperationCanceled), Ok(()), Err(StorageError::FileAccessDenied)])
7100            .expect_err("a real walk error must not be hidden by earlier cancellation");
7101        assert_eq!(err, StorageError::FileAccessDenied);
7102
7103        assert_eq!(
7104            combine_tier_delete_walk_results([Ok(()), Err(Error::OperationCanceled)])
7105                .expect_err("cancellation must remain visible when there is no real error"),
7106            Error::OperationCanceled
7107        );
7108    }
7109
7110    impl Drop for BodyCacheHookGuard {
7111        fn drop(&mut self) {
7112            clear_get_object_body_cache_hook();
7113        }
7114    }
7115
7116    #[test]
7117    fn delete_marker_data_movement_falls_back_when_only_source_pool_has_object() {
7118        let target = select_data_movement_target_pool(Ok(1), 1, true).unwrap();
7119        assert_eq!(target, None);
7120    }
7121
7122    #[test]
7123    fn delete_marker_data_movement_falls_back_when_version_does_not_exist_yet() {
7124        let err = StorageError::ObjectNotFound("bucket".to_string(), "object".to_string());
7125        let target = select_data_movement_target_pool(Err(err), 1, true).unwrap();
7126        assert_eq!(target, None);
7127    }
7128
7129    #[test]
7130    fn non_delete_marker_data_movement_keeps_existing_pool() {
7131        let target = select_data_movement_target_pool(Ok(0), 1, false).unwrap();
7132        assert_eq!(target, Some(0));
7133    }
7134
7135    #[test]
7136    fn equivalent_data_movement_delete_marker_requires_same_version_and_mod_time() {
7137        let version_id = Uuid::nil();
7138        let mod_time = OffsetDateTime::UNIX_EPOCH;
7139        let source = ObjectInfo {
7140            version_id: Some(version_id),
7141            delete_marker: true,
7142            mod_time: Some(mod_time),
7143            ..Default::default()
7144        };
7145        let target = source.clone();
7146
7147        assert!(is_equivalent_data_movement_delete_marker(&source, &target));
7148
7149        let mismatched = ObjectInfo {
7150            mod_time: Some(mod_time + Duration::from_secs(1)),
7151            ..target
7152        };
7153        assert!(!is_equivalent_data_movement_delete_marker(&source, &mismatched));
7154    }
7155
7156    #[test]
7157    fn equivalent_data_movement_delete_marker_accepts_distinct_local_free_version_ids() {
7158        let mut source = ObjectInfo {
7159            version_id: Some(Uuid::from_u128(1)),
7160            delete_marker: true,
7161            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7162            ..Default::default()
7163        };
7164        rustfs_utils::http::insert_str(
7165            Arc::make_mut(&mut source.user_defined),
7166            rustfs_utils::http::SUFFIX_TIER_FV_ID,
7167            Uuid::from_u128(2).to_string(),
7168        );
7169        let mut target = source.clone();
7170        rustfs_utils::http::insert_str(
7171            Arc::make_mut(&mut target.user_defined),
7172            rustfs_utils::http::SUFFIX_TIER_FV_ID,
7173            Uuid::from_u128(3).to_string(),
7174        );
7175
7176        assert!(is_equivalent_data_movement_delete_marker(&source, &target));
7177
7178        Arc::make_mut(&mut target.user_defined).insert(
7179            format!("{}{}", rustfs_utils::http::MINIO_INTERNAL_PREFIX, rustfs_utils::http::SUFFIX_TIER_FV_ID),
7180            Uuid::from_u128(4).to_string(),
7181        );
7182        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7183    }
7184
7185    #[test]
7186    fn equivalent_data_movement_delete_marker_accepts_replication_alias_expansion() {
7187        let key = format!(
7188            "{}{}",
7189            rustfs_utils::http::MINIO_INTERNAL_PREFIX,
7190            rustfs_utils::http::SUFFIX_REPLICATION_STATUS
7191        );
7192        let timestamp_key = format!(
7193            "{}{}",
7194            rustfs_utils::http::MINIO_INTERNAL_PREFIX,
7195            rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP
7196        );
7197        let source = ObjectInfo {
7198            version_id: Some(Uuid::from_u128(1)),
7199            delete_marker: true,
7200            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7201            user_defined: Arc::new(HashMap::from([
7202                (key.clone(), "arn=COMPLETED;".to_string()),
7203                (timestamp_key, "1970-01-01T00:00:01Z".to_string()),
7204            ])),
7205            ..Default::default()
7206        };
7207        let mut target = source.clone();
7208        rustfs_utils::http::insert_str(
7209            Arc::make_mut(&mut target.user_defined),
7210            rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
7211            "arn=COMPLETED;".to_string(),
7212        );
7213        rustfs_utils::http::insert_str(
7214            Arc::make_mut(&mut target.user_defined),
7215            rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP,
7216            (OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND).to_string(),
7217        );
7218        assert!(is_equivalent_data_movement_delete_marker(&source, &target));
7219
7220        Arc::make_mut(&mut target.user_defined).insert(key, "arn=FAILED;".to_string());
7221        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7222    }
7223
7224    #[test]
7225    fn equivalent_data_movement_delete_marker_ignores_target_bucket_incarnation_fence() {
7226        let source = ObjectInfo {
7227            version_id: Some(Uuid::from_u128(1)),
7228            delete_marker: true,
7229            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7230            ..Default::default()
7231        };
7232        let mut target = source.clone();
7233        rustfs_utils::http::insert_str(
7234            Arc::make_mut(&mut target.user_defined),
7235            rustfs_utils::http::SUFFIX_BUCKET_INCARNATION_ID,
7236            Uuid::from_u128(2).to_string(),
7237        );
7238
7239        assert!(is_equivalent_data_movement_delete_marker(&source, &target));
7240    }
7241
7242    #[test]
7243    fn data_movement_delete_marker_source_requires_persisted_mod_time() {
7244        let source = ObjectInfo {
7245            delete_marker: true,
7246            ..Default::default()
7247        };
7248        assert!(!is_expected_data_movement_delete_marker_source(&source, None));
7249
7250        let source = ObjectInfo {
7251            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7252            ..source
7253        };
7254        assert!(is_expected_data_movement_delete_marker_source(&source, Some(OffsetDateTime::UNIX_EPOCH)));
7255        assert!(!is_expected_data_movement_delete_marker_source(&source, None));
7256    }
7257
7258    #[test]
7259    fn data_movement_delete_marker_uses_current_source_replication_state() {
7260        let expected_timestamp = OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND;
7261        let timestamp = expected_timestamp.to_string();
7262        let mut metadata = HashMap::new();
7263        rustfs_utils::http::insert_str(
7264            &mut metadata,
7265            rustfs_utils::http::SUFFIX_REPLICA_STATUS,
7266            ReplicationStatusType::Replica.to_string(),
7267        );
7268        rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, timestamp.clone());
7269        rustfs_utils::http::insert_str(&mut metadata, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, timestamp);
7270        rustfs_utils::http::insert_str(
7271            &mut metadata,
7272            rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
7273            "arn=COMPLETED;".to_string(),
7274        );
7275        rustfs_utils::http::insert_str(
7276            &mut metadata,
7277            &format!(
7278                "{}{}",
7279                rustfs_utils::http::SUFFIX_REPLICATION_RESET_ARN_PREFIX,
7280                "arn:minio:replication::TenantA:bucket"
7281            ),
7282            "reset-id".to_string(),
7283        );
7284        rustfs_utils::http::insert_str(
7285            &mut metadata,
7286            &format!(
7287                "{}{}",
7288                rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX,
7289                "arn:minio:replication::TenantA:bucket"
7290            ),
7291            "target-version".to_string(),
7292        );
7293        let source = ObjectInfo {
7294            delete_marker: true,
7295            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7296            replication_status_internal: Some("arn=COMPLETED;".to_string()),
7297            replication_decision: "arn=replicate;".to_string(),
7298            user_defined: Arc::new(metadata),
7299            ..Default::default()
7300        };
7301        let opts = ObjectOptions {
7302            mod_time: source.mod_time,
7303            delete_replication: Some(ReplicationState {
7304                replication_status_internal: Some("arn=PENDING;".to_string()),
7305                ..Default::default()
7306            }),
7307            ..Default::default()
7308        };
7309
7310        let target_opts = current_data_movement_delete_marker_opts(&source, &opts).expect("valid current source state");
7311        let state = target_opts.delete_replication.as_ref().expect("current replication state");
7312        assert_eq!(state.replication_status_internal.as_deref(), Some("arn=COMPLETED;"));
7313        assert_eq!(state.replica_status, crate::bucket::replication::ReplicationStatusType::Replica);
7314        assert_eq!(state.replica_timestamp, Some(expected_timestamp));
7315        assert_eq!(state.replication_timestamp, state.replica_timestamp);
7316        assert_eq!(state.replicate_decision_str, "arn=replicate;");
7317        assert_eq!(
7318            state
7319                .reset_statuses_map
7320                .get("arn:minio:replication::TenantA:bucket")
7321                .map(String::as_str),
7322            Some("reset-id")
7323        );
7324        assert_eq!(
7325            state
7326                .target_delete_marker_version_ids
7327                .get("arn:minio:replication::TenantA:bucket")
7328                .map(String::as_str),
7329            Some("target-version")
7330        );
7331    }
7332
7333    #[test]
7334    fn data_movement_delete_marker_rejects_corrupt_target_version_maps() {
7335        let suffix = format!("{}not-an-arn", rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX);
7336        let mut malformed = HashMap::new();
7337        rustfs_utils::http::insert_str(&mut malformed, &suffix, "target-version".to_string());
7338        let malformed_source = ObjectInfo {
7339            delete_marker: true,
7340            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7341            user_defined: Arc::new(malformed),
7342            ..Default::default()
7343        };
7344        assert!(current_data_movement_delete_marker_opts(&malformed_source, &ObjectOptions::default()).is_none());
7345
7346        let mut conflicted = HashMap::new();
7347        let suffix = format!(
7348            "{}arn:minio:replication::target:bucket",
7349            rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX
7350        );
7351        rustfs_utils::http::insert_str(&mut conflicted, &suffix, "target-version-a".to_string());
7352        conflicted.insert(
7353            format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
7354            "target-version-b".to_string(),
7355        );
7356        let conflicted_source = ObjectInfo {
7357            user_defined: Arc::new(conflicted),
7358            ..malformed_source.clone()
7359        };
7360        assert!(current_data_movement_delete_marker_opts(&conflicted_source, &ObjectOptions::default()).is_none());
7361
7362        let mut over_cap = HashMap::new();
7363        for index in 0..=1_000 {
7364            let suffix = format!(
7365                "{}arn:minio:replication::target:bucket-{index}",
7366                rustfs_utils::http::SUFFIX_REPLICATION_DELETE_MARKER_VERSION_ARN_PREFIX
7367            );
7368            rustfs_utils::http::insert_str(&mut over_cap, &suffix, format!("target-version-{index}"));
7369        }
7370        let over_cap_source = ObjectInfo {
7371            user_defined: Arc::new(over_cap),
7372            ..malformed_source
7373        };
7374        assert!(current_data_movement_delete_marker_opts(&over_cap_source, &ObjectOptions::default()).is_none());
7375    }
7376
7377    #[test]
7378    fn data_movement_delete_marker_normalizes_legacy_missing_replication_timestamps() {
7379        let mut source_metadata = HashMap::new();
7380        rustfs_utils::http::insert_str(
7381            &mut source_metadata,
7382            rustfs_utils::http::SUFFIX_REPLICA_STATUS,
7383            ReplicationStatusType::Replica.to_string(),
7384        );
7385        rustfs_utils::http::insert_str(
7386            &mut source_metadata,
7387            rustfs_utils::http::SUFFIX_REPLICATION_STATUS,
7388            "arn=COMPLETED;".to_string(),
7389        );
7390        let source = ObjectInfo {
7391            delete_marker: true,
7392            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7393            replication_status_internal: Some("arn=COMPLETED;".to_string()),
7394            user_defined: Arc::new(source_metadata),
7395            ..Default::default()
7396        };
7397
7398        let target_opts = current_data_movement_delete_marker_opts(&source, &ObjectOptions::default())
7399            .expect("legacy status-only metadata should remain migratable");
7400        let state = target_opts
7401            .delete_replication
7402            .expect("replication state should be reconstructed");
7403        assert_eq!(state.replica_timestamp, Some(OffsetDateTime::UNIX_EPOCH));
7404        assert_eq!(state.replication_timestamp, Some(OffsetDateTime::UNIX_EPOCH));
7405
7406        let mut target_metadata = (*source.user_defined).clone();
7407        let epoch = OffsetDateTime::UNIX_EPOCH
7408            .format(&time::format_description::well_known::Rfc3339)
7409            .unwrap();
7410        rustfs_utils::http::insert_str(&mut target_metadata, rustfs_utils::http::SUFFIX_REPLICA_TIMESTAMP, epoch.clone());
7411        rustfs_utils::http::insert_str(&mut target_metadata, rustfs_utils::http::SUFFIX_REPLICATION_TIMESTAMP, epoch);
7412        assert!(is_equivalent_data_movement_delete_marker_metadata(&source.user_defined, &target_metadata));
7413    }
7414
7415    #[test]
7416    fn equivalent_data_movement_delete_marker_rejects_metadata_and_replication_mismatch() {
7417        let version_id = Uuid::nil();
7418        let mod_time = OffsetDateTime::UNIX_EPOCH;
7419        let source = ObjectInfo {
7420            version_id: Some(version_id),
7421            delete_marker: true,
7422            mod_time: Some(mod_time),
7423            user_defined: Arc::new(HashMap::from([("x-amz-meta-source".to_string(), "true".to_string())])),
7424            replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()),
7425            version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()),
7426            ..Default::default()
7427        };
7428
7429        let mut target = source.clone();
7430        target.user_defined = Arc::new(HashMap::from([("x-amz-meta-source".to_string(), "false".to_string())]));
7431        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7432
7433        let mut target = source.clone();
7434        target.replication_status_internal = Some("arn:minio:replication:target=FAILED;".to_string());
7435        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7436
7437        let mut target = source.clone();
7438        target.version_purge_status_internal = Some("arn:minio:replication:target=COMPLETE;".to_string());
7439        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7440    }
7441
7442    #[test]
7443    fn equivalent_data_movement_delete_marker_rejects_live_object() {
7444        let source = ObjectInfo {
7445            delete_marker: true,
7446            ..Default::default()
7447        };
7448        let target = ObjectInfo {
7449            delete_marker: false,
7450            ..source.clone()
7451        };
7452
7453        assert!(!is_equivalent_data_movement_delete_marker(&source, &target));
7454    }
7455
7456    #[test]
7457    fn data_movement_delete_marker_resume_accepts_equivalent_target() {
7458        let source = ObjectInfo {
7459            version_id: Some(Uuid::nil()),
7460            delete_marker: true,
7461            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7462            ..Default::default()
7463        };
7464
7465        let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 1)
7466            .expect("equivalent delete marker target should be evaluated");
7467
7468        assert!(should_resume);
7469    }
7470
7471    #[test]
7472    fn data_movement_delete_marker_resume_rejects_source_pool_target() {
7473        let source = ObjectInfo {
7474            version_id: Some(Uuid::nil()),
7475            delete_marker: true,
7476            mod_time: Some(OffsetDateTime::UNIX_EPOCH),
7477            ..Default::default()
7478        };
7479
7480        let should_resume = resolve_data_movement_delete_marker_resume_result(Ok(Some(source.clone())), &source, 0, 0)
7481            .expect("source-pool target should be rejected before target lookup");
7482
7483        assert!(!should_resume);
7484    }
7485
7486    #[test]
7487    fn data_movement_resume_target_prefers_selected_non_source_pool() {
7488        let target_pool_idx = resolve_data_movement_resume_target_pool(2, Some(3), 1);
7489        assert_eq!(target_pool_idx, 2);
7490    }
7491
7492    #[test]
7493    fn data_movement_resume_target_uses_resolved_non_source_pool_when_selected_is_source() {
7494        let target_pool_idx = resolve_data_movement_resume_target_pool(1, Some(3), 1);
7495        assert_eq!(target_pool_idx, 3);
7496        assert!(should_check_data_movement_resume_target(1, target_pool_idx));
7497    }
7498
7499    #[test]
7500    fn data_movement_resume_target_keeps_source_when_no_other_pool_is_available() {
7501        let target_pool_idx = resolve_data_movement_resume_target_pool(1, None, 1);
7502        assert_eq!(target_pool_idx, 1);
7503    }
7504
7505    #[test]
7506    fn data_movement_delete_marker_resume_propagates_target_lookup_error() {
7507        let source = ObjectInfo {
7508            delete_marker: true,
7509            ..Default::default()
7510        };
7511        let result = resolve_data_movement_delete_marker_resume_result(Err(Error::SlowDown), &source, 0, 1);
7512
7513        assert!(matches!(result, Err(Error::SlowDown)));
7514    }
7515
7516    fn tiered_equivalence_source() -> FileInfo {
7517        let version_id = Uuid::nil();
7518        let transition_version_id = Uuid::new_v4();
7519        let mod_time = OffsetDateTime::UNIX_EPOCH;
7520
7521        FileInfo {
7522            version_id: Some(version_id),
7523            size: 1024,
7524            mod_time: Some(mod_time),
7525            checksum: Some(Bytes::from_static(b"checksum")),
7526            transition_status: TRANSITION_COMPLETE.to_string(),
7527            transitioned_objname: "remote/object".to_string(),
7528            transition_tier: "WARM".to_string(),
7529            transition_version_id: Some(transition_version_id),
7530            replication_state_internal: Some(replication_state_to_filemeta(&ReplicationState {
7531                replication_status_internal: Some("arn:minio:replication:target=COMPLETED;".to_string()),
7532                targets: replication_statuses_map("arn:minio:replication:target=COMPLETED;"),
7533                version_purge_status_internal: Some("arn:minio:replication:target=PENDING;".to_string()),
7534                purge_targets: version_purge_statuses_map("arn:minio:replication:target=PENDING;"),
7535                ..Default::default()
7536            })),
7537            metadata: HashMap::from([
7538                ("etag".to_string(), "etag-value".to_string()),
7539                ("x-amz-meta-key".to_string(), "metadata-value".to_string()),
7540                (rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), "tag=value".to_string()),
7541                ("expires".to_string(), "1970-01-01T00:33:20Z".to_string()),
7542            ]),
7543            ..Default::default()
7544        }
7545    }
7546
7547    fn tiered_equivalence_target(source: &FileInfo) -> ObjectInfo {
7548        ObjectInfo::from_file_info(source, "bucket", "object", source.version_id.is_some())
7549    }
7550
7551    #[test]
7552    fn equivalent_data_movement_tiered_object_accepts_matching_persisted_metadata() {
7553        let source = tiered_equivalence_source();
7554        let target = tiered_equivalence_target(&source);
7555
7556        assert!(is_equivalent_data_movement_tiered_object(&source, &target));
7557    }
7558
7559    #[test]
7560    fn tiered_data_movement_source_match_rejects_transition_identity_changes() {
7561        let source = tiered_equivalence_source();
7562        assert!(tiered_data_movement_source_matches(&source, &source).expect("matching source metadata should parse"));
7563
7564        let mut changed_remote = source.clone();
7565        changed_remote.transitioned_objname = "remote/replaced".to_string();
7566        assert!(!tiered_data_movement_source_matches(&source, &changed_remote).expect("changed remote metadata should parse"));
7567
7568        let mut changed_backend = source.clone();
7569        rustfs_utils::http::metadata_compat::insert_str(
7570            &mut changed_backend.metadata,
7571            rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
7572            rustfs_utils::crypto::hex([9; 32]),
7573        );
7574        assert!(!tiered_data_movement_source_matches(&source, &changed_backend).expect("backend metadata should parse"));
7575    }
7576
7577    #[test]
7578    fn equivalent_data_movement_tiered_object_uses_logical_compressed_and_encrypted_sizes() {
7579        let mut compressed = tiered_equivalence_source();
7580        compressed.size = 600;
7581        rustfs_utils::http::insert_str(&mut compressed.metadata, rustfs_utils::http::SUFFIX_COMPRESSION, "S2".to_string());
7582        rustfs_utils::http::insert_str(&mut compressed.metadata, rustfs_utils::http::SUFFIX_ACTUAL_SIZE, "1024".to_string());
7583        let compressed_target = tiered_equivalence_target(&compressed);
7584        assert!(is_equivalent_data_movement_tiered_object(&compressed, &compressed_target));
7585
7586        let mut encrypted = tiered_equivalence_source();
7587        encrypted.size = 640;
7588        encrypted.metadata.insert(
7589            rustfs_utils::http::object_encryption_keys::INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(),
7590            "key-id".to_string(),
7591        );
7592        encrypted.metadata.insert(
7593            rustfs_utils::http::object_encryption_keys::INTERNAL_ENCRYPTION_ORIGINAL_SIZE_HEADER.to_string(),
7594            "1024".to_string(),
7595        );
7596        let encrypted_target = tiered_equivalence_target(&encrypted);
7597        assert!(is_equivalent_data_movement_tiered_object(&encrypted, &encrypted_target));
7598    }
7599
7600    #[test]
7601    fn equivalent_data_movement_tiered_object_accepts_transition_alias_expansion() {
7602        let mut source = tiered_equivalence_source();
7603        let suffix = rustfs_utils::http::SUFFIX_TRANSITION_TIER;
7604        source.metadata.insert(
7605            format!("{}{suffix}", rustfs_utils::http::MINIO_INTERNAL_PREFIX),
7606            source.transition_tier.clone(),
7607        );
7608        let mut target = tiered_equivalence_target(&source);
7609        Arc::make_mut(&mut target.user_defined)
7610            .insert(rustfs_utils::http::internal_key_rustfs(suffix), source.transition_tier.clone());
7611
7612        assert!(is_equivalent_data_movement_tiered_object(&source, &target));
7613    }
7614
7615    #[test]
7616    fn equivalent_data_movement_tiered_object_requires_hydrated_part_checksums() {
7617        let mut source = tiered_equivalence_source();
7618        source.parts = vec![rustfs_filemeta::ObjectPartInfo {
7619            number: 1,
7620            mod_time: Some(OffsetDateTime::UNIX_EPOCH + time::Duration::SECOND),
7621            checksums: Some(HashMap::from([("CRC32C".to_string(), "AAAAAA==".to_string())])),
7622            ..Default::default()
7623        }];
7624        rustfs_utils::http::insert_str(
7625            &mut source.metadata,
7626            rustfs_utils::http::SUFFIX_PART_CHECKSUMS,
7627            r#"[[1,[["CRC32C","AAAAAA=="]]]]"#.to_string(),
7628        );
7629        let mut target = tiered_equivalence_target(&source);
7630        Arc::make_mut(&mut target.parts)[0].mod_time = None;
7631        assert!(is_equivalent_data_movement_tiered_object(&source, &target));
7632
7633        let mut missing = target;
7634        Arc::make_mut(&mut missing.parts)[0].checksums = None;
7635        assert!(!is_equivalent_data_movement_tiered_object(&source, &missing));
7636    }
7637
7638    #[test]
7639    fn equivalent_data_movement_tiered_object_rejects_transition_mismatch() {
7640        let source = tiered_equivalence_source();
7641        let mut target = tiered_equivalence_target(&source);
7642        target.transitioned_object.name = "remote/target".to_string();
7643
7644        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7645    }
7646
7647    #[test]
7648    fn equivalent_data_movement_tiered_object_rejects_transition_version_state_mismatch() {
7649        let mut source = tiered_equivalence_source();
7650        source.transition_version_state = rustfs_filemeta::TransitionVersionState::Exact;
7651        let mut target = tiered_equivalence_target(&source);
7652        target.transition_version_state = rustfs_filemeta::TransitionVersionState::Unknown;
7653
7654        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7655    }
7656
7657    #[test]
7658    fn equivalent_data_movement_tiered_object_rejects_user_metadata_mismatch() {
7659        let source = tiered_equivalence_source();
7660        let mut target = tiered_equivalence_target(&source);
7661        target.user_defined = Arc::new(HashMap::from([("x-amz-meta-key".to_string(), "target-value".to_string())]));
7662
7663        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7664    }
7665
7666    #[test]
7667    fn equivalent_data_movement_tiered_object_rejects_tag_mismatch() {
7668        let source = tiered_equivalence_source();
7669        let mut target = tiered_equivalence_target(&source);
7670        target.user_tags = Arc::new("tag=target".to_string());
7671
7672        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7673    }
7674
7675    #[test]
7676    fn equivalent_data_movement_tiered_object_rejects_replication_mismatch() {
7677        let source = tiered_equivalence_source();
7678        let mut target = tiered_equivalence_target(&source);
7679        target.replication_status_internal = Some("arn:minio:replication:target=FAILED;".to_string());
7680        target.replication_status = ReplicationStatusType::Failed;
7681
7682        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7683    }
7684
7685    #[test]
7686    fn equivalent_data_movement_tiered_object_rejects_version_purge_mismatch() {
7687        let source = tiered_equivalence_source();
7688        let mut target = tiered_equivalence_target(&source);
7689        target.version_purge_status_internal = Some("arn:minio:replication:target=COMPLETE;".to_string());
7690        target.version_purge_status = VersionPurgeStatusType::Complete;
7691
7692        assert!(!is_equivalent_data_movement_tiered_object(&source, &target));
7693    }
7694
7695    #[test]
7696    fn data_movement_tiered_resume_accepts_equivalent_target() {
7697        let source = tiered_equivalence_source();
7698        let target = tiered_equivalence_target(&source);
7699
7700        let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 1)
7701            .expect("equivalent tiered target should be evaluated");
7702
7703        assert!(should_resume);
7704    }
7705
7706    #[test]
7707    fn data_movement_tiered_resume_rejects_source_pool_target() {
7708        let source = tiered_equivalence_source();
7709        let target = tiered_equivalence_target(&source);
7710
7711        let should_resume = resolve_data_movement_tiered_resume_result(Ok(Some(target)), &source, 0, 0)
7712            .expect("source-pool target should be rejected before target lookup");
7713
7714        assert!(!should_resume);
7715    }
7716
7717    #[test]
7718    fn data_movement_tiered_resume_rejects_missing_target() {
7719        let source = FileInfo {
7720            version_id: Some(Uuid::nil()),
7721            size: 1024,
7722            ..Default::default()
7723        };
7724
7725        let should_resume = resolve_data_movement_tiered_resume_result(Ok(None), &source, 0, 1)
7726            .expect("missing tiered target should be evaluated");
7727
7728        assert!(!should_resume);
7729    }
7730
7731    #[test]
7732    fn latest_object_access_delete_marker_error_returns_none_for_live_object() {
7733        let info = ObjectInfo::default();
7734        let opts = ObjectOptions::default();
7735
7736        assert!(latest_object_access_delete_marker_error("bucket", "object", &info, &opts).is_none());
7737    }
7738
7739    #[test]
7740    fn latest_object_access_delete_marker_error_returns_not_found_without_version_id() {
7741        let info = ObjectInfo {
7742            delete_marker: true,
7743            ..Default::default()
7744        };
7745        let opts = ObjectOptions::default();
7746
7747        let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts)
7748            .expect("delete marker should stop latest-object reads");
7749
7750        assert!(crate::error::is_err_object_not_found(&err));
7751    }
7752
7753    #[test]
7754    fn latest_object_access_delete_marker_error_returns_method_not_allowed_for_version_read() {
7755        let info = ObjectInfo {
7756            delete_marker: true,
7757            ..Default::default()
7758        };
7759        let opts = ObjectOptions {
7760            version_id: Some("vid-1".to_string()),
7761            ..Default::default()
7762        };
7763
7764        let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts)
7765            .expect("delete marker version reads should be rejected");
7766
7767        assert!(matches!(err, Error::MethodNotAllowed));
7768    }
7769
7770    #[test]
7771    fn latest_object_access_delete_marker_error_returns_not_found_for_delete_marker_lookup() {
7772        let info = ObjectInfo {
7773            delete_marker: true,
7774            ..Default::default()
7775        };
7776        let opts = ObjectOptions {
7777            version_id: Some("vid-1".to_string()),
7778            delete_marker: true,
7779            ..Default::default()
7780        };
7781
7782        let err = latest_object_access_delete_marker_error("bucket", "object", &info, &opts)
7783            .expect("delete marker lookup should keep not-found semantics");
7784
7785        assert!(crate::error::is_err_object_not_found(&err));
7786    }
7787
7788    #[test]
7789    fn resolve_latest_object_access_returns_live_object_and_pool_idx() {
7790        let info = ObjectInfo::default();
7791        let opts = ObjectOptions::default();
7792
7793        let (resolved, idx) = resolve_latest_object_access("bucket", "object", info, 7, &opts).unwrap();
7794
7795        assert_eq!(idx, 7);
7796        assert!(!resolved.delete_marker);
7797    }
7798
7799    #[test]
7800    fn resolve_latest_object_access_rejects_delete_marker_without_version_id() {
7801        let info = ObjectInfo {
7802            delete_marker: true,
7803            ..Default::default()
7804        };
7805        let opts = ObjectOptions::default();
7806
7807        let err = resolve_latest_object_access("bucket", "object", info, 2, &opts).unwrap_err();
7808
7809        assert!(crate::error::is_err_object_not_found(&err));
7810    }
7811
7812    #[test]
7813    fn resolve_latest_object_access_rejects_delete_marker_version_read() {
7814        let info = ObjectInfo {
7815            delete_marker: true,
7816            ..Default::default()
7817        };
7818        let opts = ObjectOptions {
7819            version_id: Some("vid-1".to_string()),
7820            ..Default::default()
7821        };
7822
7823        let err = resolve_latest_object_access("bucket", "object", info, 2, &opts).unwrap_err();
7824
7825        assert!(matches!(err, Error::MethodNotAllowed));
7826    }
7827
7828    #[tokio::test]
7829    async fn latest_versioned_delete_creates_a_new_marker_after_a_current_marker() {
7830        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
7831        let (_dirs, set_disks) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
7832        let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[set_disks], ctx).await);
7833        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
7834        let bucket = "versioned-delete-marker-identity";
7835        let object = "object.bin";
7836        let opts = ObjectOptions {
7837            versioned: true,
7838            ..Default::default()
7839        };
7840
7841        store
7842            .make_bucket(bucket, &MakeBucketOptions::default())
7843            .await
7844            .expect("bucket should be created");
7845
7846        let first = store
7847            .handle_delete_object(bucket, object, opts.clone())
7848            .await
7849            .expect("first versioned delete should create a marker");
7850        let second = store
7851            .handle_delete_object(bucket, object, opts)
7852            .await
7853            .expect("second versioned delete should create a marker");
7854
7855        assert!(first.delete_marker);
7856        assert!(second.delete_marker);
7857        assert_ne!(
7858            first.version_id, second.version_id,
7859            "each latest versioned delete must persist a distinct delete marker"
7860        );
7861
7862        let versions = store.pools[0]
7863            .get_disks_by_key(object)
7864            .load_file_info_versions_exact(bucket, object)
7865            .await
7866            .expect("delete-marker metadata should decode")
7867            .expect("delete-marker metadata should be persisted");
7868        let marker_ids = versions
7869            .versions
7870            .iter()
7871            .filter(|version| version.deleted)
7872            .filter_map(|version| version.version_id)
7873            .collect::<std::collections::HashSet<_>>();
7874        assert!(marker_ids.contains(&first.version_id.expect("first marker should have a version id")));
7875        assert!(marker_ids.contains(&second.version_id.expect("second marker should have a version id")));
7876        assert_eq!(marker_ids.len(), 2, "both delete markers must remain durable versions");
7877    }
7878
7879    #[tokio::test]
7880    async fn versioned_delete_quorum_failure_rolls_back_and_retries_require_quorum() {
7881        for marker_copies in [0, 2, 4] {
7882            let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
7883            let (_dirs, set_disks) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
7884            let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[set_disks], ctx).await);
7885            crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
7886            let bucket = "versioned-delete-quorum-retry";
7887            let object = "object.bin";
7888            let opts = ObjectOptions {
7889                versioned: true,
7890                ..Default::default()
7891            };
7892            let put_opts = ObjectOptions {
7893                no_lock: true,
7894                versioned: true,
7895                ..Default::default()
7896            };
7897            store
7898                .make_bucket(bucket, &MakeBucketOptions::default())
7899                .await
7900                .expect("bucket should be created");
7901            let set = store.pools[0].get_disks_by_key(object);
7902            let payload = vec![0x5a; 4096];
7903            let mut reader = PutObjReader::from_vec(payload.clone());
7904            let original = set
7905                .put_object(bucket, object, &mut reader, &put_opts)
7906                .await
7907                .expect("the versioned 4 KiB object should be written");
7908            let disks = set.disks.read().await.clone();
7909            // Reproduce both committed markers and residue from a failed older writer.
7910            let marker = rustfs_filemeta::FileInfo {
7911                name: object.to_string(),
7912                version_id: Some(Uuid::new_v4()),
7913                deleted: true,
7914                mark_deleted: true,
7915                mod_time: Some(time::OffsetDateTime::now_utc()),
7916                ..Default::default()
7917            };
7918            for disk in disks.iter().take(marker_copies).flatten() {
7919                disk.delete_version(bucket, object, marker.clone(), true, crate::disk::DeleteOptions::default())
7920                    .await
7921                    .expect("the initial marker should be persisted on the selected disks");
7922            }
7923            let mut baseline = Vec::new();
7924            for disk in disks.iter().flatten() {
7925                baseline.push(
7926                    tokio::fs::read(disk.path().join(bucket).join(object).join("xl.meta"))
7927                        .await
7928                        .expect("initial metadata should exist"),
7929                );
7930            }
7931            {
7932                let mut online = set.disks.write().await;
7933                online[2] = None;
7934                online[3] = None;
7935            }
7936            for attempt in 1..=3 {
7937                let err = store
7938                    .handle_delete_object(bucket, object, opts.clone())
7939                    .await
7940                    .expect_err("every explicit retry must fail with only two of four disks available");
7941                assert!(
7942                    matches!(
7943                        &err,
7944                        StorageError::InsufficientWriteQuorum(error_bucket, error_object)
7945                            if error_bucket == bucket && error_object == object
7946                    ),
7947                    "expected write quorum failure with {marker_copies} marker copies, attempt {attempt}: {err:?}"
7948                );
7949                for (index, disk) in disks.iter().enumerate() {
7950                    let metadata_path = disk
7951                        .as_ref()
7952                        .expect("saved disk should exist")
7953                        .path()
7954                        .join(bucket)
7955                        .join(object);
7956                    let actual = tokio::fs::read(metadata_path.join("xl.meta"))
7957                        .await
7958                        .expect("rollback must preserve readable metadata");
7959                    assert_eq!(
7960                        actual, baseline[index],
7961                        "failed DELETE must restore exact metadata on disk {index}, attempt {attempt}"
7962                    );
7963                    let mut entries = tokio::fs::read_dir(&metadata_path)
7964                        .await
7965                        .expect("the object directory should remain readable");
7966                    while let Some(entry) = entries.next_entry().await.expect("directory entries should be readable") {
7967                        assert!(
7968                            entry.file_name() != "xl.meta.bkp" && !entry.path().join("xl.meta.bkp").exists(),
7969                            "rollback must consume staged metadata backups on disk {index}"
7970                        );
7971                    }
7972                }
7973            }
7974            *set.disks.write().await = disks;
7975            if marker_copies != 0 {
7976                continue;
7977            }
7978            let committed = store
7979                .handle_delete_object(bucket, object, opts)
7980                .await
7981                .expect("DELETE should succeed after write quorum recovers");
7982            assert!(committed.delete_marker);
7983            assert_ne!(committed.version_id, marker.version_id);
7984            let mut restored = Vec::new();
7985            set.get_object_reader(
7986                bucket,
7987                object,
7988                None,
7989                HeaderMap::new(),
7990                &ObjectOptions {
7991                    version_id: original.version_id.map(|id| id.to_string()),
7992                    ..Default::default()
7993                },
7994            )
7995            .await
7996            .expect("the original version must remain readable")
7997            .stream
7998            .read_to_end(&mut restored)
7999            .await
8000            .expect("the original version should stream");
8001            assert_eq!(restored, payload);
8002        }
8003    }
8004
8005    #[test]
8006    fn latest_versioned_delete_marker_creation_excludes_specialized_deletes() {
8007        assert!(latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8008            versioned: true,
8009            ..Default::default()
8010        }));
8011        assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8012            versioned: true,
8013            version_id: Some(Uuid::new_v4().to_string()),
8014            ..Default::default()
8015        }));
8016        assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8017            versioned: true,
8018            delete_marker: true,
8019            ..Default::default()
8020        }));
8021        assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8022            versioned: true,
8023            data_movement: true,
8024            ..Default::default()
8025        }));
8026        assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8027            versioned: true,
8028            replication_request: true,
8029            ..Default::default()
8030        }));
8031        assert!(!latest_versioned_delete_creates_distinct_marker(&ObjectOptions {
8032            versioned: true,
8033            version_suspended: true,
8034            ..Default::default()
8035        }));
8036    }
8037
8038    #[test]
8039    fn should_create_delete_marker_for_missing_object_allows_latest_versioned_delete() {
8040        let opts = ObjectOptions {
8041            versioned: true,
8042            ..Default::default()
8043        };
8044
8045        assert!(should_create_delete_marker_for_missing_object(&opts));
8046    }
8047
8048    #[test]
8049    fn should_create_delete_marker_for_missing_object_rejects_specialized_deletes() {
8050        let version_delete = ObjectOptions {
8051            versioned: true,
8052            version_id: Some("vid-1".to_string()),
8053            ..Default::default()
8054        };
8055        let delete_marker_replication = ObjectOptions {
8056            versioned: true,
8057            delete_marker: true,
8058            ..Default::default()
8059        };
8060        let data_movement = ObjectOptions {
8061            versioned: true,
8062            data_movement: true,
8063            ..Default::default()
8064        };
8065
8066        assert!(!should_create_delete_marker_for_missing_object(&version_delete));
8067        assert!(!should_create_delete_marker_for_missing_object(&delete_marker_replication));
8068        assert!(!should_create_delete_marker_for_missing_object(&data_movement));
8069    }
8070
8071    // issue #4189 regression: `del_opts` pins `version_id = Uuid::nil()` on directory
8072    // keys, so deleting a ghost folder over HTTP fails the lookup with *version*-not-found
8073    // (not object-not-found). The orphan-purge guard must accept both misses, or the
8074    // ghost tree survives behind a fake 204 — the exact reported symptom.
8075    #[test]
8076    fn should_purge_orphan_dir_on_version_not_found_for_dir_key() {
8077        assert!(
8078            should_purge_orphan_dir_on_missing(&StorageError::FileVersionNotFound, "ghost/"),
8079            "the real HTTP delete path yields version-not-found on dir keys and must reach the purge"
8080        );
8081        assert!(
8082            should_purge_orphan_dir_on_missing(
8083                &StorageError::VersionNotFound("bucket".into(), "ghost/".into(), Uuid::nil().to_string()),
8084                "ghost/"
8085            ),
8086            "typed VersionNotFound on a dir key must also reach the purge"
8087        );
8088    }
8089
8090    #[test]
8091    fn should_purge_orphan_dir_on_object_not_found_for_dir_key() {
8092        assert!(should_purge_orphan_dir_on_missing(&StorageError::FileNotFound, "ghost/"));
8093        assert!(should_purge_orphan_dir_on_missing(
8094            &StorageError::ObjectNotFound("bucket".into(), "ghost/".into()),
8095            "ghost/"
8096        ));
8097    }
8098
8099    #[test]
8100    fn should_not_purge_orphan_dir_for_regular_key_or_other_errors() {
8101        // A regular (non-directory) key must never trigger a prefix purge, even on a miss.
8102        assert!(!should_purge_orphan_dir_on_missing(&StorageError::FileVersionNotFound, "regular.txt"));
8103        assert!(!should_purge_orphan_dir_on_missing(&StorageError::FileNotFound, "regular.txt"));
8104        // Non-miss errors (e.g. quorum failures) must not be masked by a purge attempt.
8105        assert!(!should_purge_orphan_dir_on_missing(&StorageError::ErasureReadQuorum, "ghost/"));
8106    }
8107
8108    #[test]
8109    fn resolve_decommission_target_pool_idx_result_passthrough_ok() {
8110        let idx = ECStore::resolve_decommission_target_pool_idx_result(Ok(3), "bucket", "object").unwrap();
8111
8112        assert_eq!(idx, 3);
8113    }
8114
8115    #[test]
8116    fn resolve_decommission_target_pool_idx_result_wraps_error_context() {
8117        let err = ECStore::resolve_decommission_target_pool_idx_result(Err(Error::other("boom")), "bucket", "object")
8118            .expect_err("expected contextual error");
8119        let rendered = err.to_string();
8120
8121        assert!(rendered.contains("failed to select decommission target pool"), "{rendered}");
8122        assert!(rendered.contains("bucket"), "{rendered}");
8123        assert!(rendered.contains("object"), "{rendered}");
8124        assert!(rendered.contains("boom"), "{rendered}");
8125    }
8126
8127    #[test]
8128    fn resolve_decommission_tiered_object_result_passthrough_ok() {
8129        ECStore::resolve_decommission_tiered_object_result(Ok(()), "bucket", "object")
8130            .expect("successful decommission result should pass through");
8131    }
8132
8133    #[test]
8134    fn resolve_decommission_tiered_object_result_wraps_error_context() {
8135        let err = ECStore::resolve_decommission_tiered_object_result(Err(Error::other("boom")), "bucket", "object")
8136            .expect_err("expected contextual error");
8137        let rendered = err.to_string();
8138
8139        assert!(rendered.contains("failed to decommission tiered object"), "{rendered}");
8140        assert!(rendered.contains("bucket"), "{rendered}");
8141        assert!(rendered.contains("object"), "{rendered}");
8142        assert!(rendered.contains("boom"), "{rendered}");
8143    }
8144
8145    #[test]
8146    fn resolve_decommission_tiered_object_result_preserves_typed_capacity_error() {
8147        let err = ECStore::resolve_decommission_tiered_object_result(
8148            Err(Error::DecommissionCapacityBlocked {
8149                message: "target gate busy".to_string(),
8150            }),
8151            "bucket",
8152            "object",
8153        )
8154        .expect_err("expected contextual error");
8155
8156        assert!(matches!(
8157            crate::data_movement::data_movement_stage_source(&err),
8158            Some(Error::DecommissionCapacityBlocked { message }) if message == "target gate busy"
8159        ));
8160    }
8161
8162    #[test]
8163    fn version_aware_lookup_opts_enables_version_aware_lookup() {
8164        let opts = ObjectOptions {
8165            version_id: Some("vid-1".to_string()),
8166            ..Default::default()
8167        };
8168
8169        let lookup_opts = version_aware_lookup_opts(&opts, true);
8170
8171        assert!(lookup_opts.no_lock);
8172        assert!(lookup_opts.metadata_chg);
8173        assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
8174    }
8175
8176    #[test]
8177    fn version_aware_lookup_opts_keeps_latest_lookup_for_unversioned_requests() {
8178        let lookup_opts = version_aware_lookup_opts(&ObjectOptions::default(), true);
8179
8180        assert!(lookup_opts.no_lock);
8181        assert!(!lookup_opts.metadata_chg);
8182        assert!(lookup_opts.version_id.is_none());
8183    }
8184
8185    #[test]
8186    fn data_movement_pool_lookup_opts_enables_version_aware_lookup_and_skip_flags() {
8187        let opts = ObjectOptions {
8188            version_id: Some("vid-1".to_string()),
8189            ..Default::default()
8190        };
8191
8192        let lookup_opts = data_movement_pool_lookup_opts(&opts, false);
8193
8194        assert!(!lookup_opts.no_lock);
8195        assert!(lookup_opts.metadata_chg);
8196        assert!(lookup_opts.skip_decommissioned);
8197        assert!(lookup_opts.skip_rebalancing);
8198        assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
8199    }
8200
8201    #[test]
8202    fn writer_pool_lookup_opts_skips_rebalance_sources() {
8203        let lookup_opts = writer_pool_lookup_opts(
8204            &ObjectOptions {
8205                version_id: Some("vid-1".to_string()),
8206                ..Default::default()
8207            },
8208            true,
8209        );
8210
8211        assert!(lookup_opts.no_lock);
8212        assert!(lookup_opts.metadata_chg);
8213        assert!(lookup_opts.skip_decommissioned);
8214        assert!(lookup_opts.skip_rebalancing);
8215        assert_eq!(lookup_opts.version_id.as_deref(), Some("vid-1"));
8216    }
8217
8218    #[test]
8219    fn ordinary_delete_lookup_includes_decommission_source_and_skips_rebalance_source() {
8220        let lookup_opts = delete_pool_lookup_opts(&ObjectOptions::default(), true);
8221
8222        assert!(lookup_opts.no_lock);
8223        assert!(!lookup_opts.skip_decommissioned);
8224        assert!(lookup_opts.skip_rebalancing);
8225
8226        let explicit_version = delete_pool_lookup_opts(
8227            &ObjectOptions {
8228                versioned: true,
8229                version_id: Some(uuid::Uuid::new_v4().to_string()),
8230                ..Default::default()
8231            },
8232            true,
8233        );
8234        assert!(!explicit_version.skip_decommissioned);
8235    }
8236
8237    #[test]
8238    fn delete_fans_out_for_unversioned_and_explicit_version_mutations() {
8239        assert!(should_delete_from_all_pools(&ObjectOptions::default(), 1));
8240        assert!(should_delete_from_all_pools(
8241            &ObjectOptions {
8242                versioned: true,
8243                version_id: Some(uuid::Uuid::new_v4().to_string()),
8244                ..Default::default()
8245            },
8246            2,
8247        ));
8248        assert!(!should_delete_from_all_pools(
8249            &ObjectOptions {
8250                versioned: true,
8251                ..Default::default()
8252            },
8253            1,
8254        ));
8255        assert!(!should_delete_from_all_pools(&ObjectOptions::default(), 0));
8256    }
8257
8258    #[test]
8259    fn batch_delete_identifies_only_latest_versioned_markers() {
8260        let versioned = DeleteReplicationConfigSnapshot::from_configs_for_test(
8261            s3s::dto::VersioningConfiguration {
8262                status: Some(s3s::dto::BucketVersioningStatus::from_static(s3s::dto::BucketVersioningStatus::ENABLED)),
8263                ..Default::default()
8264            },
8265            None,
8266        );
8267        let latest = ObjectToDelete {
8268            object_name: "latest".to_string(),
8269            ..Default::default()
8270        };
8271        assert!(batch_delete_creates_latest_marker(&latest, &versioned));
8272        assert!(!batch_delete_targets_pool(true, Some(1), 0));
8273        assert!(batch_delete_targets_pool(true, Some(1), 1));
8274        assert!(!batch_delete_targets_pool(true, Some(1), 2));
8275
8276        let explicit = ObjectToDelete {
8277            object_name: "explicit".to_string(),
8278            version_id: Some(uuid::Uuid::new_v4()),
8279            ..Default::default()
8280        };
8281        assert!(!batch_delete_creates_latest_marker(&explicit, &versioned));
8282        assert!(batch_delete_targets_pool(false, Some(1), 0));
8283
8284        let unversioned = DeleteReplicationConfigSnapshot::default();
8285        assert!(!batch_delete_creates_latest_marker(&latest, &unversioned));
8286        assert!(batch_delete_targets_pool(false, None, 0));
8287    }
8288
8289    #[test]
8290    fn batch_delete_pool_failures_override_success_in_any_pool_order() {
8291        let success = DeletedObject {
8292            object_name: "object".to_string(),
8293            found: true,
8294            ..Default::default()
8295        };
8296        let source_errors = [
8297            StorageError::ErasureWriteQuorum,
8298            StorageError::NamespaceLockQuorumUnavailable {
8299                mode: "delete_objects_commit",
8300                bucket: "bucket".to_string(),
8301                object: "object".to_string(),
8302                required: 1,
8303                achieved: 0,
8304            },
8305        ];
8306
8307        for source_error in source_errors {
8308            for source_first in [true, false] {
8309                let failed = (DeletedObject::default(), Some(source_error.clone()));
8310                let succeeded = (success.clone(), None);
8311                let pool_results = if source_first {
8312                    vec![failed, succeeded]
8313                } else {
8314                    vec![succeeded, failed]
8315                };
8316
8317                let (_, error, attempted) =
8318                    resolve_batch_delete_pool_results(None, pool_results.iter().map(|(deleted, error)| (deleted, error)));
8319
8320                assert!(attempted);
8321                assert_eq!(error, Some(source_error.clone()));
8322            }
8323        }
8324    }
8325
8326    #[test]
8327    fn batch_delete_ignores_missing_pool_only_after_another_pool_succeeds() {
8328        let success = DeletedObject {
8329            object_name: "object".to_string(),
8330            found: true,
8331            ..Default::default()
8332        };
8333        let missing_errors = [
8334            StorageError::ObjectNotFound("bucket".to_string(), "object".to_string()),
8335            StorageError::VersionNotFound("bucket".to_string(), "object".to_string(), "version".to_string()),
8336        ];
8337
8338        for missing_error in missing_errors {
8339            let missing = (DeletedObject::default(), Some(missing_error.clone()));
8340            for missing_first in [true, false] {
8341                let succeeded = (success.clone(), None);
8342                let pool_results = if missing_first {
8343                    vec![missing.clone(), succeeded]
8344                } else {
8345                    vec![succeeded, missing.clone()]
8346                };
8347                let (deleted, error, attempted) =
8348                    resolve_batch_delete_pool_results(None, pool_results.iter().map(|(deleted, error)| (deleted, error)));
8349
8350                assert!(attempted);
8351                let deleted = deleted.expect("successful pool result should be retained");
8352                assert!(deleted.found);
8353                assert_eq!(deleted.object_name, success.object_name.as_str());
8354                assert!(error.is_none());
8355            }
8356
8357            let missing_only = [missing];
8358            let (_, error, attempted) =
8359                resolve_batch_delete_pool_results(None, missing_only.iter().map(|(deleted, error)| (deleted, error)));
8360            assert!(attempted);
8361            assert_eq!(error, Some(missing_error));
8362        }
8363
8364        let silent_missing = [(DeletedObject::default(), None)];
8365        let (_, error, attempted) =
8366            resolve_batch_delete_pool_results(None, silent_missing.iter().map(|(deleted, error)| (deleted, error)));
8367        assert!(attempted);
8368        assert!(error.is_none());
8369    }
8370
8371    #[tokio::test]
8372    async fn lifecycle_null_version_cas_distinguishes_absent_and_replacement_pools() {
8373        use s3s::dto::{BucketVersioningStatus, VersioningConfiguration};
8374
8375        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
8376        let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
8377        let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
8378        let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await;
8379        let bucket = RUSTFS_META_BUCKET;
8380        let suspended_snapshot = Arc::new(DeleteReplicationConfigSnapshot::from_configs_for_test(
8381            VersioningConfiguration {
8382                status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::SUSPENDED)),
8383                ..Default::default()
8384            },
8385            None,
8386        ));
8387        let write_opts = ObjectOptions {
8388            no_lock: true,
8389            version_suspended: true,
8390            ..Default::default()
8391        };
8392        let delete_opts = ObjectOptions {
8393            version_suspended: true,
8394            delete_replication_config_snapshot: Some(Arc::clone(&suspended_snapshot)),
8395            ..Default::default()
8396        };
8397
8398        let success_object = "lifecycle-null-only-in-first-pool";
8399        let mut success_reader = PutObjReader::from_vec(b"single-pool null generation".to_vec());
8400        let success_source = first_set
8401            .put_object(bucket, success_object, &mut success_reader, &write_opts)
8402            .await
8403            .expect("the historical null generation should be written to one pool");
8404        let success_target = crate::bucket::lifecycle::bucket_lifecycle_ops::lifecycle_version_delete_target(&success_source)
8405            .expect("the null generation should produce an exact lifecycle target");
8406
8407        let (deleted, errors) = store
8408            .handle_delete_objects(bucket, vec![success_target.clone()], delete_opts.clone())
8409            .await;
8410        assert!(
8411            errors.iter().all(Option::is_none),
8412            "an absent peer pool must not override success: {errors:?}"
8413        );
8414        assert!(deleted[0].found, "the pool containing the exact null generation must report success");
8415
8416        let (deleted, errors) = store
8417            .handle_delete_objects(bucket, vec![success_target], delete_opts.clone())
8418            .await;
8419        assert!(
8420            errors.iter().all(Option::is_none),
8421            "an exact lifecycle delete retry must be idempotent: {errors:?}"
8422        );
8423        assert!(!deleted[0].found, "the retried generation should already be absent from every pool");
8424
8425        let mismatch_object = "lifecycle-null-replaced-in-second-pool";
8426        let mut original_reader = PutObjReader::from_vec(b"original null generation".to_vec());
8427        let original = first_set
8428            .put_object(bucket, mismatch_object, &mut original_reader, &write_opts)
8429            .await
8430            .expect("the original null generation should be written to the first pool");
8431        let stale_target = crate::bucket::lifecycle::bucket_lifecycle_ops::lifecycle_version_delete_target(&original)
8432            .expect("the original null generation should produce an exact lifecycle target");
8433        let mut replacement_reader = PutObjReader::from_vec(b"replacement null generation".to_vec());
8434        let replacement = second_set
8435            .put_object(bucket, mismatch_object, &mut replacement_reader, &write_opts)
8436            .await
8437            .expect("the replacement null generation should be written to the second pool");
8438        assert_ne!(original.data_dir, replacement.data_dir);
8439
8440        let (_deleted, errors) = store.handle_delete_objects(bucket, vec![stale_target], delete_opts).await;
8441        assert!(
8442            matches!(errors.as_slice(), [Some(StorageError::PreconditionFailed)]),
8443            "a different null generation in any pool must fail the aggregate CAS: {errors:?}"
8444        );
8445        let retained = second_set
8446            .get_object_info(
8447                bucket,
8448                mismatch_object,
8449                &ObjectOptions {
8450                    version_id: Some(Uuid::nil().to_string()),
8451                    version_suspended: true,
8452                    no_lock: true,
8453                    ..Default::default()
8454                },
8455            )
8456            .await
8457            .expect("the replacement null generation must remain after a stale CAS");
8458        assert_eq!(retained.data_dir, replacement.data_dir);
8459    }
8460
8461    #[test]
8462    fn data_movement_pool_lookup_opts_keeps_no_lock_for_tiered_moves() {
8463        let lookup_opts = data_movement_pool_lookup_opts(
8464            &ObjectOptions {
8465                version_id: Some("vid-1".to_string()),
8466                ..Default::default()
8467            },
8468            true,
8469        );
8470
8471        assert!(lookup_opts.no_lock);
8472        assert!(lookup_opts.metadata_chg);
8473        assert!(lookup_opts.skip_decommissioned);
8474        assert!(lookup_opts.skip_rebalancing);
8475    }
8476
8477    #[test]
8478    fn capacity_owned_unversioned_move_uses_data_movement_pool_selection() {
8479        let mut opts = ObjectOptions {
8480            data_movement: true,
8481            ..Default::default()
8482        };
8483        assert!(!uses_data_movement_pool_selection(&opts));
8484
8485        DecommissionCapacityOwner {
8486            source_pool_index: 1,
8487            operation_id: Uuid::new_v4(),
8488            generation: 2,
8489            owner_nonce: Uuid::new_v4(),
8490            mutation_id: None,
8491        }
8492        .apply_to(&mut opts);
8493
8494        assert!(uses_data_movement_pool_selection(&opts));
8495    }
8496
8497    #[test]
8498    fn transition_restore_pool_opts_skips_decommissioned_and_preserves_locking() {
8499        let lookup_opts = transition_restore_pool_opts(&ObjectOptions {
8500            no_lock: false,
8501            skip_decommissioned: false,
8502            ..Default::default()
8503        });
8504
8505        assert!(lookup_opts.skip_decommissioned);
8506        assert!(lookup_opts.skip_rebalancing);
8507        assert!(!lookup_opts.no_lock);
8508    }
8509
8510    #[test]
8511    fn transition_restore_pool_opts_rejects_ambient_no_lock() {
8512        let lookup_opts = transition_restore_pool_opts(&ObjectOptions {
8513            no_lock: true,
8514            ..Default::default()
8515        });
8516
8517        assert!(lookup_opts.skip_decommissioned);
8518        assert!(lookup_opts.skip_rebalancing);
8519        assert!(!lookup_opts.no_lock);
8520    }
8521
8522    #[test]
8523    fn delete_objects_lock_names_are_sorted_and_unique() {
8524        let objects = vec![
8525            ObjectToDelete {
8526                object_name: "beta".to_string(),
8527                ..Default::default()
8528            },
8529            ObjectToDelete {
8530                object_name: "alpha".to_string(),
8531                ..Default::default()
8532            },
8533            ObjectToDelete {
8534                object_name: "beta".to_string(),
8535                ..Default::default()
8536            },
8537        ];
8538
8539        assert_eq!(sorted_unique_delete_object_names(&objects), vec!["alpha", "beta"]);
8540    }
8541
8542    async fn new_read_lock_test_store() -> ECStore {
8543        let format = FormatV3::new(1, 2);
8544        let endpoints = vec![
8545            Endpoint::try_from("http://127.0.0.1:9000/data0").expect("first endpoint should parse"),
8546            Endpoint::try_from("http://127.0.0.1:9001/data1").expect("second endpoint should parse"),
8547        ];
8548        let pool_endpoints = PoolEndpoints {
8549            legacy: false,
8550            set_count: 1,
8551            drives_per_set: 2,
8552            endpoints: Endpoints::from(endpoints),
8553            cmd_line: "read-lock-metadata-cache-safe-test".to_string(),
8554            platform: "test".to_string(),
8555        };
8556        let endpoint_pools = EndpointServerPools::from(vec![pool_endpoints.clone()]);
8557        let sets = Sets::new(vec![None, None], &pool_endpoints, &format, 0, 1)
8558            .await
8559            .expect("test sets should be created with empty disks");
8560
8561        ECStore {
8562            id: Uuid::new_v4(),
8563            disk_map: HashMap::new(),
8564            pools: vec![sets],
8565            peer_sys: S3PeerSys::new(&endpoint_pools),
8566            pool_meta: RwLock::new(PoolMeta::default()),
8567            rebalance_meta: RwLock::new(None),
8568            decommission_cancelers: RwLock::new(Vec::new()),
8569            start_gate: Mutex::new(()),
8570            pool_meta_save_gate: Mutex::default(),
8571            ctx: crate::runtime::instance::bootstrap_ctx(),
8572            bucket_fence_registry: std::sync::Arc::default(),
8573        }
8574    }
8575
8576    #[tokio::test]
8577    async fn delete_object_handler_futures_remain_stack_bounded() {
8578        let store = new_read_lock_test_store().await;
8579        let unified_future = store.handle_delete_object_with_journal("bucket", "object", ObjectOptions::default(), None);
8580        let unified_future_size = std::mem::size_of_val(&unified_future);
8581
8582        assert!(
8583            unified_future_size <= 4 * 1024,
8584            "unified delete handler future must remain stack-bounded; measured {unified_future_size} bytes"
8585        );
8586        drop(unified_future);
8587
8588        let batch_future =
8589            store.handle_delete_objects_with_journal_and_accounting("bucket", Vec::new(), ObjectOptions::default(), None);
8590        let batch_future_size = std::mem::size_of_val(&batch_future);
8591        assert!(
8592            batch_future_size <= 4 * 1024,
8593            "batch delete handler future must remain stack-bounded; measured {batch_future_size} bytes"
8594        );
8595        drop(batch_future);
8596
8597        let outer_future = store.handle_delete_object("bucket", "object", ObjectOptions::default());
8598        let outer_future_size = std::mem::size_of_val(&outer_future);
8599        assert!(
8600            outer_future_size <= 16 * 1024,
8601            "outer delete handler future must remain stack-bounded; measured {outer_future_size} bytes"
8602        );
8603    }
8604
8605    async fn new_prepared_reader_test_store(set_disks: &[Arc<SetDisks>]) -> ECStore {
8606        new_prepared_reader_test_store_with_ctx(set_disks, crate::runtime::instance::bootstrap_ctx()).await
8607    }
8608
8609    async fn new_prepared_reader_test_store_with_ctx(
8610        set_disks: &[Arc<SetDisks>],
8611        ctx: Arc<crate::runtime::instance::InstanceContext>,
8612    ) -> ECStore {
8613        let mut pool_configs = Vec::with_capacity(set_disks.len());
8614        let mut pools = Vec::with_capacity(set_disks.len());
8615
8616        for (pool_idx, set_disks) in set_disks.iter().enumerate() {
8617            let mut endpoints = Endpoints::from(set_disks.set_endpoints.clone());
8618            for endpoint in endpoints.as_mut() {
8619                endpoint.set_pool_index(pool_idx);
8620            }
8621            let pool_config = PoolEndpoints {
8622                legacy: false,
8623                set_count: 1,
8624                drives_per_set: set_disks.set_drive_count,
8625                endpoints,
8626                cmd_line: format!("prepared-reader-test-pool-{pool_idx}"),
8627                platform: "test".to_string(),
8628            };
8629            let disks = set_disks.disks.read().await.clone();
8630            let pool = Sets::new_with_instance_ctx(
8631                disks,
8632                &pool_config,
8633                &set_disks.format,
8634                pool_idx,
8635                set_disks.default_parity_count,
8636                Arc::clone(&ctx),
8637            )
8638            .await
8639            .expect("prepared-reader test pool should be created from local disks");
8640            pool_configs.push(pool_config);
8641            pools.push(pool);
8642        }
8643
8644        new_prepared_reader_test_store_from_pools(pools, pool_configs, ctx)
8645    }
8646
8647    fn new_prepared_reader_test_store_from_pools(
8648        pools: Vec<Arc<Sets>>,
8649        pool_configs: Vec<PoolEndpoints>,
8650        ctx: Arc<crate::runtime::instance::InstanceContext>,
8651    ) -> ECStore {
8652        let endpoint_pools = EndpointServerPools::from(pool_configs);
8653        ECStore {
8654            id: Uuid::new_v4(),
8655            disk_map: HashMap::new(),
8656            pools,
8657            peer_sys: S3PeerSys::new_with_instance_ctx(&endpoint_pools, Arc::clone(&ctx)),
8658            pool_meta: RwLock::new(PoolMeta::default()),
8659            rebalance_meta: RwLock::new(None),
8660            decommission_cancelers: RwLock::new(Vec::new()),
8661            start_gate: Mutex::new(()),
8662            pool_meta_save_gate: Mutex::default(),
8663            ctx,
8664            bucket_fence_registry: std::sync::Arc::default(),
8665        }
8666    }
8667
8668    #[tokio::test]
8669    #[serial_test::serial]
8670    async fn object_lock_diagnostics_report_historical_get_blocked_by_put_commit() {
8671        use crate::diagnostics::object_lock::tests::CapturedLockEvents;
8672        use crate::object_api::WriteCompletion;
8673        use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause};
8674        use tracing::{Instrument, instrument::WithSubscriber};
8675
8676        let capture = CapturedLockEvents::default();
8677        temp_env::async_with_vars(
8678            [
8679                (rustfs_config::ENV_OBJECT_LOCK_DIAG_ENABLE, Some("true")),
8680                (rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1")),
8681            ],
8682            async {
8683                let bucket = "historical-get-lock-diagnostics";
8684                let object = "object.bin";
8685                let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
8686                let (_dirs, original) = make_local_set_disks_with_ctx(4, 2, ctx.clone()).await;
8687                let store = new_prepared_reader_test_store_with_ctx(&[original], ctx).await;
8688                let set = store.pools[0].get_disks_by_key(object);
8689                set.make_bucket(bucket, &MakeBucketOptions::default())
8690                    .await
8691                    .expect("create bucket");
8692                let opts = ObjectOptions {
8693                    versioned: true,
8694                    write_completion: WriteCompletion::TailDrained,
8695                    ..Default::default()
8696                };
8697                let mut versions = Vec::new();
8698                for value in [0x11, 0x22, 0x33] {
8699                    let info = set
8700                        .put_object(bucket, object, &mut PutObjReader::from_vec(vec![value; 8193]), &opts)
8701                        .await
8702                        .expect("seed version");
8703                    versions.push((info.version_id.expect("explicit seed version"), value));
8704                }
8705
8706                let barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::AfterNamespace);
8707                let put_set = set.clone();
8708                let put = tokio::spawn(
8709                    async move {
8710                        put_set
8711                            .put_object(bucket, object, &mut PutObjReader::from_vec(vec![0x44; 8193]), &opts)
8712                            .await
8713                    }
8714                    .instrument(tracing::info_span!("request", request_id = "diagnostic-put"))
8715                    .with_current_subscriber(),
8716                );
8717                barrier.wait_until_paused().await;
8718
8719                let historical_version = versions[0].0.to_string();
8720                let result = store
8721                    .prepare_get_object_reader(
8722                        bucket,
8723                        object,
8724                        None,
8725                        HeaderMap::new(),
8726                        &ObjectOptions {
8727                            version_id: Some(historical_version.clone()),
8728                            versioned: true,
8729                            ..Default::default()
8730                        },
8731                    )
8732                    .instrument(tracing::info_span!("request", request_id = "diagnostic-get"))
8733                    .await;
8734                assert!(
8735                    matches!(result, Err(StorageError::Lock(rustfs_lock::LockError::Timeout { .. }))),
8736                    "a held PUT commit must preserve the typed GET lock timeout"
8737                );
8738                let rows = capture.rows();
8739                let failure = rows
8740                    .iter()
8741                    .find(|row| row["fields"]["op"] == "prepare_get_object" && row["fields"]["state"] == "failed")
8742                    .expect("failed GET acquisition must be observable before error propagation");
8743                assert_eq!(failure["fields"]["requested_version_id"], historical_version);
8744                assert_eq!(failure["fields"]["failure"], "timeout");
8745                assert_eq!(failure["fields"]["timeout_ms"], 1000);
8746                assert!(failure["fields"]["acquire_ms"].as_u64().expect("wait duration") >= 1000);
8747                assert!(
8748                    failure["spans"]
8749                        .as_array()
8750                        .expect("request spans")
8751                        .iter()
8752                        .any(|span| span["request_id"] == "diagnostic-get")
8753                );
8754                let put_acquired = rows
8755                    .iter()
8756                    .rev()
8757                    .find(|row| row["fields"]["op"] == "put_object_commit" && row["fields"]["state"] == "acquired")
8758                    .expect("PUT lock holder event");
8759                assert_ne!(put_acquired["fields"]["lock_attempt_id"], failure["fields"]["lock_attempt_id"]);
8760                assert!(
8761                    !rows
8762                        .iter()
8763                        .any(|row| row["fields"]["lock_attempt_id"] == put_acquired["fields"]["lock_attempt_id"]
8764                            && row["fields"]["state"] == "guard_dropped"),
8765                    "a paused commit still owns the lock"
8766                );
8767
8768                barrier.release();
8769                let new_version = put
8770                    .await
8771                    .expect("PUT task")
8772                    .expect("new version commits")
8773                    .version_id
8774                    .expect("new version id");
8775                versions.push((new_version, 0x44));
8776                for (version_id, value) in versions {
8777                    let prepared = store
8778                        .prepare_get_object_reader(
8779                            bucket,
8780                            object,
8781                            None,
8782                            HeaderMap::new(),
8783                            &ObjectOptions {
8784                                version_id: Some(version_id.to_string()),
8785                                versioned: true,
8786                                ..Default::default()
8787                            },
8788                        )
8789                        .await
8790                        .expect("every acknowledged version remains readable");
8791                    assert_eq!(prepared.object_info().version_id, Some(version_id));
8792                    let mut reader = prepared.into_reader().await.expect("open historical body");
8793                    let mut body = Vec::new();
8794                    reader
8795                        .stream
8796                        .read_to_end(&mut body)
8797                        .await
8798                        .expect("read complete version body");
8799                    assert_eq!(body, vec![value; 8193]);
8800                }
8801            }
8802            .with_subscriber(capture.subscriber()),
8803        )
8804        .await;
8805    }
8806
8807    async fn multipool_version_test_store(bucket: &str) -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
8808        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
8809        let (mut dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
8810        let (second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
8811        dirs.extend(second_dirs);
8812        let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[first_set, second_set], ctx).await);
8813        crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
8814        store
8815            .handle_make_bucket(
8816                bucket,
8817                &MakeBucketOptions {
8818                    versioning_enabled: true,
8819                    ..Default::default()
8820                },
8821            )
8822            .await
8823            .expect("create the versioned bucket in both pools");
8824        (dirs, store)
8825    }
8826
8827    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8828    async fn retired_marker_converges_after_bucket_recreation_on_sixteen_disks() {
8829        let bucket = "retired-marker-c11";
8830        let object = "marker.bin";
8831        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
8832        let (dirs, original) = make_local_set_disks_with_ctx(16, 4, ctx.clone()).await;
8833        let store = Arc::new(new_prepared_reader_test_store_with_ctx(&[original], ctx).await);
8834        crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
8835        let create = MakeBucketOptions {
8836            versioning_enabled: true,
8837            ..Default::default()
8838        };
8839        store.handle_make_bucket(bucket, &create).await.expect("create old bucket");
8840        let old_incarnation = store.bucket_incarnation_id_from_disk(bucket).await.expect("old identity");
8841        let mut historical = Vec::new();
8842        for value in 1..=3 {
8843            historical.push(
8844                store
8845                    .put_object(
8846                        bucket,
8847                        "history.bin",
8848                        &mut PutObjReader::from_vec(vec![value; 4097]),
8849                        &ObjectOptions {
8850                            versioned: true,
8851                            ..Default::default()
8852                        },
8853                    )
8854                    .await
8855                    .expect("write old version")
8856                    .version_id
8857                    .unwrap(),
8858            );
8859        }
8860        let marker = store
8861            .delete_object(
8862                bucket,
8863                object,
8864                ObjectOptions {
8865                    versioned: true,
8866                    ..Default::default()
8867                },
8868            )
8869            .await
8870            .expect("create old marker")
8871            .version_id
8872            .expect("explicit marker version");
8873        assert!(
8874            store
8875                .handle_delete_bucket(bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
8876                .await
8877                .is_err()
8878        );
8879        assert!(
8880            !crate::bucket::retirement::is_retired(store.clone(), bucket, old_incarnation)
8881                .await
8882                .unwrap(),
8883            "failed bucket deletion cannot publish retirement"
8884        );
8885        let set = &store.pools[0].disk_set[0];
8886        let before = set.disks.read().await.clone();
8887        for disk in before.iter().flatten() {
8888            let info = disk
8889                .read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
8890                .await
8891                .expect("read original marker");
8892            assert_eq!(info.delete_marker_incarnation(), Some(old_incarnation));
8893        }
8894        // Taking four actual members offline preserves their existing xl.meta.
8895        // All deletion/recreation work below goes through the normal store API.
8896        for slot in 12..16 {
8897            set.disks.write().await[slot] = None;
8898        }
8899        for version in historical {
8900            store
8901                .delete_object(
8902                    bucket,
8903                    "history.bin",
8904                    ObjectOptions {
8905                        versioned: true,
8906                        version_id: Some(version.to_string()),
8907                        ..Default::default()
8908                    },
8909                )
8910                .await
8911                .expect("delete exact old data version with four drives offline");
8912        }
8913        store
8914            .delete_object(
8915                bucket,
8916                object,
8917                ObjectOptions {
8918                    versioned: true,
8919                    version_id: Some(marker.to_string()),
8920                    ..Default::default()
8921                },
8922            )
8923            .await
8924            .expect("delete exact old marker with four drives offline");
8925        store
8926            .handle_delete_bucket(bucket, &crate::storage_api_contracts::bucket::DeleteBucketOptions::default())
8927            .await
8928            .expect("delete empty old bucket");
8929        assert!(
8930            crate::bucket::retirement::is_retired(store.clone(), bucket, old_incarnation)
8931                .await
8932                .expect("durable retirement")
8933        );
8934        store
8935            .handle_make_bucket(bucket, &create)
8936            .await
8937            .expect("recreate same bucket name");
8938        let current = store.bucket_incarnation_id_from_disk(bucket).await.expect("new identity");
8939        assert_ne!(old_incarnation, current);
8940        let mut new_versions = Vec::new();
8941        for (key, payload) in [
8942            (object, b"new generation reused key".as_slice()),
8943            ("new.bin", b"new generation control".as_slice()),
8944        ] {
8945            let info = store
8946                .put_object(
8947                    bucket,
8948                    key,
8949                    &mut PutObjReader::from_vec(payload.to_vec()),
8950                    &ObjectOptions {
8951                        versioned: true,
8952                        ..Default::default()
8953                    },
8954                )
8955                .await
8956                .expect("write new generation");
8957            new_versions.push((key, info.version_id.unwrap(), payload));
8958        }
8959        *set.disks.write().await = before.clone();
8960        let heal = rustfs_heal_contracts::heal_channel::HealOpts {
8961            remove: true,
8962            scan_mode: rustfs_heal_contracts::heal_channel::HealScanMode::Deep,
8963            ..Default::default()
8964        };
8965        let observed = store
8966            .clone()
8967            .inner_list_object_versions(bucket, "", None, None, None, 100)
8968            .await
8969            .expect("list after rejoin");
8970        assert_eq!(observed.objects.len(), 2, "stale marker must not contaminate listing");
8971        assert!(observed.objects.iter().all(|info| !info.delete_marker));
8972        for (key, version, _) in &new_versions {
8973            let repaired = store
8974                .heal_object_with_proof(bucket, key, &version.to_string(), &heal)
8975                .await
8976                .expect("heal new generation");
8977            assert!(repaired.error.is_none(), "new version heal: {:?}", repaired.error);
8978        }
8979        let no_remove = store
8980            .heal_object_with_proof(
8981                bucket,
8982                object,
8983                &marker.to_string(),
8984                &rustfs_heal_contracts::heal_channel::HealOpts { remove: false, ..heal },
8985            )
8986            .await
8987            .expect("inspect marker without removal");
8988        assert!(no_remove.error.as_ref().is_some_and(Error::is_retired_marker_deferred));
8989        assert!(no_remove.absence.is_none());
8990        let dry = store
8991            .heal_object_with_proof(
8992                bucket,
8993                object,
8994                &marker.to_string(),
8995                &rustfs_heal_contracts::heal_channel::HealOpts { dry_run: true, ..heal },
8996            )
8997            .await
8998            .expect("dry run");
8999        assert!(dry.absence.is_none());
9000        for disk in before.iter().skip(12).flatten() {
9001            assert!(
9002                disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9003                    .await
9004                    .is_ok()
9005            );
9006        }
9007        let original_marker = before[12]
9008            .as_ref()
9009            .unwrap()
9010            .read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9011            .await
9012            .unwrap();
9013        for stamp in [None, Some(current), Some(Uuid::new_v4())] {
9014            for disk in before.iter().skip(12).flatten() {
9015                let mut changed = original_marker.clone();
9016                changed.metadata.clear();
9017                if let Some(stamp) = stamp {
9018                    changed.set_delete_marker_incarnation(stamp);
9019                }
9020                disk.write_metadata("", bucket, object, changed).await.unwrap();
9021            }
9022            let unresolved = store
9023                .heal_object_with_proof(bucket, object, &marker.to_string(), &heal)
9024                .await
9025                .unwrap();
9026            assert!(
9027                unresolved.error.as_ref().is_some_and(Error::is_retired_marker_deferred),
9028                "legacy/current/unproven markers must be deferred: {:?}",
9029                unresolved.error
9030            );
9031            assert!(unresolved.absence.is_none());
9032            for disk in before.iter().skip(12).flatten() {
9033                assert!(
9034                    disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9035                        .await
9036                        .is_ok()
9037                );
9038                disk.write_metadata("", bucket, object, original_marker.clone())
9039                    .await
9040                    .unwrap();
9041            }
9042        }
9043        set.disks.write().await[15] = None;
9044        let incomplete = store
9045            .heal_object_with_proof(bucket, object, &marker.to_string(), &heal)
9046            .await
9047            .unwrap();
9048        assert!(incomplete.error.as_ref().is_some_and(Error::is_retired_marker_deferred));
9049        assert!(incomplete.absence.is_none(), "offline targets cannot be omitted from a cleanup receipt");
9050        set.disks.write().await[15] = before[15].clone();
9051        let failure =
9052            crate::set_disk::DanglingDeleteFailure::install(bucket, object, 15, crate::disk::error::DiskError::FaultyDisk);
9053        let partial = store
9054            .heal_object_with_proof(bucket, object, &marker.to_string(), &heal)
9055            .await
9056            .unwrap();
9057        assert!(partial.error.as_ref().is_some_and(Error::is_retired_marker_deferred));
9058        assert!(
9059            partial.absence.is_none(),
9060            "fifteen successful/absent disks cannot hide one remaining marker"
9061        );
9062        assert!(
9063            before[15]
9064                .as_ref()
9065                .unwrap()
9066                .read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9067                .await
9068                .is_ok()
9069        );
9070        drop(failure);
9071        let cleaned = store
9072            .heal_object_at_incarnation(bucket, object, &marker.to_string(), current, &heal)
9073            .await
9074            .expect("cleanup retired marker");
9075        assert!(cleaned.error.is_none(), "cleanup error: {:?}", cleaned.error);
9076        let proof = cleaned.absence.expect("complete cleanup receipt");
9077        assert!(proof.removed);
9078        assert_eq!(proof.bucket_incarnation_id, current);
9079        for disk in before.iter().flatten() {
9080            assert!(matches!(
9081                disk.read_version("", bucket, object, &marker.to_string(), &crate::disk::ReadOptions::default())
9082                    .await,
9083                Err(crate::disk::error::DiskError::FileVersionNotFound | crate::disk::error::DiskError::FileNotFound)
9084            ));
9085        }
9086        let replay = store
9087            .heal_object_with_proof(bucket, object, &marker.to_string(), &heal)
9088            .await
9089            .expect("idempotent replay");
9090        assert!(replay.error.is_none());
9091        assert!(!replay.absence.expect("authoritative absence receipt").removed);
9092        for (key, version, payload) in new_versions {
9093            let mut reader = store
9094                .handle_get_object_reader(
9095                    bucket,
9096                    key,
9097                    None,
9098                    HeaderMap::new(),
9099                    &ObjectOptions {
9100                        version_id: Some(version.to_string()),
9101                        ..Default::default()
9102                    },
9103                )
9104                .await
9105                .expect("new generation still readable");
9106            let mut actual = Vec::new();
9107            reader.stream.read_to_end(&mut actual).await.expect("read new bytes");
9108            assert_eq!(actual, payload);
9109        }
9110        assert_eq!(dirs.len(), 16);
9111    }
9112
9113    #[tokio::test]
9114    async fn multipool_delete_marker_stays_with_existing_versions() {
9115        let bucket = "multipool-marker-routing";
9116        let object = "history.bin";
9117        let (_dirs, store) = multipool_version_test_store(bucket).await;
9118
9119        let mut expected_versions = Vec::new();
9120        for value in 1..=3_u8 {
9121            let written = store.pools[1]
9122                .put_object(
9123                    bucket,
9124                    object,
9125                    &mut PutObjReader::from_vec(vec![value; 4097]),
9126                    &ObjectOptions {
9127                        versioned: true,
9128                        user_defined: HashMap::from([
9129                            (rustfs_utils::http::AMZ_OBJECT_TAGGING.to_string(), format!("generation={value}")),
9130                            ("x-amz-meta-generation".to_string(), value.to_string()),
9131                        ]),
9132                        ..Default::default()
9133                    },
9134                )
9135                .await
9136                .expect("write a historical version deterministically to pool 1");
9137            expected_versions.push(written.version_id.expect("versioned PUT must acknowledge a UUID"));
9138        }
9139        let marker = store
9140            .delete_object(
9141                bucket,
9142                object,
9143                ObjectOptions {
9144                    versioned: true,
9145                    ..Default::default()
9146                },
9147            )
9148            .await
9149            .expect("delete the current version");
9150        assert!(marker.delete_marker);
9151        let marker_id = marker.version_id.expect("DELETE must acknowledge a marker UUID");
9152        assert!(!expected_versions.contains(&marker_id));
9153
9154        let local = store.pools[1]
9155            .clone()
9156            .inner_list_object_versions(bucket, object, None, None, None, 10)
9157            .await
9158            .expect("list the object-owning pool");
9159        assert_eq!(local.objects.len(), 4, "the marker must be committed beside the three existing versions");
9160        assert_eq!(local.objects[0].version_id, Some(marker_id));
9161        assert!(local.objects[0].delete_marker && local.objects[0].is_latest);
9162        let versions = store
9163            .clone()
9164            .inner_list_object_versions(bucket, object, None, None, None, 10)
9165            .await
9166            .expect("list all pools");
9167        assert_eq!(versions.objects.len(), 4);
9168        assert_eq!(versions.objects.iter().filter(|version| version.is_latest).count(), 1);
9169        for (index, version_id) in expected_versions.iter().copied().enumerate() {
9170            assert!(
9171                versions
9172                    .objects
9173                    .iter()
9174                    .any(|version| version.version_id == Some(version_id) && !version.is_latest)
9175            );
9176            let mut reader = store
9177                .handle_get_object_reader(
9178                    bucket,
9179                    object,
9180                    None,
9181                    HeaderMap::new(),
9182                    &ObjectOptions {
9183                        version_id: Some(version_id.to_string()),
9184                        ..Default::default()
9185                    },
9186                )
9187                .await
9188                .expect("historical version should remain readable");
9189            let mut payload = Vec::new();
9190            reader
9191                .stream
9192                .read_to_end(&mut payload)
9193                .await
9194                .expect("read all historical bytes");
9195            let value = u8::try_from(index + 1).expect("fixture generation fits u8");
9196            assert_eq!(payload, vec![value; 4097]);
9197            let listed = versions
9198                .objects
9199                .iter()
9200                .find(|version| version.version_id == Some(version_id))
9201                .expect("listed historical version");
9202            assert_eq!(listed.user_tags.as_str(), format!("generation={value}"));
9203            assert_eq!(listed.user_defined.get("x-amz-meta-generation"), Some(&value.to_string()));
9204        }
9205
9206        let repeated = store
9207            .delete_object(
9208                bucket,
9209                object,
9210                ObjectOptions {
9211                    versioned: true,
9212                    ..Default::default()
9213                },
9214            )
9215            .await
9216            .expect("repeat simple DELETE");
9217        assert!(repeated.delete_marker);
9218        assert_ne!(
9219            repeated.version_id,
9220            Some(marker_id),
9221            "each enabled-versioning DELETE creates a new marker"
9222        );
9223        store
9224            .delete_object(
9225                bucket,
9226                object,
9227                ObjectOptions {
9228                    versioned: true,
9229                    version_id: repeated.version_id.map(|id| id.to_string()),
9230                    ..Default::default()
9231                },
9232            )
9233            .await
9234            .expect("remove the newest marker by identity");
9235        store
9236            .delete_object(
9237                bucket,
9238                object,
9239                ObjectOptions {
9240                    versioned: true,
9241                    version_id: Some(marker_id.to_string()),
9242                    ..Default::default()
9243                },
9244            )
9245            .await
9246            .expect("remove the original marker by identity");
9247        let current = store
9248            .get_object_info(bucket, object, &ObjectOptions::default())
9249            .await
9250            .expect("previous version becomes current");
9251        assert_eq!(current.version_id, expected_versions.last().copied());
9252        assert!(!current.delete_marker);
9253    }
9254
9255    #[tokio::test]
9256    async fn multipool_existing_split_history_lists_and_paginates_without_metadata_writes() {
9257        for marker_pool in 0..2 {
9258            let bucket = format!("multipool-split-history-{marker_pool}");
9259            let object = "history.bin";
9260            let (dirs, store) = multipool_version_test_store(&bucket).await;
9261            let mut expected = Vec::new();
9262            for value in 1..=3_u8 {
9263                let version = store.pools[1 - marker_pool]
9264                    .put_object(
9265                        &bucket,
9266                        object,
9267                        &mut PutObjReader::from_vec(vec![value; 4097]),
9268                        &ObjectOptions {
9269                            versioned: true,
9270                            ..Default::default()
9271                        },
9272                    )
9273                    .await
9274                    .expect("seed history through the owning pool's normal write path");
9275                expected.push(version.version_id);
9276            }
9277            let marker = store.pools[marker_pool]
9278                .delete_object(
9279                    &bucket,
9280                    object,
9281                    ObjectOptions {
9282                        versioned: true,
9283                        ..Default::default()
9284                    },
9285                )
9286                .await
9287                .expect("reproduce the previously committed split marker with a normal pool DELETE");
9288            assert!(marker.delete_marker);
9289            expected.push(marker.version_id);
9290            expected.reverse();
9291            let mut before = Vec::new();
9292            for dir in &dirs {
9293                before.push(
9294                    tokio::fs::read(dir.path().join(&bucket).join(object).join("xl.meta"))
9295                        .await
9296                        .expect("snapshot persisted version metadata"),
9297                );
9298            }
9299            for max_keys in [1, 2, 4, 10] {
9300                let mut key_marker = None;
9301                let mut version_marker = None;
9302                let mut listed = Vec::new();
9303                let mut completed = false;
9304                for _ in 0..6 {
9305                    let page = store
9306                        .clone()
9307                        .inner_list_object_versions(&bucket, object, key_marker.clone(), version_marker.clone(), None, max_keys)
9308                        .await
9309                        .expect("read a complete merged version page");
9310                    listed.extend(page.objects);
9311                    if !page.is_truncated {
9312                        completed = true;
9313                        break;
9314                    }
9315                    assert_ne!(
9316                        (&page.next_marker, &page.next_version_idmarker),
9317                        (&key_marker, &version_marker),
9318                        "version cursor must advance"
9319                    );
9320                    key_marker = page.next_marker;
9321                    version_marker = page.next_version_idmarker;
9322                }
9323                assert!(completed, "pagination must terminate");
9324                assert_eq!(listed.iter().map(|version| version.version_id).collect::<Vec<_>>(), expected);
9325                assert!(listed[0].delete_marker && listed[0].is_latest);
9326                assert!(listed[1..].iter().all(|version| !version.delete_marker && !version.is_latest));
9327            }
9328            let visible = store
9329                .clone()
9330                .list_objects_generic(&bucket, "", None, None, 10, false)
9331                .await
9332                .expect("list current objects");
9333            assert!(visible.objects.is_empty(), "the global current marker hides the object");
9334            for (dir, before) in dirs.iter().zip(before) {
9335                assert_eq!(
9336                    tokio::fs::read(dir.path().join(&bucket).join(object).join("xl.meta"))
9337                        .await
9338                        .expect("read unchanged metadata"),
9339                    before
9340                );
9341            }
9342        }
9343    }
9344
9345    #[tokio::test]
9346    async fn multipool_conflicting_version_metadata_fails_the_listing_request() {
9347        let bucket = "multipool-version-conflict";
9348        let (_dirs, store) = multipool_version_test_store(bucket).await;
9349        store.pools[0]
9350            .put_object(
9351                bucket,
9352                "a.bin",
9353                &mut PutObjReader::from_vec(b"preceding result".to_vec()),
9354                &ObjectOptions {
9355                    versioned: true,
9356                    ..Default::default()
9357                },
9358            )
9359            .await
9360            .expect("seed an entry before the conflict");
9361        let version_id = Uuid::new_v4();
9362        let mod_time = OffsetDateTime::now_utc();
9363        let mut etags = Vec::new();
9364        for (pool_idx, value) in [(0, 1), (1, 2)] {
9365            let written = store.pools[pool_idx]
9366                .put_object(
9367                    bucket,
9368                    "z.bin",
9369                    &mut PutObjReader::from_vec(vec![value; 4097]),
9370                    &ObjectOptions {
9371                        versioned: true,
9372                        version_id: Some(version_id.to_string()),
9373                        mod_time: Some(mod_time),
9374                        ..Default::default()
9375                    },
9376                )
9377                .await
9378                .expect("persist independent conflicting copies");
9379            assert_eq!(written.version_id, Some(version_id));
9380            etags.push(written.etag);
9381        }
9382        assert_ne!(etags[0], etags[1], "fixture must contain a semantic conflict");
9383        let err = store
9384            .clone()
9385            .inner_list_object_versions(bucket, "", None, None, None, 10)
9386            .await
9387            .expect_err("partial output must not hide the merge error");
9388        assert_eq!(err, StorageError::FileCorrupt);
9389
9390        let cancellation = tokio_util::sync::CancellationToken::new();
9391        let (sender, mut receiver) = tokio::sync::mpsc::channel(1);
9392        let walk = store
9393            .clone()
9394            .walk(cancellation.clone(), bucket, "", sender, WalkOptions::default());
9395        let drain = async { while receiver.recv().await.is_some() {} };
9396        let (walk_result, ()) = tokio::time::timeout(Duration::from_secs(5), async { tokio::join!(walk, drain) })
9397            .await
9398            .expect("bounded walk output must drain and terminate on a merge error");
9399        assert_eq!(
9400            walk_result.expect_err("walk must report the same metadata conflict"),
9401            StorageError::FileCorrupt
9402        );
9403        assert!(!cancellation.is_cancelled(), "worker failure must not cancel the caller's request");
9404    }
9405
9406    #[tokio::test]
9407    async fn multipool_marker_rejects_unwritable_owner_without_falling_back() {
9408        let bucket = "multipool-unwritable-owner";
9409        let object = "history.bin";
9410        let (_dirs, store) = multipool_version_test_store(bucket).await;
9411        store.pools[1]
9412            .put_object(
9413                bucket,
9414                object,
9415                &mut PutObjReader::from_vec(vec![1; 4097]),
9416                &ObjectOptions {
9417                    versioned: true,
9418                    ..Default::default()
9419                },
9420            )
9421            .await
9422            .expect("seed the nonzero owner");
9423        *store.pool_meta.write().await = PoolMeta {
9424            pools: vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, true)],
9425            ..Default::default()
9426        };
9427        let err = store
9428            .delete_object(
9429                bucket,
9430                object,
9431                ObjectOptions {
9432                    versioned: true,
9433                    ..Default::default()
9434                },
9435            )
9436            .await
9437            .expect_err("suspended owner must reject marker creation");
9438        assert_eq!(err, StorageError::SlowDown);
9439        *store.pool_meta.write().await = PoolMeta::default();
9440
9441        let mut rebalancing = crate::services::rebalance::RebalanceStats {
9442            participating: true,
9443            ..Default::default()
9444        };
9445        rebalancing.info.status = crate::services::rebalance::RebalStatus::Started;
9446        *store.rebalance_meta.write().await = Some(crate::services::rebalance::RebalanceMeta {
9447            pool_stats: vec![crate::services::rebalance::RebalanceStats::default(), rebalancing],
9448            ..Default::default()
9449        });
9450        let error = store
9451            .delete_object(
9452                bucket,
9453                object,
9454                ObjectOptions {
9455                    versioned: true,
9456                    ..Default::default()
9457                },
9458            )
9459            .await;
9460        *store.rebalance_meta.write().await = None;
9461        assert_eq!(
9462            error.expect_err("rebalance must not hide the owner during marker lookup"),
9463            StorageError::SlowDown
9464        );
9465
9466        // Isolate object quorum from the bucket metadata preflight: disabling
9467        // a whole pool can otherwise fail before object ownership is looked up.
9468        let quorum_bucket = RUSTFS_META_BUCKET;
9469        store.pools[1]
9470            .put_object(
9471                quorum_bucket,
9472                object,
9473                &mut PutObjReader::from_vec(vec![1; 4097]),
9474                &ObjectOptions {
9475                    versioned: true,
9476                    ..Default::default()
9477                },
9478            )
9479            .await
9480            .expect("seed the object-quorum fixture");
9481        for pool in &store.pools {
9482            pool.put_object(
9483                quorum_bucket,
9484                "split.bin",
9485                &mut PutObjReader::from_vec(vec![2; 4097]),
9486                &ObjectOptions {
9487                    versioned: true,
9488                    ..Default::default()
9489                },
9490            )
9491            .await
9492            .expect("seed another key with history in both pools");
9493        }
9494
9495        let owner = &store.pools[1].disk_set[0];
9496        let healthy = owner.disks.read().await.clone();
9497        for disk in owner.disks.write().await.iter_mut().skip(1) {
9498            *disk = None;
9499        }
9500        let error = store
9501            .delete_object(
9502                quorum_bucket,
9503                object,
9504                ObjectOptions {
9505                    versioned: true,
9506                    ..Default::default()
9507                },
9508            )
9509            .await;
9510        let split_error = store
9511            .delete_object(
9512                quorum_bucket,
9513                "split.bin",
9514                ObjectOptions {
9515                    versioned: true,
9516                    ..Default::default()
9517                },
9518            )
9519            .await;
9520        *owner.disks.write().await = healthy;
9521        assert_eq!(
9522            error.expect_err("subquorum owner must not become a new-pool marker"),
9523            StorageError::ErasureWriteQuorum
9524        );
9525        assert_eq!(
9526            split_error.expect_err("a readable older pool does not prove the global current version"),
9527            StorageError::ErasureWriteQuorum
9528        );
9529        let split = store.pools[0]
9530            .clone()
9531            .inner_list_object_versions(quorum_bucket, "split.bin", None, None, None, 10)
9532            .await
9533            .expect("inspect the readable older pool");
9534        assert_eq!(split.objects.len(), 1);
9535        assert!(!split.objects[0].delete_marker);
9536        let other = store.pools[0]
9537            .clone()
9538            .inner_list_object_versions(quorum_bucket, object, None, None, None, 10)
9539            .await
9540            .expect("inspect the other pool");
9541        assert!(other.objects.is_empty());
9542        let history = store.pools[1]
9543            .clone()
9544            .inner_list_object_versions(quorum_bucket, object, None, None, None, 10)
9545            .await
9546            .expect("inspect the restored owner");
9547        assert_eq!(history.objects.len(), 1);
9548        assert!(!history.objects[0].delete_marker);
9549    }
9550
9551    #[tokio::test]
9552    async fn multipool_suspended_and_batch_markers_keep_the_null_slot_on_the_owner() {
9553        let bucket = "multipool-suspended-markers";
9554        let object = "history.bin";
9555        let (_dirs, store) = multipool_version_test_store(bucket).await;
9556        let original = store.pools[1]
9557            .put_object(
9558                bucket,
9559                object,
9560                &mut PutObjReader::from_vec(vec![1; 4097]),
9561                &ObjectOptions {
9562                    versioned: true,
9563                    ..Default::default()
9564                },
9565            )
9566            .await
9567            .expect("seed the UUID history in pool 1");
9568        store
9569            .update_bucket_metadata_config(
9570                bucket,
9571                crate::bucket::metadata::BUCKET_VERSIONING_CONFIG,
9572                b"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>".to_vec(),
9573            )
9574            .await
9575            .expect("persist suspended bucket versioning");
9576        let null = store
9577            .put_object(
9578                bucket,
9579                object,
9580                &mut PutObjReader::from_vec(vec![2; 4097]),
9581                &ObjectOptions {
9582                    version_suspended: true,
9583                    ..Default::default()
9584                },
9585            )
9586            .await
9587            .expect("write the null version beside the history");
9588        assert!(null.version_id.is_none_or(|id| id.is_nil()));
9589        for _ in 0..2 {
9590            let marker = store
9591                .delete_object(
9592                    bucket,
9593                    object,
9594                    ObjectOptions {
9595                        version_suspended: true,
9596                        ..Default::default()
9597                    },
9598                )
9599                .await
9600                .expect("replace the null slot with a marker");
9601            assert!(marker.delete_marker);
9602            assert!(marker.version_id.is_none_or(|id| id.is_nil()));
9603        }
9604        let (deleted, errors) = store
9605            .delete_objects(
9606                bucket,
9607                vec![ObjectToDelete {
9608                    object_name: object.to_string(),
9609                    ..Default::default()
9610                }],
9611                ObjectOptions::default(),
9612            )
9613            .await;
9614        assert!(errors.iter().all(Option::is_none), "batch DELETE should succeed: {errors:?}");
9615        assert_eq!(deleted.len(), 1);
9616        assert!(deleted[0].delete_marker);
9617        let versions = store.pools[1]
9618            .clone()
9619            .inner_list_object_versions(bucket, object, None, None, None, 10)
9620            .await
9621            .expect("inspect the owner after batch DELETE");
9622        assert_eq!(versions.objects.len(), 2, "only one null marker and the UUID history remain");
9623        assert!(versions.objects[0].delete_marker && versions.objects[0].is_latest);
9624        assert_eq!(versions.objects[1].version_id, original.version_id);
9625        let other = store.pools[0]
9626            .clone()
9627            .inner_list_object_versions(bucket, object, None, None, None, 10)
9628            .await
9629            .expect("inspect the unused pool");
9630        assert!(other.objects.is_empty());
9631    }
9632
9633    async fn assert_prepared_reader_blocks_writer(store: &ECStore, bucket: &str, object: &str) {
9634        assert_pool_writer_is_blocked(store, 0, bucket, object).await;
9635    }
9636
9637    async fn assert_pool_writer_is_blocked(store: &ECStore, pool_idx: usize, bucket: &str, object: &str) {
9638        let manager = Arc::clone(store.pools[pool_idx].get_disks_by_key(object).local_lock_manager_for_test());
9639        let lock = rustfs_lock::NamespaceLock::with_local_manager("prepared-reader-writer".to_string(), manager);
9640        let err = lock
9641            .get_write_lock(rustfs_lock::ObjectKey::new(bucket, object), "competing-writer", Duration::from_millis(50))
9642            .await
9643            .expect_err("prepared read lock should block the writer");
9644        assert!(matches!(err, rustfs_lock::LockError::Timeout { .. }));
9645    }
9646
9647    async fn acquire_prepared_reader_writer(store: &ECStore, bucket: &str, object: &str) -> rustfs_lock::NamespaceLockGuard {
9648        acquire_pool_writer(store, 0, bucket, object).await
9649    }
9650
9651    async fn acquire_pool_writer(
9652        store: &ECStore,
9653        pool_idx: usize,
9654        bucket: &str,
9655        object: &str,
9656    ) -> rustfs_lock::NamespaceLockGuard {
9657        let manager = Arc::clone(store.pools[pool_idx].get_disks_by_key(object).local_lock_manager_for_test());
9658        let lock = rustfs_lock::NamespaceLock::with_local_manager("prepared-reader-writer".to_string(), manager);
9659        lock.get_write_lock(rustfs_lock::ObjectKey::new(bucket, object), "competing-writer", Duration::from_secs(1))
9660            .await
9661            .expect("prepared read lock should have been released")
9662    }
9663
9664    #[tokio::test]
9665    #[serial_test::serial(body_cache_hook)]
9666    async fn prepared_reader_uses_authoritative_hook_miss_once_and_streams_full_body() {
9667        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9668        let store = new_prepared_reader_test_store(&[set_disks]).await;
9669        let bucket = "prepared-reader-hook-miss";
9670        let object = "object.bin";
9671        let payload = b"prepared-reader-hook-miss-payload-".repeat(40_000);
9672        let opts = ObjectOptions {
9673            no_lock: true,
9674            ..Default::default()
9675        };
9676
9677        store.pools[0]
9678            .make_bucket(bucket, &MakeBucketOptions::default())
9679            .await
9680            .expect("bucket should be created");
9681        let mut put_reader = PutObjReader::from_vec(payload.clone());
9682        store.pools[0]
9683            .put_object(bucket, object, &mut put_reader, &opts)
9684            .await
9685            .expect("object should be written");
9686
9687        clear_get_object_body_cache_hook();
9688        let hook = Arc::new(CountingMissHook {
9689            calls: AtomicUsize::new(0),
9690        });
9691        register_get_object_body_cache_hook(Arc::clone(&hook) as Arc<dyn GetObjectBodyCacheHook>);
9692        let _hook_guard = BodyCacheHookGuard;
9693
9694        let prepared = store
9695            .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
9696            .await
9697            .expect("prepared reader metadata should resolve");
9698        assert!(matches!(
9699            lookup_get_object_body_cache_hook(bucket, object, &None, &opts, prepared.object_info()).await,
9700            GetObjectBodyCacheHookLookup::Miss
9701        ));
9702        assert_eq!(hook.calls.load(Ordering::Relaxed), 1, "the authoritative probe should call the hook once");
9703
9704        let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
9705        assert_eq!(reader.body_source, GetObjectBodySource::HookMissed);
9706        let mut restored = Vec::new();
9707        reader
9708            .stream
9709            .read_to_end(&mut restored)
9710            .await
9711            .expect("prepared body should stream");
9712
9713        assert_eq!(restored, payload);
9714        assert_eq!(hook.calls.load(Ordering::Relaxed), 1, "reader construction must not probe the hook again");
9715    }
9716
9717    #[tokio::test]
9718    #[serial_test::serial(body_cache_hook)]
9719    async fn prepared_reader_holds_namespace_lock_until_eof_or_drop() {
9720        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
9721            let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9722            let store = new_prepared_reader_test_store(&[set_disks]).await;
9723            let bucket = "prepared-reader-lock-lifetime";
9724            let object = "object.bin";
9725            let payload = b"prepared-reader-lock-lifetime-payload-".repeat(40_000);
9726            let put_opts = ObjectOptions {
9727                no_lock: true,
9728                ..Default::default()
9729            };
9730
9731            store.pools[0]
9732                .make_bucket(bucket, &MakeBucketOptions::default())
9733                .await
9734                .expect("bucket should be created");
9735            let mut put_reader = PutObjReader::from_vec(payload.clone());
9736            store.pools[0]
9737                .put_object(bucket, object, &mut put_reader, &put_opts)
9738                .await
9739                .expect("object should be written");
9740
9741            let prepared = store
9742                .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
9743                .await
9744                .expect("prepared reader metadata should resolve");
9745            assert!(prepared.read_lock_guard.is_some());
9746            assert_prepared_reader_blocks_writer(&store, bucket, object).await;
9747
9748            let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
9749            assert_prepared_reader_blocks_writer(&store, bucket, object).await;
9750            let mut restored = Vec::new();
9751            reader
9752                .stream
9753                .read_to_end(&mut restored)
9754                .await
9755                .expect("prepared body should stream");
9756            assert_eq!(restored, payload);
9757            drop(acquire_prepared_reader_writer(&store, bucket, object).await);
9758
9759            let prepared = store
9760                .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
9761                .await
9762                .expect("second prepared reader metadata should resolve");
9763            let reader = prepared.into_reader().await.expect("second prepared body reader should open");
9764            assert_prepared_reader_blocks_writer(&store, bucket, object).await;
9765            drop(reader);
9766            drop(acquire_prepared_reader_writer(&store, bucket, object).await);
9767        })
9768        .await;
9769    }
9770
9771    #[tokio::test]
9772    #[serial_test::serial(body_cache_hook)]
9773    async fn select_snapshot_holds_namespace_lock_independent_of_get_optimization() {
9774        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
9775            let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9776            let store = new_prepared_reader_test_store(&[set_disks]).await;
9777            let bucket = "select-snapshot-lock-lifetime";
9778            let object = "object.bin";
9779            let payload = b"select-snapshot-lock-lifetime-payload-".repeat(40_000);
9780            let put_opts = ObjectOptions {
9781                no_lock: true,
9782                ..Default::default()
9783            };
9784
9785            store.pools[0]
9786                .make_bucket(bucket, &MakeBucketOptions::default())
9787                .await
9788                .expect("bucket should be created");
9789            let mut put_reader = PutObjReader::from_vec(payload.clone());
9790            store.pools[0]
9791                .put_object(bucket, object, &mut put_reader, &put_opts)
9792                .await
9793                .expect("object should be written");
9794
9795            use rustfs_utils::http::headers::{SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER};
9796            let mut request_headers = HeaderMap::new();
9797            request_headers.insert(SSEC_ALGORITHM_HEADER, "AES256".parse().expect("valid SSE-C algorithm header"));
9798            request_headers.insert(SSEC_KEY_HEADER, "secret-key".parse().expect("valid SSE-C key header"));
9799            request_headers.insert(SSEC_KEY_MD5_HEADER, "key-md5".parse().expect("valid SSE-C key digest header"));
9800            request_headers.insert("authorization", "credential".parse().expect("valid authorization header"));
9801            let snapshot = store
9802                .prepare_select_object_snapshot(bucket, object, &request_headers, &ObjectOptions::default())
9803                .await
9804                .expect("SelectObjectContent snapshot should be prepared");
9805            assert_eq!(snapshot.headers.len(), 3);
9806            assert_eq!(snapshot.headers.get(SSEC_ALGORITHM_HEADER), request_headers.get(SSEC_ALGORITHM_HEADER));
9807            assert_eq!(snapshot.headers.get(SSEC_KEY_HEADER), request_headers.get(SSEC_KEY_HEADER));
9808            assert_eq!(snapshot.headers.get(SSEC_KEY_MD5_HEADER), request_headers.get(SSEC_KEY_MD5_HEADER));
9809            assert!(snapshot.headers.get("authorization").is_none());
9810            assert!(snapshot.headers.values().all(http::HeaderValue::is_sensitive));
9811            assert!(!format!("{:?}", snapshot.headers).contains("secret-key"));
9812            assert_eq!(
9813                snapshot.logical_size(),
9814                u64::try_from(payload.len()).expect("test payload length should fit in u64")
9815            );
9816            assert_eq!(
9817                snapshot.object_info().size,
9818                i64::try_from(payload.len()).expect("test payload length should fit in i64")
9819            );
9820            assert_prepared_reader_blocks_writer(&store, bucket, object).await;
9821
9822            let mut reader = snapshot.open_reader(None).await.expect("snapshot reader should open");
9823            let mut restored = Vec::new();
9824            reader
9825                .stream
9826                .read_to_end(&mut restored)
9827                .await
9828                .expect("snapshot body should stream");
9829            assert_eq!(restored, payload);
9830            drop(reader);
9831            assert_prepared_reader_blocks_writer(&store, bucket, object).await;
9832
9833            drop(snapshot);
9834            drop(acquire_prepared_reader_writer(&store, bucket, object).await);
9835        })
9836        .await;
9837    }
9838
9839    #[tokio::test]
9840    async fn select_snapshot_identity_compares_encoded_directory_object_key() {
9841        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9842        let store = new_prepared_reader_test_store(&[set_disks]).await;
9843        let bucket = "select-snapshot-directory-identity";
9844        let source_object = "source.bin";
9845
9846        store.pools[0]
9847            .make_bucket(bucket, &MakeBucketOptions::default())
9848            .await
9849            .expect("bucket should be created");
9850        let mut put_reader = PutObjReader::from_vec(b"source".to_vec());
9851        store.pools[0]
9852            .put_object(
9853                bucket,
9854                source_object,
9855                &mut put_reader,
9856                &ObjectOptions {
9857                    no_lock: true,
9858                    ..Default::default()
9859                },
9860            )
9861            .await
9862            .expect("source object should be written");
9863
9864        let mut snapshot = store
9865            .prepare_select_object_snapshot(bucket, source_object, &HeaderMap::new(), &ObjectOptions::default())
9866            .await
9867            .expect("source object snapshot should be prepared");
9868        snapshot.object = encode_dir_object("directory/");
9869
9870        assert!(snapshot.is_for(bucket, "directory/"));
9871        assert!(!snapshot.is_for(bucket, "different/"));
9872    }
9873
9874    #[tokio::test]
9875    #[serial_test::serial(body_cache_hook)]
9876    async fn select_snapshot_reuses_initial_metadata_fanout_for_first_reader() {
9877        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9878        let store = new_prepared_reader_test_store(&[set_disks]).await;
9879        let bucket = "select-snapshot-initial-metadata";
9880        let object = "initial-metadata.bin";
9881        let payload = b"select snapshot initial metadata".repeat(4_000);
9882        let no_lock_opts = ObjectOptions {
9883            no_lock: true,
9884            ..Default::default()
9885        };
9886
9887        store.pools[0]
9888            .make_bucket(bucket, &MakeBucketOptions::default())
9889            .await
9890            .expect("bucket should be created");
9891        let mut put_reader = PutObjReader::from_vec(payload.clone());
9892        store.pools[0]
9893            .put_object(bucket, object, &mut put_reader, &no_lock_opts)
9894            .await
9895            .expect("object should be written");
9896
9897        let calls = disk_call_counters::observe(object);
9898        let snapshot = store
9899            .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
9900            .await
9901            .expect("snapshot should prepare metadata");
9902        assert_eq!(
9903            calls.total(disk_call_counters::KIND_READ_VERSION),
9904            4,
9905            "snapshot preparation should fan out metadata once"
9906        );
9907
9908        let mut reader = snapshot.open_reader(None).await.expect("first snapshot reader should open");
9909        assert_eq!(
9910            calls.total(disk_call_counters::KIND_READ_VERSION),
9911            4,
9912            "first reader should consume the metadata captured during preparation"
9913        );
9914        let mut restored = Vec::new();
9915        reader
9916            .stream
9917            .read_to_end(&mut restored)
9918            .await
9919            .expect("snapshot body should stream");
9920        assert_eq!(restored, payload);
9921    }
9922
9923    #[tokio::test]
9924    #[serial_test::serial(body_cache_hook)]
9925    async fn select_snapshot_rejects_identity_change_before_second_reader() {
9926        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
9927        let store = new_prepared_reader_test_store(&[Arc::clone(&set_disks)]).await;
9928        let bucket = "select-snapshot-identity-change";
9929        let object = "identity-change.bin";
9930        let no_lock_opts = ObjectOptions {
9931            no_lock: true,
9932            ..Default::default()
9933        };
9934
9935        set_disks
9936            .make_bucket(bucket, &MakeBucketOptions::default())
9937            .await
9938            .expect("bucket should be created");
9939        let mut initial_reader = PutObjReader::from_vec(b"first generation".to_vec());
9940        set_disks
9941            .put_object(bucket, object, &mut initial_reader, &no_lock_opts)
9942            .await
9943            .expect("initial generation should be written");
9944        let mut snapshot = store
9945            .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
9946            .await
9947            .expect("snapshot should be prepared");
9948        drop(
9949            snapshot
9950                .open_reader(None)
9951                .await
9952                .expect("first snapshot reader should consume captured metadata"),
9953        );
9954        snapshot.opts.metadata_cache_safe = false;
9955
9956        let mut replacement_reader = PutObjReader::from_vec(b"replacement generation".to_vec());
9957        set_disks
9958            .put_object(bucket, object, &mut replacement_reader, &no_lock_opts)
9959            .await
9960            .expect("test-only no-lock write should replace the object");
9961        let error = match snapshot.open_reader(None).await {
9962            Ok(_) => panic!("a later reader must reject the replacement generation"),
9963            Err(error) => error,
9964        };
9965        assert!(matches!(
9966            error,
9967            SelectObjectSnapshotReadError::Consistency(SnapshotConsistencyError::ObjectChanged)
9968        ));
9969    }
9970
9971    #[tokio::test]
9972    #[serial_test::serial(body_cache_hook)]
9973    async fn select_snapshot_rejects_latest_versioned_delete_marker_during_prepare() {
9974        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
9975        let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
9976        let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
9977        let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await;
9978        let bucket = "select-snapshot-latest-delete-marker";
9979        let object = "versioned-object.bin";
9980        let versioned_opts = ObjectOptions {
9981            no_lock: true,
9982            versioned: true,
9983            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
9984            ..Default::default()
9985        };
9986
9987        for set_disks in [&first_set, &second_set] {
9988            set_disks
9989                .make_bucket(bucket, &MakeBucketOptions::default())
9990                .await
9991                .expect("bucket should be created");
9992        }
9993        let mut older_reader = PutObjReader::from_vec(b"older visible generation".to_vec());
9994        first_set
9995            .put_object(bucket, object, &mut older_reader, &versioned_opts)
9996            .await
9997            .expect("older versioned object should be written");
9998        let mut put_reader = PutObjReader::from_vec(b"hidden generation".to_vec());
9999        second_set
10000            .put_object(bucket, object, &mut put_reader, &versioned_opts)
10001            .await
10002            .expect("versioned object should be written");
10003        let marker = second_set
10004            .delete_object(bucket, object, versioned_opts.clone())
10005            .await
10006            .expect("versioned delete should create a marker");
10007        assert!(marker.delete_marker);
10008        assert!(marker.version_id.is_some_and(|version_id| !version_id.is_nil()));
10009
10010        let error = store
10011            .prepare_select_object_snapshot(
10012                bucket,
10013                object,
10014                &HeaderMap::new(),
10015                &ObjectOptions {
10016                    versioned: true,
10017                    ..Default::default()
10018                },
10019            )
10020            .await
10021            .expect_err("latest delete marker should hide the prior object generation");
10022        assert!(matches!(
10023            error,
10024            PrepareSelectObjectSnapshotError::Storage(ref error) if is_err_object_not_found(error)
10025        ));
10026    }
10027
10028    #[tokio::test]
10029    #[serial_test::serial(storage_class_env)]
10030    async fn select_snapshot_blocks_store_delete_for_object_in_nonzero_set() {
10031        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
10032        let (_dirs, sets) = make_local_two_set_sets_with_ctx(Arc::clone(&ctx)).await;
10033        ctx.update_erasure_type(SetupType::DistErasure).await;
10034        assert!(
10035            sets.disk_set[0]
10036                .lockers
10037                .iter()
10038                .all(|first| sets.disk_set[1].lockers.iter().all(|second| !Arc::ptr_eq(first, second)))
10039        );
10040        let pool_config = sets.endpoints.clone();
10041        let store = Arc::new(new_prepared_reader_test_store_from_pools(vec![Arc::clone(&sets)], vec![pool_config], ctx));
10042        let bucket = RUSTFS_META_BUCKET;
10043        let object = (0..1_000)
10044            .map(|index| format!("nonzero-set-{index}.bin"))
10045            .find(|candidate| Arc::ptr_eq(&sets.get_disks_by_key(candidate), &sets.disk_set[1]))
10046            .expect("a key should hash to the second set");
10047        let mut put_reader = PutObjReader::from_vec(b"stable snapshot body".to_vec());
10048        sets.put_object(
10049            bucket,
10050            &object,
10051            &mut put_reader,
10052            &ObjectOptions {
10053                no_lock: true,
10054                ..Default::default()
10055            },
10056        )
10057        .await
10058        .expect("object should be written to the second set");
10059        assert!(Arc::ptr_eq(&sets.get_disks_by_key(&object), &sets.disk_set[1]));
10060
10061        let snapshot = store
10062            .prepare_select_object_snapshot(bucket, &object, &HeaderMap::new(), &ObjectOptions::default())
10063            .await
10064            .expect("snapshot should acquire both lock domains");
10065        let hashed_set_writer = rustfs_lock::NamespaceLock::with_clients_and_quorum(
10066            "select-nonzero-set-writer".to_string(),
10067            sets.disk_set[1].lockers.clone(),
10068            2,
10069        );
10070        let writer_error = hashed_set_writer
10071            .get_write_lock(
10072                rustfs_lock::ObjectKey::new(bucket, object.as_str()),
10073                "competing-hashed-set-writer",
10074                Duration::from_millis(50),
10075            )
10076            .await
10077            .expect_err("Select must hold the nonzero hashed-set read lock");
10078        assert!(matches!(writer_error, rustfs_lock::LockError::Timeout { .. }));
10079        let barrier = DeleteAfterObjectLockSnapshotBarrier::install(bucket);
10080        let delete_store = Arc::clone(&store);
10081        let delete_object = object.clone();
10082        let mut delete = tokio::spawn(async move {
10083            delete_store
10084                .delete_object(bucket, &delete_object, ObjectOptions::default())
10085                .await
10086        });
10087        barrier.wait_until_paused().await;
10088        barrier.release();
10089        assert!(
10090            tokio::time::timeout(Duration::from_millis(100), &mut delete).await.is_err(),
10091            "store DELETE must wait for the fixed-domain Select read lock"
10092        );
10093
10094        drop(snapshot);
10095        let deleted = tokio::time::timeout(Duration::from_secs(10), delete)
10096            .await
10097            .expect("DELETE should resume after the snapshot is dropped")
10098            .expect("DELETE task should not panic")
10099            .expect("DELETE should complete");
10100        assert_eq!(deleted.name, object);
10101    }
10102
10103    #[tokio::test]
10104    #[serial_test::serial]
10105    async fn select_snapshot_fails_closed_when_local_locking_is_disabled() {
10106        let ctx = Arc::new(crate::runtime::instance::InstanceContext::with_lock_manager_for_test(Arc::new(
10107            rustfs_lock::GlobalLockManager::Disabled(rustfs_lock::DisabledLockManager::new()),
10108        )));
10109        let (_dirs, set_disks) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
10110        let store = new_prepared_reader_test_store_with_ctx(&[set_disks], ctx).await;
10111        let bucket = "select-snapshot-lock-disabled";
10112        let object = "object.bin";
10113        let mut put_reader = PutObjReader::from_vec(b"payload".to_vec());
10114        let no_lock_opts = ObjectOptions {
10115            no_lock: true,
10116            ..Default::default()
10117        };
10118
10119        store.pools[0]
10120            .make_bucket(bucket, &MakeBucketOptions::default())
10121            .await
10122            .expect("bucket should be created");
10123        store.pools[0]
10124            .put_object(bucket, object, &mut put_reader, &no_lock_opts)
10125            .await
10126            .expect("object should be written without namespace locking");
10127
10128        let error = store
10129            .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
10130            .await
10131            .expect_err("SelectObjectContent must reject a disabled lock manager");
10132        assert!(matches!(
10133            error,
10134            PrepareSelectObjectSnapshotError::Consistency(SnapshotConsistencyError::LockingDisabled)
10135        ));
10136    }
10137
10138    #[tokio::test]
10139    #[serial_test::serial(body_cache_hook)]
10140    async fn prepared_object_info_releases_namespace_lock_immediately() {
10141        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10142        let store = new_prepared_reader_test_store(&[set_disks]).await;
10143        let bucket = "prepared-object-info-lock-release";
10144        let object = "object.bin";
10145        let payload = b"prepared-object-info-lock-release".to_vec();
10146        let put_opts = ObjectOptions {
10147            no_lock: true,
10148            ..Default::default()
10149        };
10150
10151        store.pools[0]
10152            .make_bucket(bucket, &MakeBucketOptions::default())
10153            .await
10154            .expect("bucket should be created");
10155        let mut put_reader = PutObjReader::from_vec(payload.clone());
10156        store.pools[0]
10157            .put_object(bucket, object, &mut put_reader, &put_opts)
10158            .await
10159            .expect("object should be written");
10160
10161        let prepared = store
10162            .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
10163            .await
10164            .expect("prepared reader metadata should resolve");
10165        assert_prepared_reader_blocks_writer(&store, bucket, object).await;
10166        assert_eq!(prepared.into_object_info().size, payload.len() as i64);
10167
10168        drop(acquire_prepared_reader_writer(&store, bucket, object).await);
10169    }
10170
10171    #[tokio::test]
10172    #[serial_test::serial(body_cache_hook)]
10173    async fn prepared_reader_reuses_metadata_across_three_pools() {
10174        let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10175        let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10176        let (_third_dirs, third_set) = make_local_set_disks(4, 2).await;
10177        let store = new_prepared_reader_test_store(&[first_set, second_set, third_set]).await;
10178        let bucket = "prepared-reader-three-pools";
10179        let object = "object.bin";
10180        let payload = b"prepared-reader-three-pool-payload-".repeat(40_000);
10181        let opts = ObjectOptions {
10182            no_lock: true,
10183            ..Default::default()
10184        };
10185
10186        for pool in &store.pools {
10187            pool.make_bucket(bucket, &MakeBucketOptions::default())
10188                .await
10189                .expect("bucket should be created in each pool");
10190        }
10191        let mut put_reader = PutObjReader::from_vec(payload.clone());
10192        store.pools[2]
10193            .put_object(bucket, object, &mut put_reader, &opts)
10194            .await
10195            .expect("object should be written only to the third pool");
10196
10197        let calls = disk_call_counters::observe(object);
10198        let prepared = store
10199            .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
10200            .await
10201            .expect("prepared reader should resolve the third-pool object");
10202        assert_eq!(prepared.object_info().size, payload.len() as i64);
10203        let metadata_calls = calls.total(disk_call_counters::KIND_READ_VERSION);
10204        assert_eq!(metadata_calls, 12, "three 4-disk pools must fan out metadata exactly once each");
10205        let mut reader = prepared.into_reader().await.expect("prepared body reader should open");
10206        assert_eq!(
10207            calls.total(disk_call_counters::KIND_READ_VERSION),
10208            metadata_calls,
10209            "the selected pool must reuse its prepared metadata"
10210        );
10211        let mut restored = Vec::new();
10212        reader
10213            .stream
10214            .read_to_end(&mut restored)
10215            .await
10216            .expect("prepared body should stream");
10217        assert_eq!(restored, payload);
10218    }
10219
10220    #[tokio::test]
10221    #[serial_test::serial(body_cache_hook)]
10222    async fn select_snapshot_reuses_metadata_across_three_pools() {
10223        let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10224        let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10225        let (_third_dirs, third_set) = make_local_set_disks(4, 2).await;
10226        let store = new_prepared_reader_test_store(&[first_set, second_set, third_set]).await;
10227        let bucket = "select-snapshot-three-pools";
10228        let object = "object.bin";
10229        let payload = b"select-snapshot-three-pool-payload-".repeat(40_000);
10230        let write_opts = ObjectOptions {
10231            no_lock: true,
10232            ..Default::default()
10233        };
10234
10235        for pool in &store.pools {
10236            pool.make_bucket(bucket, &MakeBucketOptions::default())
10237                .await
10238                .expect("bucket should be created in each pool");
10239        }
10240        let mut put_reader = PutObjReader::from_vec(payload.clone());
10241        store.pools[2]
10242            .put_object(bucket, object, &mut put_reader, &write_opts)
10243            .await
10244            .expect("object should be written only to the third pool");
10245
10246        let calls = disk_call_counters::observe(object);
10247        let snapshot = store
10248            .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
10249            .await
10250            .expect("SelectObjectContent snapshot should resolve the third-pool object");
10251        assert_eq!(snapshot.object_info().size, payload.len() as i64);
10252        let metadata_calls = calls.total(disk_call_counters::KIND_READ_VERSION);
10253        assert_eq!(metadata_calls, 12, "three 4-disk pools must fan out metadata exactly once each");
10254
10255        let mut reader = snapshot.open_reader(None).await.expect("snapshot body reader should open");
10256        assert_eq!(
10257            calls.total(disk_call_counters::KIND_READ_VERSION),
10258            metadata_calls,
10259            "SelectObjectContent must consume the prepared winner without a second fanout"
10260        );
10261        let mut restored = Vec::new();
10262        reader
10263            .stream
10264            .read_to_end(&mut restored)
10265            .await
10266            .expect("snapshot body should stream");
10267        assert_eq!(restored, payload);
10268    }
10269
10270    #[tokio::test]
10271    #[serial_test::serial(body_cache_hook)]
10272    async fn legacy_reader_reuses_selected_pool_metadata() {
10273        let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10274        let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10275        let store = new_prepared_reader_test_store(&[first_set, second_set]).await;
10276        let bucket = "legacy-reader-second-pool";
10277        let object = "object.bin";
10278        let payload = b"legacy-reader-second-pool-payload-".repeat(40_000);
10279        let opts = ObjectOptions {
10280            no_lock: true,
10281            ..Default::default()
10282        };
10283
10284        for pool in &store.pools {
10285            pool.make_bucket(bucket, &MakeBucketOptions::default())
10286                .await
10287                .expect("bucket should be created in each pool");
10288        }
10289        let mut put_reader = PutObjReader::from_vec(payload.clone());
10290        store.pools[1]
10291            .put_object(bucket, object, &mut put_reader, &opts)
10292            .await
10293            .expect("object should be written only to the second pool");
10294
10295        clear_get_object_body_cache_hook();
10296        let hook = Arc::new(CountingMissHook {
10297            calls: AtomicUsize::new(0),
10298        });
10299        register_get_object_body_cache_hook(Arc::clone(&hook) as Arc<dyn GetObjectBodyCacheHook>);
10300        let _hook_guard = BodyCacheHookGuard;
10301
10302        let calls = disk_call_counters::observe(object);
10303        let mut reader = store
10304            .handle_get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
10305            .await
10306            .expect("legacy reader should resolve the second-pool object");
10307        assert_eq!(
10308            hook.calls.load(Ordering::Relaxed),
10309            1,
10310            "legacy reader must probe the body cache exactly once"
10311        );
10312        assert_eq!(reader.body_source, GetObjectBodySource::HookMissed);
10313        assert!(
10314            calls.total(disk_call_counters::KIND_READ_VERSION) <= 8,
10315            "legacy reader must fan out each 4-disk pool at most once"
10316        );
10317        let mut restored = Vec::new();
10318        reader
10319            .stream
10320            .read_to_end(&mut restored)
10321            .await
10322            .expect("legacy reader body should stream");
10323        assert_eq!(restored, payload);
10324    }
10325
10326    fn prepared_pool_test_status(id: usize, suspended: bool) -> PoolStatus {
10327        PoolStatus {
10328            id,
10329            cmd_line: format!("prepared-pool-{id}"),
10330            last_update: OffsetDateTime::now_utc(),
10331            decommission: suspended.then(|| PoolDecommissionInfo {
10332                start_time: Some(OffsetDateTime::now_utc()),
10333                ..Default::default()
10334            }),
10335        }
10336    }
10337
10338    #[tokio::test]
10339    #[serial_test::serial(body_cache_hook)]
10340    async fn prepared_reader_refetches_when_final_pool_state_changes_winner() {
10341        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10342        let store = Arc::new(new_prepared_reader_test_store(&[Arc::clone(&set_disks), Arc::clone(&set_disks)]).await);
10343        let bucket = "prepared-reader-pool-state-fallback";
10344        let object = "object.bin";
10345        let payload = b"pool-state fallback payload".repeat(8_000);
10346        let opts = ObjectOptions {
10347            no_lock: true,
10348            ..Default::default()
10349        };
10350
10351        set_disks
10352            .make_bucket(bucket, &MakeBucketOptions::default())
10353            .await
10354            .expect("bucket should be created");
10355        let mut put_reader = PutObjReader::from_vec(payload.clone());
10356        set_disks
10357            .put_object(bucket, object, &mut put_reader, &opts)
10358            .await
10359            .expect("shared object should be written");
10360
10361        let calls = disk_call_counters::observe(object);
10362        let barrier = crate::store::rebalance::PreparedPoolReadFallbackBarrier::install(object, false);
10363        let read_store = Arc::clone(&store);
10364        let read_opts = opts.clone();
10365        let read = tokio::spawn(async move {
10366            read_store
10367                .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts)
10368                .await
10369        });
10370        barrier.wait_after_fanout().await;
10371        *store.pool_meta.write().await = PoolMeta {
10372            pools: vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, true)],
10373            ..Default::default()
10374        };
10375        barrier.release_after_fanout();
10376
10377        let prepared = read
10378            .await
10379            .expect("prepared read task should not panic")
10380            .expect("final active pool should be refetched");
10381        assert!(Arc::ptr_eq(&prepared.pool, &store.pools[0]));
10382        assert_eq!(
10383            calls.total(disk_call_counters::KIND_READ_VERSION),
10384            12,
10385            "two initial 4-disk fanouts plus one fallback refetch are required"
10386        );
10387        let mut reader = prepared.into_reader().await.expect("fallback body reader should open");
10388        let mut restored = Vec::new();
10389        reader
10390            .stream
10391            .read_to_end(&mut restored)
10392            .await
10393            .expect("fallback body should stream");
10394        assert_eq!(restored, payload);
10395    }
10396
10397    #[tokio::test]
10398    #[serial_test::serial(body_cache_hook)]
10399    async fn prepared_reader_fallback_rejects_generation_change_before_refetch() {
10400        let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
10401        let store = Arc::new(new_prepared_reader_test_store(&[Arc::clone(&set_disks), Arc::clone(&set_disks)]).await);
10402        let bucket = "prepared-reader-pool-state-generation-change";
10403        let object = "object.bin";
10404        let opts = ObjectOptions {
10405            no_lock: true,
10406            ..Default::default()
10407        };
10408
10409        set_disks
10410            .make_bucket(bucket, &MakeBucketOptions::default())
10411            .await
10412            .expect("bucket should be created");
10413        let mut initial_reader = PutObjReader::from_vec(b"initial generation".to_vec());
10414        set_disks
10415            .put_object(bucket, object, &mut initial_reader, &opts)
10416            .await
10417            .expect("initial object should be written");
10418
10419        let barrier = crate::store::rebalance::PreparedPoolReadFallbackBarrier::install(object, true);
10420        let read_store = Arc::clone(&store);
10421        let read_opts = opts.clone();
10422        let read = tokio::spawn(async move {
10423            read_store
10424                .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts)
10425                .await
10426        });
10427        barrier.wait_after_fanout().await;
10428        *store.pool_meta.write().await = PoolMeta {
10429            pools: vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, true)],
10430            ..Default::default()
10431        };
10432        barrier.release_after_fanout();
10433        barrier.wait_before_refetch().await;
10434
10435        let mut replacement_reader = PutObjReader::from_vec(b"replacement generation".to_vec());
10436        set_disks
10437            .put_object(bucket, object, &mut replacement_reader, &opts)
10438            .await
10439            .expect("replacement generation should be written before fallback refetch");
10440        barrier.release_before_refetch();
10441
10442        let error = match read.await.expect("prepared read task should not panic") {
10443            Ok(_) => panic!("changed fallback generation must not be accepted"),
10444            Err(error) => error,
10445        };
10446        assert_eq!(error, Error::ErasureReadQuorum);
10447    }
10448
10449    #[tokio::test]
10450    #[serial_test::serial(body_cache_hook)]
10451    async fn prepared_reader_rejects_latest_delete_marker_without_refetching_metadata() {
10452        let ctx = Arc::new(crate::runtime::instance::InstanceContext::new());
10453        let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
10454        let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await;
10455        let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await;
10456        let bucket = "prepared-reader-latest-delete-marker";
10457        let object = "versioned-object.bin";
10458        let versioned_opts = ObjectOptions {
10459            no_lock: true,
10460            versioned: true,
10461            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
10462            ..Default::default()
10463        };
10464
10465        for set_disks in [&first_set, &second_set] {
10466            set_disks
10467                .make_bucket(bucket, &MakeBucketOptions::default())
10468                .await
10469                .expect("bucket should be created");
10470        }
10471        let mut older_reader = PutObjReader::from_vec(b"older visible generation".to_vec());
10472        first_set
10473            .put_object(bucket, object, &mut older_reader, &versioned_opts)
10474            .await
10475            .expect("older object should be written");
10476        let mut hidden_reader = PutObjReader::from_vec(b"hidden generation".to_vec());
10477        second_set
10478            .put_object(bucket, object, &mut hidden_reader, &versioned_opts)
10479            .await
10480            .expect("newer object should be written");
10481        let marker = second_set
10482            .delete_object(bucket, object, versioned_opts.clone())
10483            .await
10484            .expect("delete marker should be committed");
10485        assert!(marker.delete_marker);
10486
10487        let calls = disk_call_counters::observe(object);
10488        let error = match store
10489            .prepare_get_object_reader(
10490                bucket,
10491                object,
10492                None,
10493                HeaderMap::new(),
10494                &ObjectOptions {
10495                    no_lock: true,
10496                    versioned: true,
10497                    ..Default::default()
10498                },
10499            )
10500            .await
10501        {
10502            Ok(_) => panic!("latest delete marker should hide the older live object"),
10503            Err(error) => error,
10504        };
10505
10506        assert!(is_err_object_not_found(&error));
10507        assert!(
10508            calls.total(disk_call_counters::KIND_READ_VERSION) <= 8,
10509            "delete-marker resolution must fan out each pool at most once"
10510        );
10511    }
10512
10513    #[tokio::test]
10514    #[serial_test::serial(body_cache_hook)]
10515    async fn prepared_reader_explicit_version_reuses_the_matching_pool_metadata() {
10516        let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10517        let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10518        let store = new_prepared_reader_test_store(&[Arc::clone(&first_set), Arc::clone(&second_set)]).await;
10519        let bucket = "prepared-reader-explicit-version";
10520        let object = "versioned-object.bin";
10521        let payload = b"explicit version from first pool".repeat(8_000);
10522        let versioned_opts = ObjectOptions {
10523            no_lock: true,
10524            versioned: true,
10525            object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
10526            ..Default::default()
10527        };
10528
10529        for set_disks in [&first_set, &second_set] {
10530            set_disks
10531                .make_bucket(bucket, &MakeBucketOptions::default())
10532                .await
10533                .expect("bucket should be created");
10534        }
10535        let mut first_reader = PutObjReader::from_vec(payload.clone());
10536        let first = first_set
10537            .put_object(bucket, object, &mut first_reader, &versioned_opts)
10538            .await
10539            .expect("requested version should be written to the first pool");
10540        let mut second_reader = PutObjReader::from_vec(b"different pool version".to_vec());
10541        second_set
10542            .put_object(bucket, object, &mut second_reader, &versioned_opts)
10543            .await
10544            .expect("a different version should be written to the second pool");
10545
10546        let requested_version = first
10547            .version_id
10548            .expect("versioned PUT should return a version id")
10549            .to_string();
10550        let read_opts = ObjectOptions {
10551            no_lock: true,
10552            versioned: true,
10553            version_id: Some(requested_version),
10554            ..Default::default()
10555        };
10556        let calls = disk_call_counters::observe(object);
10557        let prepared = store
10558            .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts)
10559            .await
10560            .expect("explicit version should resolve from the matching pool");
10561        assert_eq!(prepared.object_info().version_id, first.version_id);
10562        let metadata_calls = calls.total(disk_call_counters::KIND_READ_VERSION);
10563        assert!(metadata_calls <= 8, "explicit-version lookup must fan out each pool at most once");
10564
10565        let mut reader = prepared
10566            .into_reader()
10567            .await
10568            .expect("prepared explicit-version body should open");
10569        assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), metadata_calls);
10570        let mut restored = Vec::new();
10571        reader
10572            .stream
10573            .read_to_end(&mut restored)
10574            .await
10575            .expect("explicit-version body should stream");
10576        assert_eq!(restored, payload);
10577    }
10578
10579    #[tokio::test]
10580    #[serial_test::serial]
10581    async fn select_snapshot_locks_the_hashed_set_in_every_pool() {
10582        let (_first_dirs, first_set) = make_local_set_disks(4, 2).await;
10583        let (_second_dirs, second_set) = make_local_set_disks(4, 2).await;
10584        let store = new_prepared_reader_test_store(&[first_set, second_set]).await;
10585        let bucket = "select-snapshot-all-pools";
10586        let object = "object.bin";
10587        let payload = b"second-pool-snapshot".to_vec();
10588        let no_lock_opts = ObjectOptions {
10589            no_lock: true,
10590            ..Default::default()
10591        };
10592
10593        for pool in &store.pools {
10594            pool.make_bucket(bucket, &MakeBucketOptions::default())
10595                .await
10596                .expect("bucket should be created in each pool");
10597        }
10598        let mut put_reader = PutObjReader::from_vec(payload.clone());
10599        store.pools[1]
10600            .put_object(bucket, object, &mut put_reader, &no_lock_opts)
10601            .await
10602            .expect("object should be written only to the second pool");
10603
10604        let snapshot = store
10605            .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default())
10606            .await
10607            .expect("snapshot should resolve the second-pool object");
10608        assert_pool_writer_is_blocked(&store, 0, bucket, object).await;
10609        assert_pool_writer_is_blocked(&store, 1, bucket, object).await;
10610
10611        let mut reader = snapshot.open_reader(None).await.expect("snapshot reader should open");
10612        let mut restored = Vec::new();
10613        reader
10614            .stream
10615            .read_to_end(&mut restored)
10616            .await
10617            .expect("snapshot body should stream");
10618        assert_eq!(restored, payload);
10619        drop(reader);
10620        drop(snapshot);
10621
10622        drop(acquire_pool_writer(&store, 0, bucket, object).await);
10623        drop(acquire_pool_writer(&store, 1, bucket, object).await);
10624    }
10625
10626    // Phase 5 Slice 2 (backlog#939): the instance context flows down the whole
10627    // object graph — ECStore, its Sets, and their SetDisks must all carry the
10628    // same `Arc<InstanceContext>` in a single-instance deployment.
10629    #[tokio::test]
10630    async fn instance_context_flows_through_object_graph() {
10631        let store = new_read_lock_test_store().await;
10632
10633        let sets = store.pools.first().expect("test store has one pool");
10634        assert!(
10635            std::sync::Arc::ptr_eq(&store.ctx, sets.instance_ctx()),
10636            "Sets must carry the store's instance context"
10637        );
10638
10639        let set_disks = sets.disk_set.first().expect("pool has one set");
10640        assert!(
10641            std::sync::Arc::ptr_eq(sets.instance_ctx(), set_disks.instance_ctx()),
10642            "SetDisks must carry the Sets' instance context"
10643        );
10644    }
10645
10646    // Phase 5 Slice 3 (backlog#939): a SetDisks sources its lock manager from
10647    // its instance context (not an independent process lookup), and in a
10648    // single-instance build that context aliases the process lock-manager
10649    // singleton — so the lock namespace is unchanged.
10650    #[tokio::test]
10651    async fn set_disks_lock_manager_comes_from_instance_context() {
10652        let store = new_read_lock_test_store().await;
10653        let set_disks = store.pools[0].disk_set.first().expect("pool has one set");
10654
10655        assert!(
10656            std::sync::Arc::ptr_eq(set_disks.local_lock_manager_for_test(), &set_disks.instance_ctx().lock_manager()),
10657            "SetDisks lock manager must be sourced from its instance context"
10658        );
10659        assert!(
10660            std::sync::Arc::ptr_eq(set_disks.local_lock_manager_for_test(), &rustfs_lock::get_global_lock_manager()),
10661            "single-instance lock manager must alias the process singleton"
10662        );
10663    }
10664
10665    #[tokio::test]
10666    #[serial_test::serial]
10667    async fn delete_objects_write_locks_cover_each_unique_object() {
10668        let store = new_read_lock_test_store().await;
10669        let objects = vec![
10670            ObjectToDelete {
10671                object_name: "beta".to_string(),
10672                ..Default::default()
10673            },
10674            ObjectToDelete {
10675                object_name: "alpha".to_string(),
10676                ..Default::default()
10677            },
10678            ObjectToDelete {
10679                object_name: "beta".to_string(),
10680                ..Default::default()
10681            },
10682        ];
10683        let mut opts = ObjectOptions::default();
10684
10685        let guards = store
10686            .acquire_delete_objects_write_locks("bucket", &objects, &mut opts)
10687            .await
10688            .expect("delete object locks should be acquired");
10689
10690        assert_eq!(guards.len(), 2, "duplicate object names should share one namespace lock");
10691        assert!(opts.no_lock, "set layer should not reacquire locks already held by ECStore");
10692        assert!(
10693            opts.namespace_lock_fence.is_some(),
10694            "set layer must receive the outer write-lock loss fence"
10695        );
10696
10697        let alpha_lock = store
10698            .handle_new_ns_lock("bucket", "alpha")
10699            .await
10700            .expect("alpha namespace lock should be created");
10701        let err = alpha_lock
10702            .get_read_lock(Duration::from_millis(20))
10703            .await
10704            .expect_err("batch delete write guard should block alpha readers");
10705        assert!(matches!(err, rustfs_lock::LockError::Timeout { .. }));
10706
10707        drop(guards);
10708        alpha_lock
10709            .get_read_lock(Duration::from_secs(1))
10710            .await
10711            .expect("alpha read lock should be available after dropping batch guards");
10712    }
10713
10714    #[tokio::test]
10715    async fn acquired_read_lock_marks_metadata_cache_safe_for_set_layer() {
10716        let store = new_read_lock_test_store().await;
10717        let mut opts = ObjectOptions::default();
10718
10719        let guard = store
10720            .acquire_object_read_lock_if_needed("get_object", "bucket", "object", &mut opts)
10721            .await
10722            .expect("read lock should be acquired");
10723
10724        assert!(guard.is_some(), "read lock should be held by the outer store layer");
10725        assert!(opts.no_lock, "set layer should not reacquire the object lock");
10726        assert!(
10727            opts.metadata_cache_safe,
10728            "metadata cache is safe only because the outer store layer acquired the read lock"
10729        );
10730    }
10731
10732    #[tokio::test]
10733    async fn prelocked_read_request_does_not_mark_metadata_cache_safe() {
10734        let store = new_read_lock_test_store().await;
10735        let mut opts = ObjectOptions {
10736            no_lock: true,
10737            ..Default::default()
10738        };
10739
10740        let guard = store
10741            .acquire_object_read_lock_if_needed("get_object", "bucket", "object", &mut opts)
10742            .await
10743            .expect("prelocked read should not acquire another lock");
10744
10745        assert!(guard.is_none(), "prelocked caller should keep lock ownership outside ECStore");
10746        assert!(
10747            !opts.metadata_cache_safe,
10748            "generic no_lock callers must stay ineligible for metadata cache unless explicitly marked safe"
10749        );
10750    }
10751
10752    // NOTE: #4877's `restore_transitioned_object_waits_for_existing_reader`
10753    // was removed with the whole-copy-back write lock it asserted
10754    // (backlog#1304): restore entry no longer serializes on the object lock.
10755    // The replacement semantics — non-blocking reads during the copy-back and
10756    // fast rejection of a concurrent restore — are covered end-to-end by
10757    // `restore_object_usecase_reports_ongoing_conflict`
10758    // (rustfs/src/app/lifecycle_transition_api_test.rs), while the SetDisks
10759    // transition matrix covers the final local commit. Restore-vs-reader data
10760    // protection lives in the inner put_object/complete_multipart_upload locks.
10761    #[tokio::test]
10762    #[serial_test::serial]
10763    async fn restore_accept_guard_serializes_concurrent_accepts() {
10764        // backlog#1304: the accept guard is the compare-and-set boundary for
10765        // the restore ongoing flag — a second accept of the same object must
10766        // wait behind (here: time out on) the first.
10767        let store = Arc::new(new_read_lock_test_store().await);
10768        let _first = store
10769            .acquire_restore_accept_guard("bucket", "object")
10770            .await
10771            .expect("first accept guard should be acquired");
10772
10773        let err = temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("1"))], async {
10774            match store.acquire_restore_accept_guard("bucket", "object").await {
10775                Ok(_) => panic!("second accept guard must wait behind the first"),
10776                Err(err) => err,
10777            }
10778        })
10779        .await;
10780
10781        assert!(matches!(err, StorageError::Lock(rustfs_lock::LockError::Timeout { .. })));
10782    }
10783
10784    #[tokio::test]
10785    #[serial_test::serial]
10786    async fn restore_worker_guard_proves_generation_liveness_until_drop() {
10787        let store = Arc::new(new_read_lock_test_store().await);
10788        let operation_id = Uuid::from_u128(1);
10789
10790        let first = store
10791            .acquire_restore_worker_guard(operation_id)
10792            .await
10793            .expect("first worker must own its newly generated lock");
10794        assert!(
10795            store
10796                .try_acquire_restore_worker_guard(operation_id)
10797                .await
10798                .expect("a contended liveness probe should remain typed")
10799                .is_none(),
10800            "the same generation must remain live while its worker guard is held"
10801        );
10802
10803        drop(first);
10804        assert!(
10805            store
10806                .try_acquire_restore_worker_guard(operation_id)
10807                .await
10808                .expect("post-drop liveness probe should not fail")
10809                .is_some(),
10810            "dropping the worker guard must make the orphan generation recoverable"
10811        );
10812    }
10813
10814    #[tokio::test]
10815    #[serial_test::serial]
10816    async fn reader_lock_is_held_when_optimization_is_disabled() {
10817        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
10818            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
10819            let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
10820            let key = rustfs_lock::ObjectKey::new("bucket", "object");
10821            let read_guard = lock
10822                .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
10823                .await
10824                .expect("read lock should be acquired");
10825            let read_guard = ObjectLockDiagGuard::new(
10826                read_guard,
10827                true,
10828                "test_get_object",
10829                Some("bucket".to_string()),
10830                Some("object".to_string()),
10831                Some("reader".to_string()),
10832                ObjectLockDiagMode::Read,
10833            );
10834            let reader = GetObjectReader {
10835                stream: Box::new(Cursor::new(Vec::<u8>::new())),
10836                object_info: ObjectInfo::default(),
10837                buffered_body: None,
10838                body_source: Default::default(),
10839            };
10840
10841            let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
10842
10843            lock.get_write_lock(key.clone(), "writer", Duration::from_millis(20))
10844                .await
10845                .expect_err("reader should hold the read lock");
10846            drop(reader);
10847            lock.get_write_lock(key, "writer", Duration::from_secs(1))
10848                .await
10849                .expect("dropping the reader should release the read lock");
10850        })
10851        .await;
10852    }
10853
10854    #[tokio::test]
10855    #[serial_test::serial]
10856    async fn reader_lock_is_not_held_for_stream_when_optimization_is_enabled() {
10857        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
10858            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
10859            let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
10860            let key = rustfs_lock::ObjectKey::new("bucket", "object");
10861            let read_guard = lock
10862                .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
10863                .await
10864                .expect("read lock should be acquired");
10865            let read_guard = ObjectLockDiagGuard::new(
10866                read_guard,
10867                true,
10868                "test_get_object",
10869                Some("bucket".to_string()),
10870                Some("object".to_string()),
10871                Some("reader".to_string()),
10872                ObjectLockDiagMode::Read,
10873            );
10874            let reader = GetObjectReader {
10875                stream: Box::new(Cursor::new(vec![1, 2, 3])),
10876                object_info: ObjectInfo::default(),
10877                buffered_body: None,
10878                body_source: Default::default(),
10879            };
10880
10881            let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
10882
10883            lock.get_write_lock(key, "writer", Duration::from_secs(1))
10884                .await
10885                .expect("lock optimization should release the read lock before returning the stream");
10886            drop(reader);
10887        })
10888        .await;
10889    }
10890
10891    #[tokio::test]
10892    #[serial_test::serial]
10893    async fn reader_lock_is_not_held_for_buffered_body_when_optimization_is_enabled() {
10894        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true"))], async {
10895            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
10896            let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
10897            let key = rustfs_lock::ObjectKey::new("bucket", "object");
10898            let read_guard = lock
10899                .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
10900                .await
10901                .expect("read lock should be acquired");
10902            let read_guard = ObjectLockDiagGuard::new(
10903                read_guard,
10904                true,
10905                "test_get_object",
10906                Some("bucket".to_string()),
10907                Some("object".to_string()),
10908                Some("reader".to_string()),
10909                ObjectLockDiagMode::Read,
10910            );
10911            let reader = GetObjectReader {
10912                stream: Box::new(Cursor::new(vec![1, 2, 3])),
10913                object_info: ObjectInfo::default(),
10914                buffered_body: Some(Bytes::from_static(b"123")),
10915                body_source: Default::default(),
10916            };
10917
10918            let reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
10919
10920            lock.get_write_lock(key, "writer", Duration::from_secs(1))
10921                .await
10922                .expect("buffered reader should release the read lock immediately");
10923            drop(reader);
10924        })
10925        .await;
10926    }
10927
10928    #[tokio::test]
10929    #[serial_test::serial]
10930    async fn reader_lock_is_released_after_stream_eof() {
10931        temp_env::async_with_vars([(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("false"))], async {
10932            let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
10933            let lock = rustfs_lock::NamespaceLock::with_local_manager("test".to_string(), manager);
10934            let key = rustfs_lock::ObjectKey::new("bucket", "object");
10935            let read_guard = lock
10936                .get_read_lock(key.clone(), "reader", Duration::from_secs(1))
10937                .await
10938                .expect("read lock should be acquired");
10939            let read_guard = ObjectLockDiagGuard::new(
10940                read_guard,
10941                true,
10942                "test_get_object",
10943                Some("bucket".to_string()),
10944                Some("object".to_string()),
10945                Some("reader".to_string()),
10946                ObjectLockDiagMode::Read,
10947            );
10948            let reader = GetObjectReader {
10949                stream: Box::new(Cursor::new(vec![1, 2, 3])),
10950                object_info: ObjectInfo::default(),
10951                buffered_body: None,
10952                body_source: Default::default(),
10953            };
10954
10955            let mut reader = ECStore::attach_read_lock_guard(reader, Some(read_guard));
10956            let mut output = Vec::new();
10957            reader.stream.read_to_end(&mut output).await.expect("reader should reach EOF");
10958            assert_eq!(output, vec![1, 2, 3]);
10959
10960            lock.get_write_lock(key, "writer", Duration::from_secs(1))
10961                .await
10962                .expect("EOF should release the read lock before the reader is dropped");
10963            drop(reader);
10964        })
10965        .await;
10966    }
10967}